diff --git a/Robustness_Eval/CLEVER/COPYRIGHT b/Robustness_Eval/CLEVER/COPYRIGHT new file mode 100644 index 0000000..8446fad --- /dev/null +++ b/Robustness_Eval/CLEVER/COPYRIGHT @@ -0,0 +1,25 @@ +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Huan Zhang and Lily Weng +Copyright (c) 2016, Nicholas Carlini + +LICENSE + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Robustness_Eval/CLEVER/LICENSE b/Robustness_Eval/CLEVER/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/Robustness_Eval/CLEVER/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Robustness_Eval/CLEVER/clever.py b/Robustness_Eval/CLEVER/clever.py new file mode 100644 index 0000000..0c12168 --- /dev/null +++ b/Robustness_Eval/CLEVER/clever.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +clever.py +Compute CLEVER score using collected Lipschitz constants +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" + +import os +import sys +import glob +from functools import partial +from multiprocessing import Pool +import scipy +import scipy.io as sio +from scipy.stats import weibull_min +import scipy.optimize +import numpy as np +import argparse +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +# We observe that the scipy.optimize.fmin optimizer (using Nelder–Mead method) +# sometimes diverges to very large parameters a, b and c. Thus, we add a very +# small regularization to the MLE optimization process to avoid this divergence +def fmin_with_reg(func, x0, args=(), xtol=1e-4, ftol=1e-4, maxiter=None, maxfun=None, + full_output=0, disp=1, retall=0, callback=None, initial_simplex=None, shape_reg = 0.01): + # print('my optimier with shape regularizer = {}'.format(shape_reg)) + def func_with_reg(theta, x): + shape = theta[2] + log_likelyhood = func(theta, x) + reg = shape_reg * shape * shape + # penalize the shape parameter + return log_likelyhood + reg + return scipy.optimize.fmin(func_with_reg, x0, args, xtol, ftol, maxiter, maxfun, + full_output, disp, retall, callback, initial_simplex) + +# fit using weibull_min.fit and run a K-S test +def fit_and_test(rescaled_sample, sample, loc_shift, shape_rescale, optimizer, c_i): + [c, loc, scale] = weibull_min.fit(-rescaled_sample, c_i, optimizer=optimizer) + loc = - loc_shift + loc * shape_rescale + scale *= shape_rescale + ks, pVal = scipy.stats.kstest(-sample, 'weibull_min', args = (c, loc, scale)) + return c, loc, scale, ks, pVal + +def plot_weibull(sample,c,loc,scale,ks,pVal,p,q,figname): + + # compare the sample histogram and fitting result + fig, ax = plt.subplots(1,1) + + x = np.linspace(-1.01*max(sample),-0.99*min(sample),100); + ax.plot(x,weibull_min.pdf(x,c,loc,scale),'r-',label='fitted pdf '+p+'-bnd') + ax.hist(-sample, density=True, bins=20, histtype='stepfilled') + ax.legend(loc='best', frameon=False) + plt.xlabel('-Lips_'+q) + plt.ylabel('pdf') + plt.title('c = {:.2f}, loc = {:.2f}, scale = {:.2f}, ks = {:.2f}, pVal = {:.2f}'.format(c,loc,scale,ks,pVal)) + plt.savefig(figname) + plt.close() + #model = figname.split("_")[1] + #plt.savefig('./debug/'+model+'/'+figname) + #plt.show() # can be used to pause the program + +# We observe than the MLE estimator in scipy sometimes can converge to a bad +# value if the inital shape parameter c is too far from the true value. Thus we +# test a few different initializations and choose the one with best p-value all +# the initializations are tested in parallel; remove some of them to speedup +# computation. +# c_init = [0.01,0.1,0.5,1,5,10,20,50,70,100,200] +c_init = [0.1,1,5,10,20,50,100] + +def get_best_weibull_fit(sample, use_reg = False, shape_reg = 0.01): + + # initialize dictionary to save the fitting results + fitted_paras = {"c":[], "loc":[], "scale": [], "ks": [], "pVal": []} + # reshape the data into a better range + # this helps the MLE solver find the solution easier + loc_shift = np.amax(sample) + dist_range = np.amax(sample) - np.amin(sample) + # if dist_range > 2.5: + shape_rescale = dist_range + # else: + # shape_rescale = 1.0 + print("shape rescale = {}".format(shape_rescale)) + rescaled_sample = np.copy(sample) + rescaled_sample -= loc_shift + rescaled_sample /= shape_rescale + + print("loc_shift = {}".format(loc_shift)) + ##print("rescaled_sample = {}".format(rescaled_sample)) + + # fit weibull distn: sample follows reverse weibull dist, so -sample follows weibull distribution + if use_reg: + results = pool.map(partial(fit_and_test, rescaled_sample, sample, loc_shift, shape_rescale, partial(fmin_with_reg, shape_reg = shape_reg)), c_init) + else: + results = pool.map(partial(fit_and_test, rescaled_sample, sample, loc_shift, shape_rescale, scipy.optimize.fmin), c_init) + + for res, c_i in zip(results, c_init): + c = res[0] + loc = res[1] + scale = res[2] + ks = res[3] + pVal = res[4] + print("[DEBUG][L2] c_init = {:5.5g}, fitted c = {:6.2f}, loc = {:7.2f}, scale = {:7.2f}, ks = {:4.2f}, pVal = {:4.2f}, max = {:7.2f}".format(c_i,c,loc,scale,ks,pVal,loc_shift)) + + ## plot every fitted result + #plot_weibull(sample,c,loc,scale,ks,pVal,p) + + fitted_paras['c'].append(c) + fitted_paras['loc'].append(loc) + fitted_paras['scale'].append(scale) + fitted_paras['ks'].append(ks) + fitted_paras['pVal'].append(pVal) + + + # get the paras of best pVal among c_init + max_pVal = np.nanmax(fitted_paras['pVal']) + if np.isnan(max_pVal) or max_pVal < 0.001: + print("ill-conditioned samples. Using maximum sample value.") + # handle the ill conditioned case + return -1, -1, -max(sample), -1, -1, -1 + + max_pVal_idx = fitted_paras['pVal'].index(max_pVal) + + c_init_best = c_init[max_pVal_idx] + c_best = fitted_paras['c'][max_pVal_idx] + loc_best = fitted_paras['loc'][max_pVal_idx] + scale_best = fitted_paras['scale'][max_pVal_idx] + ks_best = fitted_paras['ks'][max_pVal_idx] + pVal_best = fitted_paras['pVal'][max_pVal_idx] + + return c_init_best, c_best, loc_best, scale_best, ks_best, pVal_best + + +# G_max is the input array of max values +# Return the Weibull position parameter +def get_lipschitz_estimate(G_max, norm = "L2", figname = "", use_reg = False, shape_reg = 0.01): + global plot_res + c_init, c, loc, scale, ks, pVal = get_best_weibull_fit(G_max, use_reg, shape_reg) + + # the norm here is Lipschitz constant norm, not the bound's norm + if norm == "L1": + p = "i"; q = "1" + elif norm == "L2": + p = "2"; q = "2" + elif norm == "Li": + p = "1"; q = "i" + else: + print("Lipschitz norm is not in 1, 2, i!") + + if plot_res is not None: + plot_res.get() + + # plot_weibull(G_max,c,loc,scale,ks,pVal,p,q,figname) + if figname: + figname = figname + '_'+ "L"+ p + ".png" + plot_res = pool.apply_async(plot_weibull, (G_max,c,loc,scale,ks,pVal,p,q,figname)) + + return {'Lips_est':-loc, 'shape':c, 'loc': loc, 'scale': scale, 'ks': ks, 'pVal': pVal} + #return np.max(G_max) + +# file name contains some information, like true_id, true_label and target_label +def parse_filename(filename): + basename = os.path.basename(filename) + name, _ = os.path.splitext(basename) + name_arr = name.split('_') + Nsamp = int(name_arr[0]) + Niters = int(name_arr[1]) + true_id = int(name_arr[2]) + true_label = int(name_arr[3]) + target_label = int(name_arr[4]) + image_info = name_arr[5] + activation = name_arr[6] + order = name_arr[7][-1] + return Nsamp, Niters, true_id, true_label, target_label, image_info, activation, order + +if __name__ == "__main__": + # parse command line parameters + parser = argparse.ArgumentParser(description='Compute CLEVER scores using collected gradient norm data.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('data_folder', help='data folder path') + parser.add_argument('--min', dest='reduce_op', action='store_const', + default=lambda x: sum(x) / len(x) if len(x) > 0 else 0, const=min, + help='report min of all CLEVER scores instead of avg') + parser.add_argument('--user_type', + default="", + help='replace user type with string, used for ImageNet data processing') + parser.add_argument('--use_slope', + action="store_true", + help='report slope estimate. To use this option, collect_gradients.py needs to be run with --compute_slope') + parser.add_argument('--untargeted', + action="store_true", + help='process untargeted attack results (for MNIST and CIFAR)') + parser.add_argument('--num_samples', + type=int, + default=0, + help='the number of samples to use. Default 0 is to use all samples') + parser.add_argument('--num_images', + type=int, + default=0, + help='number of images to use, 0 to use all images') + parser.add_argument('--shape_reg', + default=0.01, + type=float, + help='to avoid the MLE solver in Scipy to diverge, we add a small regularization (default 0.01 is sufficient)') + parser.add_argument('--nthreads', + default=0, + type=int, + help='number of threads (default is len(c_init)+1)') + parser.add_argument('--plot_dir', + default='', + help='output path for weibull fit figures (empty to disable)') + parser.add_argument('--method', + default="mle_reg", + choices=['mle','mle_reg','maxsamp'], + help='Fitting algorithm. Please use mle_reg for best results') + args = vars(parser.parse_args()) + reduce_op = args['reduce_op'] + if args['plot_dir']: + os.system("mkdir -p " + args['plot_dir']) + print(args) + + # create thread pool + if args['nthreads'] == 0: + args['nthreads'] = len(c_init) + 1 + print("using {} threads".format(args['nthreads'])) + pool = Pool(processes = args['nthreads']) + # pool = Pool(1) + # used for asynchronous plotting in background + plot_res = None + + # get a list of all '.mat' files in folder + file_list = glob.glob(args['data_folder'] + '/**/*.mat', recursive = True) + # sort by image ID, then by information (least likely, random, top-2) + file_list = sorted(file_list, key = lambda x: (parse_filename(x)[2], parse_filename(x)[5])) + # get the first num_images files + if args['num_images']: + file_list = file_list[:args['num_images']] + + if args['untargeted']: + bounds = {} + # bounds will be inserted per image + else: + # aggregate information for three different types: least, random and top2 + # each has three bounds: L1, L2, and Linf + bounds = {"least" : [[], [], []], + "random": [[], [], []], + "top2" : [[], [], []]} + + for fname in file_list: + nsamps, niters, true_id, true_label, target_label, img_info, activation, order = parse_filename(fname) + + # keys in mat: + # ['Li_max', 'pred', 'G1_max', 'g_x0', 'path', 'info', 'G2_max', 'true_label', 'args', 'L1_max', 'Gi_max', 'L2_max', 'id', 'target_label'] + mat = sio.loadmat(fname) + print('loading {}'.format(fname)) + + if order == "1" and args['use_slope']: + G1_max = np.squeeze(mat['L1_max']) + G2_max = np.squeeze(mat['L2_max']) + Gi_max = np.squeeze(mat['Li_max']) + elif order == "1": + G1_max = np.squeeze(mat['G1_max']) + G2_max = np.squeeze(mat['G2_max']) + Gi_max = np.squeeze(mat['Gi_max']) + elif order == "2": + """ For Jun 25 experiments: forgot to save g_x0_grad_2_norm, so rerun a 1 sample 1 iterations cases "1_1_*.mat" and load g_x0_grad_2_norm from it + fname_ref = os.path.dirname(fname)+'_1/'+"1_1_"+str(true_id)+"_"+str(true_label)+"_"+str(target_label)+"_"+img_info+"_"+activation+"_order2.mat" + ##fname_ref = 'lipschitz_mat/mnist_normal/'+"1_1_"+str(true_id)+"_"+str(true_label)+"_"+str(target_label)+"_"+img_info+"_"+activation+"_order2.mat" + print("loading {}".format(fname_ref)) + mat_ref = sio.loadmat(fname_ref) + g_x0_grad_2_norm = np.squeeze(mat_ref['g_x0_grad_2_norm']) + print("g_x0_grad_2_norm = {}".format(g_x0_grad_2_norm)) + + #import time + #time.sleep(30) + """ + G2_max = np.abs(np.squeeze(mat['H2_max'])) # forgot to add abs when save in mat file + G1_max = -1*np.empty_like(G2_max) # currently only implemented 2nd order bound for p = 2 + Gi_max = -1*np.empty_like(G2_max) + g_x0_grad_2_norm = np.squeeze(mat['g_x0_grad_2_norm']) + else: + raise RuntimeError('!!! order is {}'.format(order)) + + if args['num_samples'] != 0: + prev_len = len(G1_max) + G1_max = G1_max[:args['num_samples']] + G2_max = G2_max[:args['num_samples']] + Gi_max = Gi_max[:args['num_samples']] + print('Using {} out of {} total samples'.format(len(G1_max), prev_len)) + g_x0 = np.squeeze(mat['g_x0']) + target_label = np.squeeze(mat['target_label']) + true_id = np.squeeze(mat['id']) + true_label = np.squeeze(mat['true_label']) + img_info = mat['info'][0] + if args['user_type'] != "" and img_info == "user": + img_info = args['user_type'] + + # get the filename (.mat) + print('[Filename] {}'.format(fname)) + # get the model name (inception, cifar_2-layer) + possible_names = ["mnist", "cifar", "mobilenet", "inception", "resnet"] + model = "unknown" + for path_seg in args["data_folder"].split("/"): + for n in possible_names: + if n in path_seg: + model = path_seg.replace('_', '-') + break + # model = args["data_folder"].split("/")[1] + + if args['num_samples'] == 0: # default, use all G1_max + figname = 'Fig_'+model+'_'+img_info+'_'+str(true_id)+'_'+str(true_label)+'_'+str(target_label)+'_Nsamp_'+str(len(G1_max)); + elif args['num_samples'] <= len(G1_max) and args['num_samples'] > 0: + figname = 'Fig_'+model+'_'+img_info+'_'+str(true_id)+'_'+str(true_label)+'_'+str(target_label)+'_Nsamp_'+str(args['num_samples']); + else: + print('Warning!! Input arg num_samp = {} exceed len(G1_max) in data_process.py'.format(args['num_samples'])) + continue + + + if args['use_slope']: + figname = figname + '_slope' + + if args['plot_dir']: + figname = os.path.join(args['plot_dir'], figname) + # figname + print('[Figname] {}'.format(figname)) + else: + # disable debugging figure + figname = "" + + + + if args['method'] == "maxsamp": + if order == "1": + Est_G1 = {'Lips_est': max(G1_max), 'shape': -1, 'loc': -1, 'scale': -1, 'ks': -1, 'pVal': -1} + Est_G2 = {'Lips_est': max(G2_max), 'shape': -1, 'loc': -1, 'scale': -1, 'ks': -1, 'pVal': -1} + Est_Gi = {'Lips_est': max(Gi_max), 'shape': -1, 'loc': -1, 'scale': -1, 'ks': -1, 'pVal': -1} + else: # currently only compare bounds in L2 for both order = 1 and order = 2 + Est_G2 = {'Lips_est': max(G2_max), 'shape': -1, 'loc': -1, 'scale': -1, 'ks': -1, 'pVal': -1} + Est_G1 = Est_G2 + Est_Gi = Est_G2 + + elif args['method'] == "mle": + # estimate Lipschitz constant: Est_G1 is a dictionary containing Lips_est and weibull paras + if order == "1": + Est_G1 = get_lipschitz_estimate(G1_max, "L1", figname) + Est_G2 = get_lipschitz_estimate(G2_max, "L2", figname) + Est_Gi = get_lipschitz_estimate(Gi_max, "Li", figname) + else: # currently only compare bounds in L2 for both order = 1 and order = 2 + Est_G2 = get_lipschitz_estimate(G2_max, "L2", figname) + Est_G1 = Est_G2 # haven't implemented + Est_Gi = Est_G2 # haven't implemented + + elif args['method'] == "mle_reg": + + if order == "1": + print('estimating L1...') + Est_G1 = get_lipschitz_estimate(G1_max, "L1", figname, True, args['shape_reg']) + print('estimating L2...') + Est_G2 = get_lipschitz_estimate(G2_max, "L2", figname, True, args['shape_reg']) + print('estimating Li...') + Est_Gi = get_lipschitz_estimate(Gi_max, "Li", figname, True, args['shape_reg']) + else: # currently only compare bounds in L2 for both order = 1 and order = 2 + print('estimating L2...') + Est_G2 = get_lipschitz_estimate(G2_max, "L2", figname, True, args['shape_reg']) + Est_G1 = Est_G2 + Est_Gi = Est_G1 + else: + raise RuntimeError("method not supported") + + # the estimated Lipschitz constant + Lip_G1 = Est_G1['Lips_est'] + Lip_G2 = Est_G2['Lips_est'] + Lip_Gi = Est_Gi['Lips_est'] + + # the estimated shape parameter (c) in Weibull distn + shape_G1 = Est_G1['shape'] + shape_G2 = Est_G2['shape'] + shape_Gi = Est_Gi['shape'] + + # the estimated loc parameters in Weibull distn + loc_G1 = Est_G1['loc'] + loc_G2 = Est_G2['loc'] + loc_Gi = Est_Gi['loc'] + + # the estimated scale parameters in Weibull distn + scale_G1 = Est_G1['scale'] + scale_G2 = Est_G2['scale'] + scale_Gi = Est_Gi['scale'] + + # the computed ks score + ks_G1 = Est_G1['ks'] + ks_G2 = Est_G2['ks'] + ks_Gi = Est_Gi['ks'] + + # the computed pVal + pVal_G1 = Est_G1['pVal'] + pVal_G2 = Est_G2['pVal'] + pVal_Gi = Est_Gi['pVal'] + + + # compute robustness bound + if order == "1": + bnd_L1 = g_x0 / Lip_Gi + bnd_L2 = g_x0 / Lip_G2 + bnd_Li = g_x0 / Lip_G1 + else: + bnd_L2 = (-g_x0_grad_2_norm + np.sqrt(g_x0_grad_2_norm**2+2*g_x0*Lip_G2))/Lip_G2 + bnd_L1 = bnd_L2 # haven't implemented + bnd_Li = bnd_L2 # haven't implemented + # save bound of each image + if args['untargeted']: + true_id = int(true_id) + if true_id not in bounds: + bounds[true_id] = [[], [], []] + bounds[true_id][0].append(bnd_L1) + bounds[true_id][1].append(bnd_L2) + bounds[true_id][2].append(bnd_Li) + else: + bounds[img_info][0].append(bnd_L1) + bounds[img_info][1].append(bnd_L2) + bounds[img_info][2].append(bnd_Li) + + # original data_process mode + #print('[STATS][L1] id = {}, true_label = {}, target_label = {}, info = {}, bnd_L1 = {:.5g}, bnd_L2 = {:.5g}, bnd_Li = {:.5g}'.format(true_id, true_label, target_label, img_info, bnd_L1, bnd_L2, bnd_Li)) + + bndnorm_L1 = "1"; + bndnorm_L2 = "2"; + bndnorm_Li = "i"; + + # if use g_x0 = {:.5g}.format(g_x0), then it will have type error. Not sure why yet. + #print('g_x0 = '+str(g_x0)) + + + if args['method'] == "maxsamp": + if order == "1": + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L1, bnd_L1)) + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L2, bnd_L2)) + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_Li, bnd_Li)) + else: # currently only compare L2 bound + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L2, bnd_L2)) + + elif args['method'] == "mle" or args['method'] == "mle_reg": + if order == "1": + # estimate Lipschitz constant: Est_G1 is a dictionary containing Lips_est and weibull paras + # current debug mode: bound_L1 corresponds to Gi, bound_L2 corresponds to G2, bound_Li corresponds to G1 + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}, ks = {:.5g}, pVal = {:.5g}, shape = {:.5g}, loc = {:.5g}, scale = {:.5g}, g_x0 = {}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L1, bnd_L1, ks_Gi, pVal_Gi, shape_Gi, loc_Gi, scale_Gi, g_x0)) + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}, ks = {:.5g}, pVal = {:.5g}, shape = {:.5g}, loc = {:.5g}, scale = {:.5g}, g_x0 = {}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L2, bnd_L2, ks_G2, pVal_G2, shape_G2, loc_G2, scale_G2, g_x0)) + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}, ks = {:.5g}, pVal = {:.5g}, shape = {:.5g}, loc = {:.5g}, scale = {:.5g}, g_x0 = {}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_Li, bnd_Li, ks_G1, pVal_G1, shape_G1, loc_G1, scale_G1, g_x0)) + else: # currently only compare L2 bound + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}, ks = {:.5g}, pVal = {:.5g}, shape = {:.5g}, loc = {:.5g}, scale = {:.5g}, g_x0 = {}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L2, bnd_L2, ks_G2, pVal_G2, shape_G2, loc_G2, scale_G2, g_x0)) + + else: + raise RuntimeError("method not supported") + + sys.stdout.flush() + + if args['untargeted']: + clever_L1s = [] + clever_L2s = [] + clever_Lis = [] + for true_id, true_id_bounds in bounds.items(): + img_clever_L1 = min(true_id_bounds[0]) + img_clever_L2 = min(true_id_bounds[1]) + img_clever_Li = min(true_id_bounds[2]) + n_classes = len(true_id_bounds[0]) + 1 + assert len(true_id_bounds[0]) == len(true_id_bounds[2]) + assert len(true_id_bounds[1]) == len(true_id_bounds[2]) + print('[STATS][L1] image = {:3d}, n_classes = {:3d}, clever_L1 = {:.5g}, clever_L2 = {:.5g}, clever_Li = {:.5g}'.format(true_id, n_classes, img_clever_L1, img_clever_L2, img_clever_Li)) + clever_L1s.append(img_clever_L1) + clever_L2s.append(img_clever_L2) + clever_Lis.append(img_clever_Li) + info = "untargeted" + clever_L1 = reduce_op(clever_L1s) + clever_L2 = reduce_op(clever_L2s) + clever_Li = reduce_op(clever_Lis) + print('[STATS][L0] info = {}, {}_clever_L1 = {:.5g}, {}_clever_L2 = {:.5g}, {}_clever_Li = {:.5g}'.format(info, info, clever_L1, info, clever_L2, info, clever_Li)) + else: + # print min/average bound + for info, info_bounds in bounds.items(): + # reduce each array to a single number (min or avg) + clever_L1 = reduce_op(info_bounds[0]) + clever_L2 = reduce_op(info_bounds[1]) + clever_Li = reduce_op(info_bounds[2]) + if order == "1": + print('[STATS][L0] info = {}, {}_clever_L1 = {:.5g}, {}_clever_L2 = {:.5g}, {}_clever_Li = {:.5g}'.format(info, info, clever_L1, info, clever_L2, info, clever_Li)) + else: # currently only compare L2 bound for both order = 1 and order = 2 + print('[STATS][L0] info = {}, {}_clever_L2 = {:.5g}'.format(info, info, clever_L2)) + + sys.stdout.flush() + + + # shutdown thread pool + pool.close() + pool.join() diff --git a/Robustness_Eval/CLEVER/collect_gradients.py b/Robustness_Eval/CLEVER/collect_gradients.py new file mode 100644 index 0000000..708947a --- /dev/null +++ b/Robustness_Eval/CLEVER/collect_gradients.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +The code is adapted from the Huanzhang's repo: +https://github.com/huanzhang12/CLEVER/blob/master/collect_gradients.py +""" + + +""" +collect_gradients.py + +Front end for collecting maximum gradient norm samples + +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang + +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" + +from __future__ import division + +import numpy as np +import scipy.io as sio +import random +import time +import sys +import os +from functools import partial +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data + +from estimate_gradient_norm import EstimateLipschitz +from utils import generate_data + +if __name__ == "__main__": + + import argparse + parser = argparse.ArgumentParser(description = "Collect gradient norm samples for calculating CLEVER score", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-d", "--dataset", choices=["mnist", "cifar", "imagenet"], default="mnist", help = "choose dataset") + parser.add_argument("-m", "--model_name", default="normal", + help = "Select model. For MNIST and CIFAR, options are: 2-layer (MLP), normal (7-layer CNN), distilled (7-layer CNN with defensive distillation), brelu (7-layer CNN with Bounded ReLU). For ImageNet, available options are: resnet_v2_50, resnet_v2_101, resnet_v2_152, inception_v1, inception_v2, inception_v3, inception_v4, inception_resnet_v2, vgg_16, vgg_19, mobilenet_v1_025, mobilenet_v1_050, mobilenet_v1_100, alexnet, nasnet_large, densenet121_k32, densenet169_k32, densenet161_k48") + parser.add_argument("--activation", type=str, choices=["relu", "tanh", "sigmoid", "elu", "softplus", "hard_sigmoid", "arctan"], help = "activation functions", default="relu") + parser.add_argument("-N", "--Nsamps", type=int, default=1024, help = "number of samples per iterations") + parser.add_argument("-i", "--Niters", type=int, default=500, help = "number of iterations. NITERS maximum gradient norms will be collected. A larger value will give a more accurate estimate") + parser.add_argument("-n", "--numimg", type=int, default=1, help = "number of test images to load from dataset") + parser.add_argument("--ids", default = "", help = "use a filelist of image IDs in CSV file for attack (UNSUPPORTED)") + parser.add_argument("--target_type", type=int, default=0b01111, help = "Binary mask for selecting targeted attack classes. bit0: top-2, bit1: random, bit2: least likely, bit3: use --ids override (UNSUPPORTED), bit4: use all labels (for untargeted)") + parser.add_argument("-f", "--firstimg", type=int, default=0, help = "start from which image in dataset") + parser.add_argument("--transform", default="", help = "input transformation function (defend_reduce, defend_jpeg, defend_png)") + parser.add_argument("--order", type = int, default=1, choices=[1,2], help = "1: first order bound, 2: second order bound for twice differentiable activations") + parser.add_argument("--compute_slope", action='store_true', help = "collect slope estimate") + parser.add_argument("--sample_norm", type=str, choices=["l2", "li", "l1"], help = "norm of sampling ball (l2, l1 or li)", default="l2") + parser.add_argument("--fix_dirty_bug", action='store_true', help = "do not use (UNSUPPORTED)") + parser.add_argument("-b", "--batch_size", type=int, default=0, help = "batch size to run model. 0: use default batch size") + parser.add_argument("--nthreads", type=int, default=0, help = "number of threads for generating random samples in sphere") + parser.add_argument("-s", "--save", default="./lipschitz_mat", help = "results output path") + parser.add_argument("--seed", type=int, default = 1215, help = "random seed") + + # parser.add_argument("--label_path", type=str) + + args = vars(parser.parse_args()) + print(args) + + seed = args['seed'] + Nsamp = args['Nsamps']; + Niters = args['Niters']; + dataset = args['dataset'] + model_name = args['model_name'] + start = args['firstimg'] + numimg = args['numimg'] + save_path = args['save'] + total = 0 + + # label_path = args['label_path'] + + random.seed(seed) + np.random.seed(seed) + + # create output directory + os.system("mkdir -p {}/{}_{}".format(save_path, dataset, model_name)) + + # create a Lipschitz estimator class (initial it early to save multiprocessing memory) + clever_estimator = EstimateLipschitz(sess=None, nthreads=args['nthreads']) + + + import tensorflow as tf + from setup_cifar import CIFAR + from setup_mnist import MNIST + from setup_imagenet import ImageNet + + tf.set_random_seed(seed) + config = tf.ConfigProto() + config.gpu_options.allow_growth=True + with tf.Session(config=config) as sess: + clever_estimator.sess = sess + # returns the input tensor and output prediction vector + img, output = clever_estimator.load_model(dataset, model_name, activation = args['activation'], batch_size = args['batch_size'], compute_slope = args['compute_slope'], order = args['order']) + # load dataset + datasets_loader = {"mnist": MNIST, "cifar": CIFAR, "imagenet": partial(ImageNet, clever_estimator.model.image_size)} + data = datasets_loader[dataset]() + # import the ID lists + + if args['ids']: + import pandas as pd + df = pd.read_csv(args['ids'], sep = ",") + # don't use this + # if args['fix_dirty_bug']: + # df = df[df['new_class'] != df['target']] + ids = list(map(int, df['id'])) + target_type = args['target_type'] + # use the target classes override + if target_type & 0b1000 != 0: + # data.test_labels[] + target_classes = [[np.argmax(data.test_labels[ele])] for ele in df['target']] + # if label_path: + # target_labels = np.loadtxt(label_path) + # target_classes = [[target_labels[ele]] for ele in list(df['target'])] + # else: + # target_classes = [[t] for t in list(df['target'])] + else: + # use generated classes + target_classes = None + else: + ids = None + target_classes = None + target_type = args['target_type'] + + # for prediction + predictor = lambda x: np.squeeze(sess.run(output, feed_dict = {img: x})) + + # gbt custom + if dataset=='mnist' and model_name in ['mnist_0','mnist_03']: + data.test_data = data.test_data+0.5 + data.train_data = data.train_data+0.5 + # generate target images + inputs, targets, true_labels, true_ids, img_info = generate_data(data, samples=numimg, targeted=True, + start=start, predictor=predictor, + random_and_least_likely = True, + ids = ids, target_classes = target_classes, target_type = target_type, + imagenet="imagenet" in dataset, + remove_background_class="imagenet" in dataset and + ("vgg" in model_name or "densenet" in model_name or "alexnet" in model_name)) + + + timestart = time.time() + print("got {} images".format(inputs.shape)) + for i, input_img in enumerate(inputs): + # original_predict = np.squeeze(sess.run(output, feed_dict = {img: [input_img]})) + print("processing image {}".format(i)) + original_predict = predictor([input_img]) + true_label = np.argmax(true_labels[i]) + predicted_label = np.argmax(original_predict) + least_likely_label = np.argmin(original_predict) + original_prob = np.sort(original_predict) + original_class = np.argsort(original_predict) + print("Top-10 classifications:", original_class[-1:-11:-1]) + print("True label:", true_label) + print("Top-10 probabilities/logits:", original_prob[-1:-11:-1]) + print("Most unlikely classifications:", original_class[:10]) + print("Most unlikely probabilities/logits:", original_prob[:10]) + if true_label != predicted_label: + print("[WARNING] This image is classfied wrongly by the classifier! Skipping!") + continue + total += 1 + # set target class + target_label = np.argmax(targets[i]); + print('Target class: ', target_label) + sys.stdout.flush() + + if args['order'] == 1: + [L2_max,L1_max,Li_max,G2_max,G1_max,Gi_max,g_x0,pred] = clever_estimator.estimate(input_img, true_label, target_label, Nsamp, Niters, args['sample_norm'], args['transform'], args['order']) + print("[STATS][L1] total = {}, seq = {}, id = {}, time = {:.3f}, true_class = {}, target_class = {}, info = {}".format(total, i, true_ids[i], time.time() - timestart, true_label, target_label, img_info[i])) + # save to sampling results to matlab ;) + mat_path = "{}/{}_{}/{}_{}_{}_{}_{}_{}_{}_order{}.mat".format(save_path, dataset, model_name, Nsamp, Niters, true_ids[i], true_label, target_label, img_info[i], args['activation'], args['order']) + save_dict = {'L2_max': L2_max, 'L1_max': L1_max, 'Li_max': Li_max, 'G2_max': G2_max, 'G1_max': G1_max, 'Gi_max': Gi_max, 'pred': pred, 'g_x0': g_x0, 'id': true_ids[i], 'true_label': true_label, 'target_label': target_label, 'info':img_info[i], 'args': args, 'path': mat_path} + sio.savemat(mat_path, save_dict) + print('saved to', mat_path) + sys.stdout.flush() + elif args['order'] == 2: + [H2_max,g_x0,g_x0_grad_2_norm,g_x0_grad_1_norm,g_x0_grad_inf_norm,pred] = clever_estimator.estimate(input_img, true_label, target_label, Nsamp, Niters, args['sample_norm'], args['transform'], args['order']) + print("[STATS][L1] total = {}, seq = {}, id = {}, time = {:.3f}, true_class = {}, target_class = {}, info = {}".format(total, i, true_ids[i], time.time() - timestart, true_label, target_label, img_info[i])) + # save to sampling results to matlab ;) + mat_path = "{}/{}_{}/{}_{}_{}_{}_{}_{}_{}_order{}.mat".format(save_path, dataset, model_name, Nsamp, Niters, true_ids[i], true_label, target_label, img_info[i], args['activation'], args['order']) + save_dict = {'H2_max': H2_max, 'pred': pred, 'g_x0': g_x0, 'id': true_ids[i], 'true_label': true_label, 'target_label': target_label, 'info':img_info[i], 'args': args, 'path': mat_path, 'g_x0_grad_2_norm': g_x0_grad_2_norm} + sio.savemat(mat_path, save_dict) + print('saved to', mat_path) + sys.stdout.flush() + diff --git a/Robustness_Eval/CLEVER/data/t10k-images-idx3-ubyte.gz b/Robustness_Eval/CLEVER/data/t10k-images-idx3-ubyte.gz new file mode 100644 index 0000000..5ace8ea Binary files /dev/null and b/Robustness_Eval/CLEVER/data/t10k-images-idx3-ubyte.gz differ diff --git a/Robustness_Eval/CLEVER/data/t10k-labels-idx1-ubyte.gz b/Robustness_Eval/CLEVER/data/t10k-labels-idx1-ubyte.gz new file mode 100644 index 0000000..a7e1415 Binary files /dev/null and b/Robustness_Eval/CLEVER/data/t10k-labels-idx1-ubyte.gz differ diff --git a/Robustness_Eval/CLEVER/data/train-images-idx3-ubyte.gz b/Robustness_Eval/CLEVER/data/train-images-idx3-ubyte.gz new file mode 100644 index 0000000..b50e4b6 Binary files /dev/null and b/Robustness_Eval/CLEVER/data/train-images-idx3-ubyte.gz differ diff --git a/Robustness_Eval/CLEVER/data/train-labels-idx1-ubyte.gz b/Robustness_Eval/CLEVER/data/train-labels-idx1-ubyte.gz new file mode 100644 index 0000000..707a576 Binary files /dev/null and b/Robustness_Eval/CLEVER/data/train-labels-idx1-ubyte.gz differ diff --git a/Robustness_Eval/CLEVER/defense.py b/Robustness_Eval/CLEVER/defense.py new file mode 100644 index 0000000..25eef20 --- /dev/null +++ b/Robustness_Eval/CLEVER/defense.py @@ -0,0 +1,323 @@ +import PIL +import PIL.Image +from io import BytesIO +import numpy as np +import tensorflow as tf +from numba import njit + +def defend_reduce(arr, depth=3): + arr = (arr * 255.0).astype(np.uint8) + shift = 8 - depth + arr = (arr >> shift) << shift + arr = arr.astype(np.float32)/255.0 + return arr + +def defend_jpeg(input_array): + pil_image = PIL.Image.fromarray((input_array*255.0).astype(np.uint8)) + f = BytesIO() + pil_image.save(f, format='jpeg', quality=75) # quality level specified in paper + jpeg_image = np.asarray(PIL.Image.open(f)).astype(np.float32)/255.0 + return jpeg_image + +def defend_png(input_array): + pil_image = PIL.Image.fromarray((np.around(input_array*255.0)).astype(np.uint8)) + f = BytesIO() + pil_image.save(f, format='png') + png_image = np.asarray(PIL.Image.open(f)).astype(np.float32)/255.0 + return png_image + +# for testing implementation +def defend_none(input_array): + # this convertion should be lossless + pil_image = PIL.Image.fromarray(input_array) + ret_image = np.asarray(pil_image) + return ret_image + +# based on https://github.com/scikit-image/scikit-image/blob/master/skimage/restoration/_denoise_cy.pyx + +# super slow since this is implemented in pure python :'( + +@njit +def bregman(image, mask, weight, eps=1e-3, max_iter=100): + rows, cols, dims = image.shape + rows2 = rows + 2 + cols2 = cols + 2 + total = rows * cols * dims + shape_ext = (rows2, cols2, dims) + + u = np.zeros(shape_ext) + dx = np.zeros(shape_ext) + dy = np.zeros(shape_ext) + bx = np.zeros(shape_ext) + by = np.zeros(shape_ext) + + u[1:-1, 1:-1] = image + # reflect image + u[0, 1:-1] = image[1, :] + u[1:-1, 0] = image[:, 1] + u[-1, 1:-1] = image[-2, :] + u[1:-1, -1] = image[:, -2] + + i = 0 + rmse = np.inf + lam = 2 * weight + norm = (weight + 4 * lam) + + while i < max_iter and rmse > eps: + rmse = 0 + + for k in range(dims): + for r in range(1, rows+1): + for c in range(1, cols+1): + uprev = u[r, c, k] + + # forward derivatives + ux = u[r, c+1, k] - uprev + uy = u[r+1, c, k] - uprev + + # Gauss-Seidel method + if mask[r-1, c-1]: + unew = (lam * (u[r+1, c, k] + + u[r-1, c, k] + + u[r, c+1, k] + + u[r, c-1, k] + + dx[r, c-1, k] - + dx[r, c, k] + + dy[r-1, c, k] - + dy[r, c, k] - + bx[r, c-1, k] + + bx[r, c, k] - + by[r-1, c, k] + + by[r, c, k] + ) + weight * image[r-1, c-1, k] + ) / norm + else: + # similar to the update step above, except we take + # lim_{weight->0} of the update step, effectively + # ignoring the l2 loss + unew = (u[r+1, c, k] + + u[r-1, c, k] + + u[r, c+1, k] + + u[r, c-1, k] + + dx[r, c-1, k] - + dx[r, c, k] + + dy[r-1, c, k] - + dy[r, c, k] - + bx[r, c-1, k] + + bx[r, c, k] - + by[r-1, c, k] + + by[r, c, k] + ) / 4.0 + u[r, c, k] = unew + + # update rms error + rmse += (unew - uprev)**2 + + bxx = bx[r, c, k] + byy = by[r, c, k] + + # d_subproblem + s = ux + bxx + if s > 1/lam: + dxx = s - 1/lam + elif s < -1/lam: + dxx = s + 1/lam + else: + dxx = 0 + s = uy + byy + if s > 1/lam: + dyy = s - 1/lam + elif s < -1/lam: + dyy = s + 1/lam + else: + dyy = 0 + + dx[r, c, k] = dxx + dy[r, c, k] = dyy + + bx[r, c, k] += ux - dxx + by[r, c, k] += uy - dyy + + rmse = np.sqrt(rmse / total) + i += 1 + + # return np.squeeze(np.asarray(u[1:-1, 1:-1])) + return u[1:-1, 1:-1] + +@njit +def defend_tv(input_array, keep_prob=0.5, lambda_tv=0.03): + mask = np.random.uniform(0.0, 1.0, size=input_array.shape[:2]) + mask = mask < keep_prob + return bregman(input_array, mask, weight=2.0/lambda_tv) + +def make_defend_quilt(sess): + # setup for quilting + quilt_db = np.load('data/quilt_db.npy') + quilt_db_reshaped = quilt_db.reshape(1000000, -1) + TILE_SIZE = 5 + TILE_OVERLAP = 2 + tile_skip = TILE_SIZE - TILE_OVERLAP + K = 10 + db_tensor = tf.placeholder(tf.float32, quilt_db_reshaped.shape) + query_imgs = tf.placeholder(tf.float32, (TILE_SIZE * TILE_SIZE * 3, None)) + norms = tf.reduce_sum(tf.square(db_tensor), axis=1)[:, tf.newaxis] \ + - 2*tf.matmul(db_tensor, query_imgs) + _, topk_indices = tf.nn.top_k(-tf.transpose(norms), k=K, sorted=False) + def min_error_table(arr, direction): + assert direction in ('horizontal', 'vertical') + y, x = arr.shape + cum = np.zeros_like(arr) + if direction == 'horizontal': + cum[:, -1] = arr[:, -1] + for ix in range(x-2, -1, -1): + for iy in range(y): + m = arr[iy, ix+1] + if iy > 0: + m = min(m, arr[iy-1, ix+1]) + if iy < y - 1: + m = min(m, arr[iy+1, ix+1]) + cum[iy, ix] = arr[iy, ix] + m + elif direction == 'vertical': + cum[-1, :] = arr[-1, :] + for iy in range(y-2, -1, -1): + for ix in range(x): + m = arr[iy+1, ix] + if ix > 0: + m = min(m, arr[iy+1, ix-1]) + if ix < x - 1: + m = min(m, arr[iy+1, ix+1]) + cum[iy, ix] = arr[iy, ix] + m + return cum + def index_exists(arr, index): + if arr.ndim != len(index): + return False + return all(i > 0 for i in index) and all(index[i] < arr.shape[i] for i in range(arr.ndim)) + def assign_block(ix, iy, tile, synth): + posx = tile_skip * ix + posy = tile_skip * iy + + if ix == 0 and iy == 0: + synth[posy:posy+TILE_SIZE, posx:posx+TILE_SIZE, :] = tile + elif iy == 0: + # first row, only have horizontal overlap of the block + tile_left = tile[:, :TILE_OVERLAP, :] + synth_right = synth[:TILE_SIZE, posx:posx+TILE_OVERLAP, :] + errors = np.sum(np.square(tile_left - synth_right), axis=2) + table = min_error_table(errors, direction='vertical') + # copy row by row into synth + xoff = np.argmin(table[0, :]) + synth[posy, posx+xoff:posx+TILE_SIZE] = tile[0, xoff:] + for yoff in range(1, TILE_SIZE): + # explore nearby xoffs + candidates = [(yoff, xoff), (yoff, xoff-1), (yoff, xoff+1)] + index = min((i for i in candidates if index_exists(table, i)), key=lambda i: table[i]) + xoff = index[1] + synth[posy+yoff, posx+xoff:posx+TILE_SIZE] = tile[yoff, xoff:] + elif ix == 0: + # first column, only have vertical overlap of the block + tile_up = tile[:TILE_OVERLAP, :, :] + synth_bottom = synth[posy:posy+TILE_OVERLAP, :TILE_SIZE, :] + errors = np.sum(np.square(tile_up - synth_bottom), axis=2) + table = min_error_table(errors, direction='horizontal') + # copy column by column into synth + yoff = np.argmin(table[:, 0]) + synth[posy+yoff:posy+TILE_SIZE, posx] = tile[yoff:, 0] + for xoff in range(1, TILE_SIZE): + # explore nearby yoffs + candidates = [(yoff, xoff), (yoff-1, xoff), (yoff+1, xoff)] + index = min((i for i in candidates if index_exists(table, i)), key=lambda i: table[i]) + yoff = index[0] + synth[posy+yoff:posy+TILE_SIZE, posx+xoff] = tile[yoff:, xoff] + else: + # glue cuts along diagonal + tile_up = tile[:TILE_OVERLAP, :, :] + synth_bottom = synth[posy:posy+TILE_OVERLAP, :TILE_SIZE, :] + errors_up = np.sum(np.square(tile_up - synth_bottom), axis=2) + table_up = min_error_table(errors_up, direction='horizontal') + tile_left = tile[:, :TILE_OVERLAP, :] + synth_right = synth[:TILE_SIZE, posx:posx+TILE_OVERLAP, :] + errors_left = np.sum(np.square(tile_left - synth_right), axis=2) + table_left = min_error_table(errors_left, direction='vertical') + glue_index = -1 + glue_value = np.inf + for i in range(TILE_OVERLAP): + e = table_up[i, i] + table_left[i, i] + if e < glue_value: + glue_value = e + glue_index = i + # copy left part first, up to the overlap column + xoff = glue_index + synth[posy+glue_index, posx+xoff:posx+TILE_OVERLAP] = tile[glue_index, xoff:TILE_OVERLAP] + for yoff in range(glue_index+1, TILE_SIZE): + # explore nearby xoffs + candidates = [(yoff, xoff), (yoff, xoff-1), (yoff, xoff+1)] + index = min((i for i in candidates if index_exists(table_left, i)), key=lambda i: table_left[i]) + xoff = index[1] + synth[posy+yoff, posx+xoff:posx+TILE_OVERLAP] = tile[yoff, xoff:TILE_OVERLAP] + # copy right part, down to overlap row + yoff = glue_index + synth[posy+yoff:posy+TILE_OVERLAP, posx+glue_index] = tile[yoff:TILE_OVERLAP, glue_index] + for xoff in range(glue_index+1, TILE_SIZE): + # explore nearby yoffs + candidates = [(yoff, xoff), (yoff-1, xoff), (yoff+1, xoff)] + index = min((i for i in candidates if index_exists(table_up, i)), key=lambda i: table_up[i]) + yoff = index[0] + synth[posy+yoff:posy+TILE_OVERLAP, posx+xoff] = tile[yoff:TILE_OVERLAP, xoff] + # copy rest of image + synth[posy+TILE_OVERLAP:posy+TILE_SIZE, posx+TILE_OVERLAP:posx+TILE_SIZE] = tile[TILE_OVERLAP:, TILE_OVERLAP:] + KNN_MAX_BATCH = 1000 + def quilt(arr, graphcut=True): + h, w, c = arr.shape + assert (h - TILE_SIZE) % tile_skip == 0 + assert (w - TILE_SIZE) % tile_skip == 0 + horiz_blocks = (w - TILE_SIZE) // tile_skip + 1 + vert_blocks = (h - TILE_SIZE) // tile_skip + 1 + num_patches = horiz_blocks * vert_blocks + patches = np.zeros((TILE_SIZE * TILE_SIZE * 3, num_patches)) + idx = 0 + for iy in range(vert_blocks): + for ix in range(horiz_blocks): + posx = tile_skip*ix + posy = tile_skip*iy + patches[:, idx] = arr[posy:posy+TILE_SIZE, posx:posx+TILE_SIZE, :].ravel() + idx += 1 + + ind = [] + for chunk in range(num_patches // KNN_MAX_BATCH + (1 if num_patches % KNN_MAX_BATCH != 0 else 0)): + start = KNN_MAX_BATCH * chunk + end = start + KNN_MAX_BATCH + # for some reason, the code below is 10x slower when run in a Jupyter notebook + # not sure why... + indices_ = sess.run(topk_indices, {db_tensor: quilt_db_reshaped, query_imgs: patches[:, start:end]}) + for i in indices_: + ind.append(np.random.choice(i)) + + synth = np.zeros((299, 299, 3)) + + idx = 0 + for iy in range(vert_blocks): + for ix in range(horiz_blocks): + posx = tile_skip*ix + posy = tile_skip*iy + tile = quilt_db[ind[idx]] + if not graphcut: + synth[posy:posy+TILE_SIZE, posx:posx+TILE_SIZE, :] = tile + else: + assign_block(ix, iy, tile, synth) + idx += 1 + return synth + + return quilt + +# x is a square image (3-tensor) +def defend_crop(x, crop_size=90, ensemble_size=30): + x_size = tf.to_float(x.shape[1]) + frac = crop_size/x_size + start_fraction_max = (x_size - crop_size)/x_size + def randomizing_crop(x): + start_x = tf.random_uniform((), 0, start_fraction_max) + start_y = tf.random_uniform((), 0, start_fraction_max) + return tf.image.crop_and_resize([x], boxes=[[start_y, start_x, start_y+frac, start_x+frac]], + box_ind=[0], crop_size=[crop_size, crop_size]) + + return tf.concat([randomizing_crop(x) for _ in range(ensemble_size)], axis=0) diff --git a/Robustness_Eval/CLEVER/estimate_gradient_norm.py b/Robustness_Eval/CLEVER/estimate_gradient_norm.py new file mode 100644 index 0000000..a407517 --- /dev/null +++ b/Robustness_Eval/CLEVER/estimate_gradient_norm.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +The code is adapted from the Huanzhang's repo: +https://github.com/huanzhang12/CLEVER/blob/master/estimate_gradient_norm.py +""" + + + +""" +estimate_gradient_norm.py + +A multithreaded gradient norm sampler + +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang + +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" + +from __future__ import division + +import numpy as np +import random +import ctypes +import time +import sys +import os +import tensorflow as tf + +from multiprocessing import Pool, current_process, cpu_count +from shmemarray import ShmemRawArray, NpShmemArray +from functools import partial +from randsphere import randsphere +from tensorflow.python.ops import gradients_impl + + +class EstimateLipschitz(object): + + def __init__(self, sess, seed = 1215, nthreads = 0): + """ + sess: tensorflow session + Nsamp: number of samples to take per iteration + Niters: number of iterations, each iteration we return a max L + """ + self.sess = sess + self.seed = seed + # create a pool of workers to compute samples in advance + if nthreads == 0: + self.n_processes = max(cpu_count() // 2, 1) + else: + self.n_processes = nthreads + # set up random seed during initialization + def initializer(s): + np.random.seed(s + current_process()._identity[0]) + # using only 1 OpenMP thread + os.environ['OMP_NUM_THREADS'] = "1" + self.pool = Pool(processes = self.n_processes, initializer = initializer, initargs=(self.seed,)) + + def load_model(self, dataset = "mnist", model_name = "2-layer", activation = "relu", model = None, batch_size = 0, compute_slope = False, order = 1): + """ + model: if set to None, then load dataset with model_name. Otherwise use the model directly. + dataset: mnist, cifar and imagenet. recommend to use mnist and cifar as a starting point. + model_name: possible options are 2-layer, distilled, and normal + """ + from setup_cifar import CIFAR, CIFARModel, TwoLayerCIFARModel + from setup_mnist import MNIST, MNISTModel, TwoLayerMNISTModel, mnist_0, mnist_03 + from nlayer_model import NLayerModel + from setup_imagenet import ImageNet, ImageNetModel + + # if set this to true, we will use the logit layer output instead of probability + # the logit layer's gradients are usually larger and more stable + output_logits = True + self.dataset = dataset + self.model_name = model_name + + if model is None: + print('Loading model...') + if dataset == "mnist": + self.batch_size = 1024 + if model_name == "2-layer": + model = TwoLayerMNISTModel("models/mnist_2layer", self.sess, not output_logits) + + elif model_name == "mnist_0": + if activation == "relu": + model = mnist_0("models/mnist-0", self.sess, not output_logits) + + elif model_name == "mnist_03": + if activation == "relu": + model = mnist_03("models/mnist-0.3", self.sess, not output_logits) + + elif model_name == "normal": + if activation == "relu": + model = MNISTModel("models/mnist", self.sess, not output_logits) + else: + print("actviation = {}".format(activation)) + model = MNISTModel("models/mnist_cnn_7layer_"+activation, self.sess, not output_logits, activation= activation) + time.sleep(5) + + elif model_name == "brelu": + model = MNISTModel("models/mnist_brelu", self.sess, not output_logits, use_brelu = True) + elif model_name == "distilled": + model = MNISTModel("models/mnist-distilled-100", self.sess, not output_logits) + else: + # specify model parameters as N,M,opts + model_params = model_name.split(",") + if len(model_params) < 3: + raise(RuntimeError("incorrect model option" + model_name)) + numlayer = int(model_params[0]) + nhidden = int(model_params[1]) + modelfile = "models/mnist_{}layer_relu_{}_{}".format(numlayer, nhidden, model_params[2]) + print("loading", modelfile) + model = NLayerModel([nhidden] * (numlayer - 1), modelfile) + elif dataset == "cifar": + self.batch_size = 1024 + if model_name == "2-layer": + model = TwoLayerCIFARModel("models/cifar_2layer", self.sess, not output_logits) + elif model_name == "normal": + if activation == "relu": + model = CIFARModel("models/cifar", self.sess, not output_logits) + else: + model = CIFARModel("models/cifar_cnn_7layer_"+activation, self.sess, not output_logits, activation = activation) + elif model_name == "brelu": + model = CIFARModel("models/cifar_brelu", self.sess, not output_logits, use_brelu = True) + elif model_name == "distilled": + model = CIFARModel("models/cifar-distilled-100", self.sess, not output_logits) + else: + # specify model parameters as N,M,opts + model_params = model_name.split(",") + if len(model_params) < 3: + raise(RuntimeError("incorrect model option" + model_name)) + numlayer = int(model_params[0]) + nhidden = int(model_params[1]) + modelfile = "models/cifar_{}layer_relu_{}_{}".format(numlayer, nhidden, model_params[2]) + print("loading", modelfile) + model = NLayerModel([nhidden] * (numlayer - 1), modelfile, image_size=32, image_channel=3) + elif dataset == "imagenet": + self.batch_size = 32 + model = ImageNetModel(self.sess, use_softmax = not output_logits, model_name = model_name, create_prediction = False) + else: + raise(RuntimeError("dataset unknown")) + + #print("*** Loaded model successfully") + + self.model = model + self.compute_slope = compute_slope + if batch_size != 0: + self.batch_size = batch_size + + ## placeholders: self.img, self.true_label, self.target_label + # img is the placeholder for image input + self.img = tf.placeholder(shape = [None, model.image_size, model.image_size, model.num_channels], dtype = tf.float32) + # output is the output tensor of the entire network + self.output = model.predict(self.img) + # create the graph to compute gradient + # get the desired true label and target label + self.true_label = tf.placeholder(dtype = tf.int32, shape = []) + self.target_label = tf.placeholder(dtype = tf.int32, shape = []) + true_output = self.output[:, self.true_label] + target_output = self.output[:, self.target_label] + # get the difference + self.objective = true_output - target_output + # get the gradient(deprecated arguments) + self.grad_op = tf.gradients(self.objective, self.img)[0] + # compute gradient norm: (in computation graph, so is faster) + grad_op_rs = tf.reshape(self.grad_op, (tf.shape(self.grad_op)[0], -1)) + self.grad_2_norm_op = tf.norm(grad_op_rs, axis = 1) + self.grad_1_norm_op = tf.norm(grad_op_rs, ord=1, axis = 1) + self.grad_inf_norm_op = tf.norm(grad_op_rs, ord=np.inf, axis = 1) + + ### Lily: added Hessian-vector product calculation here for 2nd order bound: + if order == 2: + ## _hessian_vector_product(ys, xs, v): return a list of tensors containing the product between the Hessian and v + ## ys: a scalar valur or a tensor or a list of tensors to be summed to yield of scalar + ## xs: a list of tensors that we should construct the Hessian over + ## v: a list of tensors with the same shape as xs that we want to multiply by the Hessian + # self.randv: shape = (Nimg,28,28,1) (the v in _hessian_vector_product) + self.randv = tf.placeholder(shape = [None, model.image_size, model.image_size, model.num_channels], dtype = tf.float32) + # hv_op_tmp: shape = (Nimg,28,28,1) for mnist, same as self.img (the xs in _hessian_vector_product) + hv_op_tmp = gradients_impl._hessian_vector_product(self.objective, [self.img], [self.randv])[0] + # hv_op_rs: reshape hv_op_tmp to hv_op_rs whose shape = (Nimg, 784) for mnist + hv_op_rs = tf.reshape(hv_op_tmp, (tf.shape(hv_op_tmp)[0],-1)) + # self.hv_norm_op: norm of hessian vector product, keep shape = (Nimg,1) using keepdims + self.hv_norm_op = tf.norm(hv_op_rs, axis = 1, keepdims=True) + # hv_op_rs_normalize: normalize Hv to Hv/||Hv||, shape = (Nimg, 784) + hv_op_rs_normalize = hv_op_rs/self.hv_norm_op + # self.hv_op: reshape hv_op_rs_normalize to shape = (Nimg,28,28,1) + self.hv_op = tf.reshape(hv_op_rs_normalize, tf.shape(hv_op_tmp)) + + ## reshape randv and compute its norm + # shape: (Nimg, 784) + randv_rs = tf.reshape(self.randv, (tf.shape(self.randv)[0],-1)) + # shape: (Nimg,) + self.randv_norm_op = tf.norm(randv_rs, axis = 1) + ## compute v'Hv: use un-normalized Hv (hv_op_tmp, hv_op_rs) + # element-wise multiplication and then sum over axis = 1 (now shape: (Nimg,)) + self.vhv_op = tf.reduce_sum(tf.multiply(randv_rs,hv_op_rs),axis=1) + ## compute Rayleigh quotient: v'Hv/v'v (estimated largest eigenvalue), shape: (Nimg,) + # note: self.vhv_op and self.randv_norm_op has to be in the same dimension (either (Nimg,) or (Nimg,1)) + self.eig_est = self.vhv_op/tf.square(self.randv_norm_op) + + ## Lily added the tf.while to compute the eigenvalue in computational graph later + # cond for computing largest abs/neg eigen-value + def cond(it, randv, eig_est, eig_est_prev, tfconst): + norm_diff = tf.norm(eig_est-eig_est_prev,axis=0) + return tf.logical_and(it < 500, norm_diff > 0.001) + + # compute largest abs eigenvalue: tfconst = 0 + # compute largest neg eigenvalue: tfconst = 10 + def body(it, randv, eig_est, eig_est_prev, tfconst): + #hv_op_tmp = gradients_impl._hessian_vector_product(self.objective, [self.img], [randv])[0]-10*randv + hv_op_tmp = gradients_impl._hessian_vector_product(self.objective, [self.img], [randv])[0]-tf.multiply(tfconst,randv) + hv_op_rs = tf.reshape(hv_op_tmp, (tf.shape(hv_op_tmp)[0],-1)) + hv_norm_op = tf.norm(hv_op_rs, axis = 1, keepdims=True) + hv_op_rs_normalize = hv_op_rs/hv_norm_op + hv_op = tf.reshape(hv_op_rs_normalize, tf.shape(hv_op_tmp)) + + randv_rs = tf.reshape(randv, (tf.shape(randv)[0],-1)) + randv_norm_op = tf.norm(randv_rs, axis = 1) + vhv_op = tf.reduce_sum(tf.multiply(randv_rs,hv_op_rs),axis=1) + eig_est_prev = eig_est + eig_est = vhv_op/tf.square(randv_norm_op) + + return (it+1, hv_op, eig_est, eig_est_prev, tfconst) + + it = tf.constant(0) + # compute largest abs eigenvalue + result = tf.while_loop(cond, body, [it, self.randv, self.vhv_op, self.eig_est, tf.constant(0.0)]) + # compute largest neg eigenvalue + self.shiftconst = tf.placeholder(shape = (), dtype = tf.float32) + result_1 = tf.while_loop(cond, body, [it, self.randv, self.vhv_op, self.eig_est, self.shiftconst]) + + # computing largest abs eig value and save result + self.it = result[0] + self.while_hv_op = result[1] + self.while_eig = result[2] + + # computing largest neg eig value and save result + self.it_1 = result_1[0] + #self.while_eig_1 = tf.add(result_1[2], tfconst) + self.while_eig_1 = tf.add(result_1[2], result_1[4]) + + show_tensor_op = False + if show_tensor_op: + print("====================") + print("Define hessian_vector_product operator: ") + print("hv_op_tmp = {}".format(hv_op_tmp)) + print("hv_op_rs = {}".format(hv_op_rs)) + print("self.hv_norm_op = {}".format(self.hv_norm_op)) + print("hv_op_rs_normalize = {}".format(hv_op_rs_normalize)) + print("self.hv_op = {}".format(self.hv_op)) + print("self.grad_op = {}".format(self.grad_op)) + print("randv_rs = {}".format(randv_rs)) + print("self.randv_norm_op = {}".format(self.randv_norm_op)) + print("self.vhv_op = {}".format(self.vhv_op)) + print("self.eig_est = {}".format(self.eig_est)) + print("====================") + + return self.img, self.output + + + def _estimate_Lipschitz_multiplerun(self, num, niters, input_image, target_label, true_label, sample_norm = "l2", transform=None, order = 1): + """ + num: number of samples per iteration + niters: number of iterations + input_image: original image (h*w*c) + """ + batch_size = self.batch_size + shape = (batch_size, self.model.image_size, self.model.image_size, self.model.num_channels) + dimension = self.model.image_size * self.model.image_size * self.model.num_channels + + if num < batch_size: + print("Increasing num to", batch_size) + num = batch_size + + """ + 1. Compute input_image related quantities: + """ + # get the original prediction and gradient, gradient norms values on input image: + pred, grad_val, grad_2_norm_val, grad_1_norm_val, grad_inf_norm_val = self.sess.run( + [self.output, self.grad_op, self.grad_2_norm_op, self.grad_1_norm_op, self.grad_inf_norm_op], + feed_dict = {self.img: [input_image], self.true_label: true_label, self.target_label: target_label}) + pred = np.squeeze(pred) + # print(pred) + # print(grad_val) + + # class c and class j in Hein's paper. c is original class + c = true_label + j = target_label + # get g_x0 = f_c(x_0) - f_j(x_0) + g_x0 = pred[c] - pred[j] + # grad_z_norm should be scalar + g_x0_grad_2_norm = np.squeeze(grad_2_norm_val) + g_x0_grad_1_norm = np.squeeze(grad_1_norm_val) + g_x0_grad_inf_norm = np.squeeze(grad_inf_norm_val) + + print("** Evaluating g_x0, grad_2_norm_val on the input image x0: ") + print("shape of input_image = {}".format(input_image.shape)) + print("g_x0 = {:.3f}, grad_2_norm_val = {:3f}, grad_1_norm_val = {:.3f}, grad_inf_norm_val = {:3f}".format(g_x0, g_x0_grad_2_norm, g_x0_grad_1_norm, g_x0_grad_inf_norm)) + + ##### Lily ##### + if order == 2: # evaluate the hv and hv norm on input_image + # set randv as a random matrix with the same shape as input_image + print("** Evaluating hv and hv_norm on the input image x0:") + randv = np.random.randn(*input_image.shape) + hv, hv_norm = self.sess.run([self.hv_op, self.hv_norm_op], + feed_dict = {self.img: [input_image], self.randv:[randv], self.true_label: true_label, self.target_label: target_label}) + print("hv shape = {}, hv_norm = {}".format(hv.shape, hv_norm)) + + + """ + 2. Prepare for sampling: + """ + def div_work_to_cores(njobs, nprocs): + process_item_list = [] + while njobs > 0: + process_item_list.append(int(np.ceil(njobs / float(nprocs)))) + njobs -= process_item_list[-1] + nprocs -= 1 + return process_item_list + # n is the dimension + + if self.dataset == "imagenet": + # for imagenet, generate random samples for this batch only + # array in shared memory storing results of all threads + total_item_size = batch_size + else: + # for cifar and mnist, generate random samples for this entire iteration + total_item_size = num + # divide the jobs evenly to all available threads + process_item_list = div_work_to_cores(total_item_size, self.n_processes) + self.n_processes = len(process_item_list) + # select random sample generation function + if sample_norm == "l2": + # the scaling constant in [a,b]: scale the L2 norm of each sample (has originally norm ~1) + a = 0; b = 3; + elif sample_norm == "li": + # for Linf we don't need the scaling + a = 0.1; b = 0.1; + elif sample_norm == "l1": + # TODO: make the sample ball radius adjustable + a = 0; b = 30; + else: + raise RuntimeError("Unknown sample_norm " + sample_norm) + print('Using sphere', sample_norm) + + ## create necessary shared array structures (saved in /dev/shm) and will be used (and written) in randsphere.py: + # result_arr, scale, input_example, all_inputs + # note: need to use scale[:] = ... not scale = ..., o.w. the contents will not be saved to the shared array + # inputs_0 is the image x_0 + inputs_0 = np.array(input_image) + tag_prefix = str(os.getpid()) + "_" + result_arr = NpShmemArray(np.float32, (total_item_size, dimension), tag_prefix + "randsphere") + # we have an extra batch_size to avoid overflow + scale = NpShmemArray(np.float32, (num+batch_size), tag_prefix + "scale") + scale[:] = (b-a)*np.random.rand(num+batch_size)+a; + input_example = NpShmemArray(np.float32, inputs_0.shape, tag_prefix + "input_example") + # this is a read-only array + input_example[:] = inputs_0 + # all_inputs is a shared memeory array and will be written in the randsphere to save the samples + # all_inputs holds the perturbations for one batch or all samples + all_inputs = NpShmemArray(np.float32, (total_item_size,) + inputs_0.shape, tag_prefix + "all_inputs") + # holds the results copied from all_inputs + clipped_all_inputs = np.empty(dtype=np.float32, shape = (total_item_size,) + inputs_0.shape) + # prepare the argument list + offset_list = [0] + for item in process_item_list[:-1]: + offset_list.append(offset_list[-1] + item) + print(self.n_processes, "threads launched with parameter", process_item_list, offset_list) + + ## create multiple process to generate samples + # randsphere: generate samples (see randsphere.py); partial is a function similar to lambda, now worker_func is a function of idx only + worker_func = partial(randsphere, n = dimension, input_shape = inputs_0.shape, total_size = total_item_size, scale_size = num+batch_size, tag_prefix = tag_prefix, r = 1.0, norm = sample_norm, transform = transform) + worker_args = list(zip(process_item_list, offset_list, [0] * self.n_processes)) + # sample_results is an object to monitor if the process has ended (meaning finish generating samples in randsphere.py) + # this line of code will initiate the worker_func to start working (like initiate the job) + sample_results = self.pool.map_async(worker_func, worker_args) + + # num: # of samples to be run, \leq samples.shape[0] + + # number of iterations + Niters = niters; + + if order == 1: + # store the max L in each iteration + L2_max = np.zeros(Niters) + L1_max = np.zeros(Niters) + Li_max = np.zeros(Niters) + # store the max G in each iteration + G2_max = np.zeros(Niters) + G1_max = np.zeros(Niters) + Gi_max = np.zeros(Niters) + # store computed Lispschitz constants in each iteration + L2 = np.zeros(num) + L1 = np.zeros(num) + Li = np.zeros(num) + # store computed gradient norm in each iteration + G2 = np.zeros(num) + G1 = np.zeros(num) + Gi = np.zeros(num) + elif order == 2: + # store the max H in each iteration + H2_max = np.zeros(Niters) + # store computed 2 norm of H in each iteration + H2 = np.zeros(num) + H2_neg = np.zeros(num) + + # how many batches we have + Nbatches = num // batch_size + + # timer + search_begin_time = time.time() + + """ + 3. Start performing sampling: + """ + ## Start + # multiple runs: generating the samples + ## use worker_func to generate x samples, and then use sess.run to evaluate the gradient norm operator + for iters in range(Niters): + iter_begin_time = time.time() + + # shuffled index + # idx_shuffle = np.random.permutation(num); + + # the scaling constant in [a,b]: scale the L2 norm of each sample (has originally norm ~1) + scale[:] = (b-a)*np.random.rand(num+batch_size)+a; + + # number of L's we have computed + L_counter = 0 + G_counter = 0 + H_counter = 0 + + overhead_time = 0.0 + overhead_start = time.time() + # for cifar and mnist, generate all the random input samples (x in the paper) at once + # for imagenet, generate one batch of input samples (x in the paper) for each iteration + if self.dataset != "imagenet": + # get samples for this iteration: make sure randsphere finished computing samples and stored in all_inputs + # if the samples have not yet done generating, then this line will block the codes until the processes are done, then it will return + sample_results.get() + # copy the results to a buffer and do clipping + np.clip(all_inputs, -0.5, 0.5, out = clipped_all_inputs) + # create multiple process again to generate samples for next batch (initiate a new job) because in below we will need to do sess.run in GPU which might be slow. So we want to generate samples on CPU while running sess.run on GPU to save time + sample_results = self.pool.map_async(worker_func, worker_args) + overhead_time += time.time() - overhead_start + + ## generate input samples "batch_inputs" and compute corresponding gradient norms samples "perturbed_grad_x_norm" + for i in range(Nbatches): + overhead_start = time.time() + # for imagenet, generate random samples for this batch only + if self.dataset == "imagenet": + # get samples for this batch + sample_results.get() + # copy the results to a buffer and do clipping + np.clip(all_inputs, -0.5, 0.5, out = clipped_all_inputs) + # create multiple threads to generate samples for next batch + worker_args = zip(process_item_list, offset_list, [(i + 1) * batch_size] * self.n_processes) + sample_results = self.pool.map_async(worker_func, worker_args) + + if self.dataset == "imagenet": + # we generate samples for each batch at a time + batch_inputs = clipped_all_inputs + else: + # we generate samples for all batches + batch_inputs = clipped_all_inputs[i * batch_size: (i + 1) * batch_size] + # print(result_arr.shape, result_arr) + # print('------------------------') + # print(batch_inputs.shape, batch_inputs.reshape(result_arr.shape)) + # print('------------------------') + overhead_time += time.time() - overhead_start + + if order == 1: + # run inference and get the gradient + perturbed_predicts, perturbed_grad_2_norm, perturbed_grad_1_norm, perturbed_grad_inf_norm = self.sess.run( + [self.output, self.grad_2_norm_op, self.grad_1_norm_op, self.grad_inf_norm_op], + feed_dict = {self.img: batch_inputs, self.target_label: target_label, self.true_label: true_label}) + + if self.compute_slope: + s = batch_inputs.reshape(batch_size, -1) + # compute distance between consecutive samples: not use sequential samples + s12_2_norm = np.linalg.norm(s[0:batch_size-1:2] - s[1:batch_size:2], axis = 1) + s12_1_norm = np.linalg.norm(s[0:batch_size-1:2] - s[1:batch_size:2], ord=1, axis = 1) + s12_i_norm = np.linalg.norm(s[0:batch_size-1:2] - s[1:batch_size:2], ord=np.inf, axis = 1) + # compute function value differences: not use sequential samples + g_x1 = perturbed_predicts[0:batch_size-1:2, c] - perturbed_predicts[0:batch_size-1:2, j] + g_x2 = perturbed_predicts[1:batch_size:2, c] - perturbed_predicts[1:batch_size:2, j] + # estimated Lipschitz constants for this batch + # for slope estimate, we need the DUAL norm + batch_L2 = np.abs(g_x1 - g_x2) / s12_2_norm + batch_L1 = np.abs(g_x1 - g_x2) / s12_i_norm + batch_Li = np.abs(g_x1 - g_x2) / s12_1_norm + L2[L_counter : L_counter + batch_size//2] = batch_L2 + L1[L_counter : L_counter + batch_size//2] = batch_L1 + Li[L_counter : L_counter + batch_size//2] = batch_Li + + G2[G_counter : G_counter + batch_size] = perturbed_grad_2_norm + G1[G_counter : G_counter + batch_size] = perturbed_grad_1_norm + Gi[G_counter : G_counter + batch_size] = perturbed_grad_inf_norm + L_counter += (batch_size//2) + G_counter += batch_size + + elif order == 2: + ##### Lily ##### + randv_batch = np.random.randn(*batch_inputs.shape) + perturbed_hv, perturbed_hv_norm = self.sess.run([self.hv_op, self.hv_norm_op], + feed_dict = {self.img: batch_inputs, self.randv: randv_batch, + self.true_label: true_label, self.target_label: target_label}) + + show_tensor_dim = False + if show_tensor_dim: + print("====================") + print("** Evaluating perturbed_hv and perturbed_hv_norm in batch {}: ".format(iters)) + print("pertubed_hv_prod shape = {}".format(perturbed_hv.shape)) + print("randv_batch shape = {}".format(randv_batch.shape)) + print("perturbed_hv_norm = {}".format(perturbed_hv_norm[:,0])) # size: (Nimg, 1) + print("perturbed_hv_norm shape = {}".format(perturbed_hv_norm.shape)) + #print("perturbed_grad_2_norm= {}".format(perturbed_grad_2_norm)) + #print("perturbed_grad_2_norm shape = {}".format(perturbed_grad_2_norm.shape)) + + + pt_hvs = [] + pt_hvs.append(perturbed_hv+0*randv_batch) + + #print("************** Using tf.while_loop:********************") + # compute max eigenvalue + temp_hv, temp_eig, niter_eig = self.sess.run([self.while_hv_op, self.while_eig, self.it], feed_dict = {self.img: batch_inputs, self.randv: randv_batch, self.true_label: true_label, self.target_label: target_label}) + ##print("converge in {} steps, temp_eig = {}".format(niter_eig, temp_eig)) + + # if max eigenvalue is positive, compute the max neg eigenvalue by using the shiftconst + if max(temp_eig) > 0: + shiftconst = max(temp_eig) + temp_eig_1, niter_eig_1 = self.sess.run([self.while_eig_1, self.it_1], feed_dict = {self.img: batch_inputs, self.randv: randv_batch, self.true_label: true_label, self.target_label: target_label, self.shiftconst: shiftconst}) + ##print("converge in {} steps, temp_eig_1 = {}".format(niter_eig_1, temp_eig_1)) + else: + temp_eig_1 = temp_eig + niter_eig_1 = -1 + + print("temp_eig (abs) converge in {} steps, temp_eig_1 (neg) converge in {} steps".format(niter_eig, niter_eig_1)) + + ## use outer while_loop + #max_eig_iters = 10 + #print_flag = True + #final_est_eig_1 = self._compute_max_abseig(pt_hvs, batch_inputs, true_label, target_label, max_eig_iters, print_flag) + #print("************** Using outer while_loop:********************") + #print("outer loop final_est_eig_1 = {}".format(final_est_eig_1)) + + ## use tf while_loop + final_est_eig = temp_eig + final_est_eig_neg = temp_eig_1 + + H2[H_counter : H_counter + batch_size] = final_est_eig + H2_neg[H_counter : H_counter + batch_size] = final_est_eig_neg + H_counter += batch_size + + if order == 1: + # at the end of each iteration: get the per-iteration max gradient norm + if self.compute_slope: + L2_max[iters] = np.max(L2) + L1_max[iters] = np.max(L1) + Li_max[iters] = np.max(Li) + G2_max[iters] = np.max(G2) + G1_max[iters] = np.max(G1) + Gi_max[iters] = np.max(Gi) + + + if self.compute_slope: + print('[STATS][L2] loop = {}, time = {:.5g}, iter_time = {:.5g}, overhead = {:.5g}, L2 = {:.5g}, L1 = {:.5g}, Linf = {:.5g}, G2 = {:.5g}, G1 = {:.5g}, Ginf = {:.5g}'.format(iters, time.time() - search_begin_time, time.time() - iter_begin_time, overhead_time, L2_max[iters], L1_max[iters], Li_max[iters], G2_max[iters], G1_max[iters], Gi_max[iters])) + else: + print('[STATS][L2] loop = {}, time = {:.5g}, iter_time = {:.5g}, overhead = {:.5g}, G2 = {:.5g}, G1 = {:.5g}, Ginf = {:.5g}'.format(iters, time.time() - search_begin_time, time.time() - iter_begin_time, overhead_time, G2_max[iters], G1_max[iters], Gi_max[iters])) + sys.stdout.flush() + # reset per iteration L and G by filling 0 + if self.compute_slope: + L2.fill(0) + L1.fill(0) + Li.fill(0) + G2.fill(0) + G1.fill(0) + Gi.fill(0) + elif order == 2: + ## consider -lambda_min + idx = H2 > 0 + H2[idx] = H2_neg[idx] + idx_max = np.argmax(abs(H2)) + H2_max[iters] = H2[idx_max] + + print('[STATS][L2] loop = {}, time = {:.5g}, iter_time = {:.5g}, overhead = {:.5g}, H2 = {:.5g}'.format(iters, time.time() - search_begin_time, time.time() - iter_begin_time, overhead_time, H2_max[iters])) + + if order == 1: + print('[STATS][L1] g_x0 = {:.5g}, L2_max = {:.5g}, L1_max = {:.5g}, Linf_max = {:.5g}, G2_max = {:.5g}, G1_max = {:.5g}, Ginf_max = {:.5g}'.format( + g_x0, np.max(L2_max), np.max(L1_max), np.max(Li_max), np.max(G2_max), np.max(G1_max), np.max(Gi_max))) + # when compute the bound we need the DUAL norm + if self.compute_slope: + print('[STATS][L1] bnd_L2_max = {:.5g}, bnd_L1_max = {:.5g}, bnd_Linf_max = {:.5g}, bnd_G2_max = {:.5g}, bnd_G1_max = {:.5g}, bnd_Ginf_max = {:.5g}'.format(g_x0/np.max(L2_max), g_x0/np.max(Li_max), g_x0/np.max(L1_max), g_x0/np.max(G2_max), g_x0/np.max(Gi_max), g_x0/np.max(G1_max))) + else: + print('[STATS][L1] bnd_G2_max = {:.5g}, bnd_G1_max = {:.5g}, bnd_Ginf_max = {:.5g}'.format(g_x0/np.max(G2_max), g_x0/np.max(Gi_max), g_x0/np.max(G1_max))) + + sys.stdout.flush() + + # discard the last batch of samples + sample_results.get() + return [L2_max,L1_max,Li_max,G2_max,G1_max,Gi_max,g_x0,pred] + + elif order == 2: + # find positive eig value and substitute with its corresponding negative eig value, then we only need to sort once + + #print("H2_max = {}".format(H2_max)) + # find max abs(H2_max) + H2_max_val = max(abs(H2_max)) + + print('[STATS][L1] g_x0 = {:.5g}, g_x0_grad_2_norm = {:.5g}, g_x0_grad_1_norm = {:.5g}, g_x0_grad_inf_norm = {:.5g}, H2_max = {:.5g}'.format(g_x0, g_x0_grad_2_norm, g_x0_grad_1_norm, g_x0_grad_inf_norm, H2_max_val)) + + bnd = (-g_x0_grad_2_norm + np.sqrt(g_x0_grad_2_norm**2+2*g_x0*H2_max_val))/H2_max_val + print('[STATS][L1] bnd_H2_max = {:.5g}'.format(bnd)) + sys.stdout.flush() + + sample_results.get() + return [H2_max, g_x0, g_x0_grad_2_norm, g_x0_grad_1_norm, g_x0_grad_inf_norm, pred] + + + def _compute_max_abseig(self, pt_hvs, batch_inputs, true_label, target_label, max_eig_iters, print_flag): + + ## compute hv and est_eig: + i = 0 + cond = False + + pt_eigs = [] + + print("pt_hvs[0] shape = {}".format(pt_hvs[0].shape)) + + # perform power iteration loop outside tensorflow + while (i 0: + cond_element = abs(tmp_est_eig-pt_eigs[i-1]) < 1e-3 + if print_flag: + print("cond = {}".format(cond_element)) + cond = cond_element.all() + i+=1 + + if i == max_eig_iters: + print("==== Reach max iterations!!! ====") + + return pt_eigs[-1] + + + def __del__(self): + # terminate the pool + self.pool.terminate() + + def estimate(self, x_0, true_label, target_label, Nsamp, Niters, sample_norm, transform, order): + result = self._estimate_Lipschitz_multiplerun(Nsamp,Niters,x_0,target_label,true_label,sample_norm, transform, order) + return result + + diff --git a/Robustness_Eval/CLEVER/labels/imagenet_val_to_carlini.py b/Robustness_Eval/CLEVER/labels/imagenet_val_to_carlini.py new file mode 100644 index 0000000..b9a7753 --- /dev/null +++ b/Robustness_Eval/CLEVER/labels/imagenet_val_to_carlini.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import os +import sys +import glob + +f = open('label2num.txt') + +mapping = {} +for line in f: + l = line.strip().split(':') + mapping[l[1]] = l[0] + +print("Total {} classes loaded".format(len(mapping))) + +os.system("mkdir -p imgs") + +file_list = glob.glob('val/**/*.JPEG', recursive=True) +print("Total {} files found".format(len(file_list))) + +cur = 1 +total = len(file_list) + +for img_path in file_list: + s = img_path.split('/')[1] + n = os.path.splitext(os.path.basename(img_path))[0].split('_')[2] + label = mapping[s] + os.system("cp {} imgs/{}.{}.jpg".format(img_path, label, n)) + if cur % 1000 == 0: + print("{}/{} finished ".format(cur, total)) + cur += 1 + +print() + diff --git a/Robustness_Eval/CLEVER/labels/label2num.txt b/Robustness_Eval/CLEVER/labels/label2num.txt new file mode 100644 index 0000000..e12899c --- /dev/null +++ b/Robustness_Eval/CLEVER/labels/label2num.txt @@ -0,0 +1,1000 @@ +1:n01440764:tench, Tinca tinca +2:n01443537:goldfish, Carassius auratus +3:n01484850:great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias +4:n01491361:tiger shark, Galeocerdo cuvieri +5:n01494475:hammerhead, hammerhead shark +6:n01496331:electric ray, crampfish, numbfish, torpedo +7:n01498041:stingray +8:n01514668:cock +9:n01514859:hen +10:n01518878:ostrich, Struthio camelus +11:n01530575:brambling, Fringilla montifringilla +12:n01531178:goldfinch, Carduelis carduelis +13:n01532829:house finch, linnet, Carpodacus mexicanus +14:n01534433:junco, snowbird +15:n01537544:indigo bunting, indigo finch, indigo bird, Passerina cyanea +16:n01558993:robin, American robin, Turdus migratorius +17:n01560419:bulbul +18:n01580077:jay +19:n01582220:magpie +20:n01592084:chickadee +21:n01601694:water ouzel, dipper +22:n01608432:kite +23:n01614925:bald eagle, American eagle, Haliaeetus leucocephalus +24:n01616318:vulture +25:n01622779:great grey owl, great gray owl, Strix nebulosa +26:n01629819:European fire salamander, Salamandra salamandra +27:n01630670:common newt, Triturus vulgaris +28:n01631663:eft +29:n01632458:spotted salamander, Ambystoma maculatum +30:n01632777:axolotl, mud puppy, Ambystoma mexicanum +31:n01641577:bullfrog, Rana catesbeiana +32:n01644373:tree frog, tree-frog +33:n01644900:tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +34:n01664065:loggerhead, loggerhead turtle, Caretta caretta +35:n01665541:leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +36:n01667114:mud turtle +37:n01667778:terrapin +38:n01669191:box turtle, box tortoise +39:n01675722:banded gecko +40:n01677366:common iguana, iguana, Iguana iguana +41:n01682714:American chameleon, anole, Anolis carolinensis +42:n01685808:whiptail, whiptail lizard +43:n01687978:agama +44:n01688243:frilled lizard, Chlamydosaurus kingi +45:n01689811:alligator lizard +46:n01692333:Gila monster, Heloderma suspectum +47:n01693334:green lizard, Lacerta viridis +48:n01694178:African chameleon, Chamaeleo chamaeleon +49:n01695060:Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +50:n01697457:African crocodile, Nile crocodile, Crocodylus niloticus +51:n01698640:American alligator, Alligator mississipiensis +52:n01704323:triceratops +53:n01728572:thunder snake, worm snake, Carphophis amoenus +54:n01728920:ringneck snake, ring-necked snake, ring snake +55:n01729322:hognose snake, puff adder, sand viper +56:n01729977:green snake, grass snake +57:n01734418:king snake, kingsnake +58:n01735189:garter snake, grass snake +59:n01737021:water snake +60:n01739381:vine snake +61:n01740131:night snake, Hypsiglena torquata +62:n01742172:boa constrictor, Constrictor constrictor +63:n01744401:rock python, rock snake, Python sebae +64:n01748264:Indian cobra, Naja naja +65:n01749939:green mamba +66:n01751748:sea snake +67:n01753488:horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +68:n01755581:diamondback, diamondback rattlesnake, Crotalus adamanteus +69:n01756291:sidewinder, horned rattlesnake, Crotalus cerastes +70:n01768244:trilobite +71:n01770081:harvestman, daddy longlegs, Phalangium opilio +72:n01770393:scorpion +73:n01773157:black and gold garden spider, Argiope aurantia +74:n01773549:barn spider, Araneus cavaticus +75:n01773797:garden spider, Aranea diademata +76:n01774384:black widow, Latrodectus mactans +77:n01774750:tarantula +78:n01775062:wolf spider, hunting spider +79:n01776313:tick +80:n01784675:centipede +81:n01795545:black grouse +82:n01796340:ptarmigan +83:n01797886:ruffed grouse, partridge, Bonasa umbellus +84:n01798484:prairie chicken, prairie grouse, prairie fowl +85:n01806143:peacock +86:n01806567:quail +87:n01807496:partridge +88:n01817953:African grey, African gray, Psittacus erithacus +89:n01818515:macaw +90:n01819313:sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +91:n01820546:lorikeet +92:n01824575:coucal +93:n01828970:bee eater +94:n01829413:hornbill +95:n01833805:hummingbird +96:n01843065:jacamar +97:n01843383:toucan +98:n01847000:drake +99:n01855032:red-breasted merganser, Mergus serrator +100:n01855672:goose +101:n01860187:black swan, Cygnus atratus +102:n01871265:tusker +103:n01872401:echidna, spiny anteater, anteater +104:n01873310:platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus +105:n01877812:wallaby, brush kangaroo +106:n01882714:koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +107:n01883070:wombat +108:n01910747:jellyfish +109:n01914609:sea anemone, anemone +110:n01917289:brain coral +111:n01924916:flatworm, platyhelminth +112:n01930112:nematode, nematode worm, roundworm +113:n01943899:conch +114:n01944390:snail +115:n01945685:slug +116:n01950731:sea slug, nudibranch +117:n01955084:chiton, coat-of-mail shell, sea cradle, polyplacophore +118:n01968897:chambered nautilus, pearly nautilus, nautilus +119:n01978287:Dungeness crab, Cancer magister +120:n01978455:rock crab, Cancer irroratus +121:n01980166:fiddler crab +122:n01981276:king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica +123:n01983481:American lobster, Northern lobster, Maine lobster, Homarus americanus +124:n01984695:spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +125:n01985128:crayfish, crawfish, crawdad, crawdaddy +126:n01986214:hermit crab +127:n01990800:isopod +128:n02002556:white stork, Ciconia ciconia +129:n02002724:black stork, Ciconia nigra +130:n02006656:spoonbill +131:n02007558:flamingo +132:n02009229:little blue heron, Egretta caerulea +133:n02009912:American egret, great white heron, Egretta albus +134:n02011460:bittern +518:n02012849:crane +136:n02013706:limpkin, Aramus pictus +137:n02017213:European gallinule, Porphyrio porphyrio +138:n02018207:American coot, marsh hen, mud hen, water hen, Fulica americana +139:n02018795:bustard +140:n02025239:ruddy turnstone, Arenaria interpres +141:n02027492:red-backed sandpiper, dunlin, Erolia alpina +142:n02028035:redshank, Tringa totanus +143:n02033041:dowitcher +144:n02037110:oystercatcher, oyster catcher +145:n02051845:pelican +146:n02056570:king penguin, Aptenodytes patagonica +147:n02058221:albatross, mollymawk +148:n02066245:grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus +149:n02071294:killer whale, killer, orca, grampus, sea wolf, Orcinus orca +150:n02074367:dugong, Dugong dugon +151:n02077923:sea lion +152:n02085620:Chihuahua +153:n02085782:Japanese spaniel +154:n02085936:Maltese dog, Maltese terrier, Maltese +155:n02086079:Pekinese, Pekingese, Peke +156:n02086240:Shih-Tzu +157:n02086646:Blenheim spaniel +158:n02086910:papillon +159:n02087046:toy terrier +160:n02087394:Rhodesian ridgeback +161:n02088094:Afghan hound, Afghan +162:n02088238:basset, basset hound +163:n02088364:beagle +164:n02088466:bloodhound, sleuthhound +165:n02088632:bluetick +166:n02089078:black-and-tan coonhound +167:n02089867:Walker hound, Walker foxhound +168:n02089973:English foxhound +169:n02090379:redbone +170:n02090622:borzoi, Russian wolfhound +171:n02090721:Irish wolfhound +172:n02091032:Italian greyhound +173:n02091134:whippet +174:n02091244:Ibizan hound, Ibizan Podenco +175:n02091467:Norwegian elkhound, elkhound +176:n02091635:otterhound, otter hound +177:n02091831:Saluki, gazelle hound +178:n02092002:Scottish deerhound, deerhound +179:n02092339:Weimaraner +180:n02093256:Staffordshire bullterrier, Staffordshire bull terrier +181:n02093428:American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier +182:n02093647:Bedlington terrier +183:n02093754:Border terrier +184:n02093859:Kerry blue terrier +185:n02093991:Irish terrier +186:n02094114:Norfolk terrier +187:n02094258:Norwich terrier +188:n02094433:Yorkshire terrier +189:n02095314:wire-haired fox terrier +190:n02095570:Lakeland terrier +191:n02095889:Sealyham terrier, Sealyham +192:n02096051:Airedale, Airedale terrier +193:n02096177:cairn, cairn terrier +194:n02096294:Australian terrier +195:n02096437:Dandie Dinmont, Dandie Dinmont terrier +196:n02096585:Boston bull, Boston terrier +197:n02097047:miniature schnauzer +198:n02097130:giant schnauzer +199:n02097209:standard schnauzer +200:n02097298:Scotch terrier, Scottish terrier, Scottie +201:n02097474:Tibetan terrier, chrysanthemum dog +202:n02097658:silky terrier, Sydney silky +203:n02098105:soft-coated wheaten terrier +204:n02098286:West Highland white terrier +205:n02098413:Lhasa, Lhasa apso +206:n02099267:flat-coated retriever +207:n02099429:curly-coated retriever +208:n02099601:golden retriever +209:n02099712:Labrador retriever +210:n02099849:Chesapeake Bay retriever +211:n02100236:German short-haired pointer +212:n02100583:vizsla, Hungarian pointer +213:n02100735:English setter +214:n02100877:Irish setter, red setter +215:n02101006:Gordon setter +216:n02101388:Brittany spaniel +217:n02101556:clumber, clumber spaniel +218:n02102040:English springer, English springer spaniel +219:n02102177:Welsh springer spaniel +220:n02102318:cocker spaniel, English cocker spaniel, cocker +221:n02102480:Sussex spaniel +222:n02102973:Irish water spaniel +223:n02104029:kuvasz +224:n02104365:schipperke +225:n02105056:groenendael +226:n02105162:malinois +227:n02105251:briard +228:n02105412:kelpie +229:n02105505:komondor +230:n02105641:Old English sheepdog, bobtail +231:n02105855:Shetland sheepdog, Shetland sheep dog, Shetland +232:n02106030:collie +233:n02106166:Border collie +234:n02106382:Bouvier des Flandres, Bouviers des Flandres +235:n02106550:Rottweiler +236:n02106662:German shepherd, German shepherd dog, German police dog, alsatian +237:n02107142:Doberman, Doberman pinscher +238:n02107312:miniature pinscher +239:n02107574:Greater Swiss Mountain dog +240:n02107683:Bernese mountain dog +241:n02107908:Appenzeller +242:n02108000:EntleBucher +243:n02108089:boxer +244:n02108422:bull mastiff +245:n02108551:Tibetan mastiff +246:n02108915:French bulldog +247:n02109047:Great Dane +248:n02109525:Saint Bernard, St Bernard +249:n02109961:Eskimo dog, husky +250:n02110063:malamute, malemute, Alaskan malamute +251:n02110185:Siberian husky +252:n02110341:dalmatian, coach dog, carriage dog +253:n02110627:affenpinscher, monkey pinscher, monkey dog +254:n02110806:basenji +255:n02110958:pug, pug-dog +256:n02111129:Leonberg +257:n02111277:Newfoundland, Newfoundland dog +258:n02111500:Great Pyrenees +259:n02111889:Samoyed, Samoyede +260:n02112018:Pomeranian +261:n02112137:chow, chow chow +262:n02112350:keeshond +263:n02112706:Brabancon griffon +264:n02113023:Pembroke, Pembroke Welsh corgi +265:n02113186:Cardigan, Cardigan Welsh corgi +266:n02113624:toy poodle +267:n02113712:miniature poodle +268:n02113799:standard poodle +269:n02113978:Mexican hairless +270:n02114367:timber wolf, grey wolf, gray wolf, Canis lupus +271:n02114548:white wolf, Arctic wolf, Canis lupus tundrarum +272:n02114712:red wolf, maned wolf, Canis rufus, Canis niger +273:n02114855:coyote, prairie wolf, brush wolf, Canis latrans +274:n02115641:dingo, warrigal, warragal, Canis dingo +275:n02115913:dhole, Cuon alpinus +276:n02116738:African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +277:n02117135:hyena, hyaena +278:n02119022:red fox, Vulpes vulpes +279:n02119789:kit fox, Vulpes macrotis +280:n02120079:Arctic fox, white fox, Alopex lagopus +281:n02120505:grey fox, gray fox, Urocyon cinereoargenteus +282:n02123045:tabby, tabby cat +283:n02123159:tiger cat +284:n02123394:Persian cat +285:n02123597:Siamese cat, Siamese +286:n02124075:Egyptian cat +287:n02125311:cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +288:n02127052:lynx, catamount +289:n02128385:leopard, Panthera pardus +290:n02128757:snow leopard, ounce, Panthera uncia +291:n02128925:jaguar, panther, Panthera onca, Felis onca +292:n02129165:lion, king of beasts, Panthera leo +293:n02129604:tiger, Panthera tigris +294:n02130308:cheetah, chetah, Acinonyx jubatus +295:n02132136:brown bear, bruin, Ursus arctos +296:n02133161:American black bear, black bear, Ursus americanus, Euarctos americanus +297:n02134084:ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +298:n02134418:sloth bear, Melursus ursinus, Ursus ursinus +299:n02137549:mongoose +300:n02138441:meerkat, mierkat +301:n02165105:tiger beetle +302:n02165456:ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +303:n02167151:ground beetle, carabid beetle +304:n02168699:long-horned beetle, longicorn, longicorn beetle +305:n02169497:leaf beetle, chrysomelid +306:n02172182:dung beetle +307:n02174001:rhinoceros beetle +308:n02177972:weevil +309:n02190166:fly +310:n02206856:bee +311:n02219486:ant, emmet, pismire +312:n02226429:grasshopper, hopper +313:n02229544:cricket +314:n02231487:walking stick, walkingstick, stick insect +315:n02233338:cockroach, roach +316:n02236044:mantis, mantid +317:n02256656:cicada, cicala +318:n02259212:leafhopper +319:n02264363:lacewing, lacewing fly +320:n02268443:dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk +321:n02268853:damselfly +322:n02276258:admiral +323:n02277742:ringlet, ringlet butterfly +324:n02279972:monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +325:n02280649:cabbage butterfly +326:n02281406:sulphur butterfly, sulfur butterfly +327:n02281787:lycaenid, lycaenid butterfly +328:n02317335:starfish, sea star +329:n02319095:sea urchin +330:n02321529:sea cucumber, holothurian +331:n02325366:wood rabbit, cottontail, cottontail rabbit +332:n02326432:hare +333:n02328150:Angora, Angora rabbit +334:n02342885:hamster +335:n02346627:porcupine, hedgehog +336:n02356798:fox squirrel, eastern fox squirrel, Sciurus niger +337:n02361337:marmot +338:n02363005:beaver +339:n02364673:guinea pig, Cavia cobaya +340:n02389026:sorrel +341:n02391049:zebra +342:n02395406:hog, pig, grunter, squealer, Sus scrofa +343:n02396427:wild boar, boar, Sus scrofa +344:n02397096:warthog +345:n02398521:hippopotamus, hippo, river horse, Hippopotamus amphibius +346:n02403003:ox +347:n02408429:water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +348:n02410509:bison +349:n02412080:ram, tup +350:n02415577:bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis +351:n02417914:ibex, Capra ibex +352:n02422106:hartebeest +353:n02422699:impala, Aepyceros melampus +354:n02423022:gazelle +355:n02437312:Arabian camel, dromedary, Camelus dromedarius +356:n02437616:llama +357:n02441942:weasel +358:n02442845:mink +359:n02443114:polecat, fitch, foulmart, foumart, Mustela putorius +360:n02443484:black-footed ferret, ferret, Mustela nigripes +361:n02444819:otter +362:n02445715:skunk, polecat, wood pussy +363:n02447366:badger +364:n02454379:armadillo +365:n02457408:three-toed sloth, ai, Bradypus tridactylus +366:n02480495:orangutan, orang, orangutang, Pongo pygmaeus +367:n02480855:gorilla, Gorilla gorilla +368:n02481823:chimpanzee, chimp, Pan troglodytes +369:n02483362:gibbon, Hylobates lar +370:n02483708:siamang, Hylobates syndactylus, Symphalangus syndactylus +371:n02484975:guenon, guenon monkey +372:n02486261:patas, hussar monkey, Erythrocebus patas +373:n02486410:baboon +374:n02487347:macaque +375:n02488291:langur +376:n02488702:colobus, colobus monkey +377:n02489166:proboscis monkey, Nasalis larvatus +378:n02490219:marmoset +379:n02492035:capuchin, ringtail, Cebus capucinus +380:n02492660:howler monkey, howler +381:n02493509:titi, titi monkey +382:n02493793:spider monkey, Ateles geoffroyi +383:n02494079:squirrel monkey, Saimiri sciureus +384:n02497673:Madagascar cat, ring-tailed lemur, Lemur catta +385:n02500267:indri, indris, Indri indri, Indri brevicaudatus +386:n02504013:Indian elephant, Elephas maximus +387:n02504458:African elephant, Loxodonta africana +388:n02509815:lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +389:n02510455:giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +390:n02514041:barracouta, snoek +391:n02526121:eel +392:n02536864:coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +393:n02606052:rock beauty, Holocanthus tricolor +394:n02607072:anemone fish +395:n02640242:sturgeon +396:n02641379:gar, garfish, garpike, billfish, Lepisosteus osseus +397:n02643566:lionfish +398:n02655020:puffer, pufferfish, blowfish, globefish +399:n02666196:abacus +400:n02667093:abaya +401:n02669723:academic gown, academic robe, judge's robe +402:n02672831:accordion, piano accordion, squeeze box +403:n02676566:acoustic guitar +404:n02687172:aircraft carrier, carrier, flattop, attack aircraft carrier +405:n02690373:airliner +406:n02692877:airship, dirigible +407:n02699494:altar +408:n02701002:ambulance +409:n02704792:amphibian, amphibious vehicle +410:n02708093:analog clock +411:n02727426:apiary, bee house +412:n02730930:apron +413:n02747177:ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin +414:n02749479:assault rifle, assault gun +415:n02769748:backpack, back pack, knapsack, packsack, rucksack, haversack +416:n02776631:bakery, bakeshop, bakehouse +417:n02777292:balance beam, beam +418:n02782093:balloon +419:n02783161:ballpoint, ballpoint pen, ballpen, Biro +420:n02786058:Band Aid +421:n02787622:banjo +422:n02788148:bannister, banister, balustrade, balusters, handrail +423:n02790996:barbell +424:n02791124:barber chair +425:n02791270:barbershop +426:n02793495:barn +427:n02794156:barometer +428:n02795169:barrel, cask +429:n02797295:barrow, garden cart, lawn cart, wheelbarrow +430:n02799071:baseball +431:n02802426:basketball +432:n02804414:bassinet +433:n02804610:bassoon +434:n02807133:bathing cap, swimming cap +435:n02808304:bath towel +436:n02808440:bathtub, bathing tub, bath, tub +437:n02814533:beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon +438:n02814860:beacon, lighthouse, beacon light, pharos +439:n02815834:beaker +440:n02817516:bearskin, busby, shako +441:n02823428:beer bottle +442:n02823750:beer glass +443:n02825657:bell cote, bell cot +444:n02834397:bib +445:n02835271:bicycle-built-for-two, tandem bicycle, tandem +446:n02837789:bikini, two-piece +447:n02840245:binder, ring-binder +448:n02841315:binoculars, field glasses, opera glasses +449:n02843684:birdhouse +450:n02859443:boathouse +451:n02860847:bobsled, bobsleigh, bob +452:n02865351:bolo tie, bolo, bola tie, bola +453:n02869837:bonnet, poke bonnet +454:n02870880:bookcase +455:n02871525:bookshop, bookstore, bookstall +456:n02877765:bottlecap +457:n02879718:bow +458:n02883205:bow tie, bow-tie, bowtie +459:n02892201:brass, memorial tablet, plaque +460:n02892767:brassiere, bra, bandeau +461:n02894605:breakwater, groin, groyne, mole, bulwark, seawall, jetty +462:n02895154:breastplate, aegis, egis +463:n02906734:broom +464:n02909870:bucket, pail +465:n02910353:buckle +466:n02916936:bulletproof vest +467:n02917067:bullet train, bullet +468:n02927161:butcher shop, meat market +469:n02930766:cab, hack, taxi, taxicab +470:n02939185:caldron, cauldron +471:n02948072:candle, taper, wax light +472:n02950826:cannon +473:n02951358:canoe +474:n02951585:can opener, tin opener +475:n02963159:cardigan +476:n02965783:car mirror +477:n02966193:carousel, carrousel, merry-go-round, roundabout, whirligig +478:n02966687:carpenter's kit, tool kit +479:n02971356:carton +480:n02974003:car wheel +481:n02977058:cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM +482:n02978881:cassette +483:n02979186:cassette player +484:n02980441:castle +485:n02981792:catamaran +486:n02988304:CD player +487:n02992211:cello, violoncello +488:n02992529:cellular telephone, cellular phone, cellphone, cell, mobile phone +489:n02999410:chain +490:n03000134:chainlink fence +491:n03000247:chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour +492:n03000684:chain saw, chainsaw +493:n03014705:chest +494:n03016953:chiffonier, commode +495:n03017168:chime, bell, gong +496:n03018349:china cabinet, china closet +497:n03026506:Christmas stocking +498:n03028079:church, church building +499:n03032252:cinema, movie theater, movie theatre, movie house, picture palace +500:n03041632:cleaver, meat cleaver, chopper +501:n03042490:cliff dwelling +502:n03045698:cloak +503:n03047690:clog, geta, patten, sabot +504:n03062245:cocktail shaker +505:n03063599:coffee mug +506:n03063689:coffeepot +507:n03065424:coil, spiral, volute, whorl, helix +508:n03075370:combination lock +509:n03085013:computer keyboard, keypad +510:n03089624:confectionery, confectionary, candy store +511:n03095699:container ship, containership, container vessel +512:n03100240:convertible +513:n03109150:corkscrew, bottle screw +514:n03110669:cornet, horn, trumpet, trump +515:n03124043:cowboy boot +516:n03124170:cowboy hat, ten-gallon hat +517:n03125729:cradle +518:n03126707:crane +519:n03127747:crash helmet +520:n03127925:crate +521:n03131574:crib, cot +522:n03133878:Crock Pot +523:n03134739:croquet ball +524:n03141823:crutch +525:n03146219:cuirass +526:n03160309:dam, dike, dyke +527:n03179701:desk +528:n03180011:desktop computer +529:n03187595:dial telephone, dial phone +530:n03188531:diaper, nappy, napkin +531:n03196217:digital clock +532:n03197337:digital watch +533:n03201208:dining table, board +534:n03207743:dishrag, dishcloth +535:n03207941:dishwasher, dish washer, dishwashing machine +536:n03208938:disk brake, disc brake +537:n03216828:dock, dockage, docking facility +538:n03218198:dogsled, dog sled, dog sleigh +539:n03220513:dome +540:n03223299:doormat, welcome mat +541:n03240683:drilling platform, offshore rig +542:n03249569:drum, membranophone, tympan +543:n03250847:drumstick +544:n03255030:dumbbell +545:n03259280:Dutch oven +546:n03271574:electric fan, blower +547:n03272010:electric guitar +548:n03272562:electric locomotive +549:n03290653:entertainment center +550:n03291819:envelope +551:n03297495:espresso maker +552:n03314780:face powder +553:n03325584:feather boa, boa +554:n03337140:file, file cabinet, filing cabinet +555:n03344393:fireboat +556:n03345487:fire engine, fire truck +557:n03347037:fire screen, fireguard +558:n03355925:flagpole, flagstaff +559:n03372029:flute, transverse flute +560:n03376595:folding chair +561:n03379051:football helmet +562:n03384352:forklift +563:n03388043:fountain +564:n03388183:fountain pen +565:n03388549:four-poster +566:n03393912:freight car +567:n03394916:French horn, horn +568:n03400231:frying pan, frypan, skillet +569:n03404251:fur coat +570:n03417042:garbage truck, dustcart +571:n03424325:gasmask, respirator, gas helmet +572:n03425413:gas pump, gasoline pump, petrol pump, island dispenser +573:n03443371:goblet +574:n03444034:go-kart +575:n03445777:golf ball +576:n03445924:golfcart, golf cart +577:n03447447:gondola +578:n03447721:gong, tam-tam +579:n03450230:gown +580:n03452741:grand piano, grand +581:n03457902:greenhouse, nursery, glasshouse +582:n03459775:grille, radiator grille +583:n03461385:grocery store, grocery, food market, market +584:n03467068:guillotine +585:n03476684:hair slide +586:n03476991:hair spray +587:n03478589:half track +588:n03481172:hammer +589:n03482405:hamper +590:n03483316:hand blower, blow dryer, blow drier, hair dryer, hair drier +591:n03485407:hand-held computer, hand-held microcomputer +592:n03485794:handkerchief, hankie, hanky, hankey +593:n03492542:hard disc, hard disk, fixed disk +594:n03494278:harmonica, mouth organ, harp, mouth harp +595:n03495258:harp +596:n03496892:harvester, reaper +597:n03498962:hatchet +598:n03527444:holster +599:n03529860:home theater, home theatre +600:n03530642:honeycomb +601:n03532672:hook, claw +602:n03534580:hoopskirt, crinoline +603:n03535780:horizontal bar, high bar +604:n03538406:horse cart, horse-cart +605:n03544143:hourglass +606:n03584254:iPod +607:n03584829:iron, smoothing iron +608:n03590841:jack-o'-lantern +609:n03594734:jean, blue jean, denim +610:n03594945:jeep, landrover +611:n03595614:jersey, T-shirt, tee shirt +612:n03598930:jigsaw puzzle +613:n03599486:jinrikisha, ricksha, rickshaw +614:n03602883:joystick +615:n03617480:kimono +616:n03623198:knee pad +617:n03627232:knot +618:n03630383:lab coat, laboratory coat +619:n03633091:ladle +620:n03637318:lampshade, lamp shade +621:n03642806:laptop, laptop computer +622:n03649909:lawn mower, mower +623:n03657121:lens cap, lens cover +624:n03658185:letter opener, paper knife, paperknife +625:n03661043:library +626:n03662601:lifeboat +627:n03666591:lighter, light, igniter, ignitor +628:n03670208:limousine, limo +629:n03673027:liner, ocean liner +630:n03676483:lipstick, lip rouge +631:n03680355:Loafer +632:n03690938:lotion +633:n03691459:loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +634:n03692522:loupe, jeweler's loupe +635:n03697007:lumbermill, sawmill +636:n03706229:magnetic compass +637:n03709823:mailbag, postbag +638:n03710193:mailbox, letter box +639:n03710637:maillot +640:n03710721:maillot, tank suit +641:n03717622:manhole cover +642:n03720891:maraca +643:n03721384:marimba, xylophone +644:n03724870:mask +645:n03729826:matchstick +646:n03733131:maypole +647:n03733281:maze, labyrinth +648:n03733805:measuring cup +649:n03742115:medicine chest, medicine cabinet +650:n03743016:megalith, megalithic structure +651:n03759954:microphone, mike +652:n03761084:microwave, microwave oven +653:n03763968:military uniform +654:n03764736:milk can +655:n03769881:minibus +656:n03770439:miniskirt, mini +657:n03770679:minivan +658:n03773504:missile +659:n03775071:mitten +660:n03775546:mixing bowl +661:n03776460:mobile home, manufactured home +662:n03777568:Model T +663:n03777754:modem +664:n03781244:monastery +665:n03782006:monitor +666:n03785016:moped +667:n03786901:mortar +668:n03787032:mortarboard +669:n03788195:mosque +670:n03788365:mosquito net +671:n03791053:motor scooter, scooter +672:n03792782:mountain bike, all-terrain bike, off-roader +673:n03792972:mountain tent +674:n03793489:mouse, computer mouse +675:n03794056:mousetrap +676:n03796401:moving van +677:n03803284:muzzle +678:n03804744:nail +679:n03814639:neck brace +680:n03814906:necklace +681:n03825788:nipple +682:n03832673:notebook, notebook computer +683:n03837869:obelisk +684:n03838899:oboe, hautboy, hautbois +685:n03840681:ocarina, sweet potato +686:n03841143:odometer, hodometer, mileometer, milometer +687:n03843555:oil filter +688:n03854065:organ, pipe organ +689:n03857828:oscilloscope, scope, cathode-ray oscilloscope, CRO +690:n03866082:overskirt +691:n03868242:oxcart +692:n03868863:oxygen mask +693:n03871628:packet +694:n03873416:paddle, boat paddle +695:n03874293:paddlewheel, paddle wheel +696:n03874599:padlock +697:n03876231:paintbrush +698:n03877472:pajama, pyjama, pj's, jammies +699:n03877845:palace +700:n03884397:panpipe, pandean pipe, syrinx +701:n03887697:paper towel +702:n03888257:parachute, chute +703:n03888605:parallel bars, bars +704:n03891251:park bench +705:n03891332:parking meter +706:n03895866:passenger car, coach, carriage +707:n03899768:patio, terrace +708:n03902125:pay-phone, pay-station +709:n03903868:pedestal, plinth, footstall +710:n03908618:pencil box, pencil case +711:n03908714:pencil sharpener +712:n03916031:perfume, essence +713:n03920288:Petri dish +714:n03924679:photocopier +715:n03929660:pick, plectrum, plectron +716:n03929855:pickelhaube +717:n03930313:picket fence, paling +718:n03930630:pickup, pickup truck +719:n03933933:pier +720:n03935335:piggy bank, penny bank +721:n03937543:pill bottle +722:n03938244:pillow +723:n03942813:ping-pong ball +724:n03944341:pinwheel +725:n03947888:pirate, pirate ship +726:n03950228:pitcher, ewer +727:n03954731:plane, carpenter's plane, woodworking plane +728:n03956157:planetarium +729:n03958227:plastic bag +730:n03961711:plate rack +731:n03967562:plow, plough +732:n03970156:plunger, plumber's helper +733:n03976467:Polaroid camera, Polaroid Land camera +734:n03976657:pole +735:n03977966:police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +736:n03980874:poncho +737:n03982430:pool table, billiard table, snooker table +738:n03983396:pop bottle, soda bottle +739:n03991062:pot, flowerpot +740:n03992509:potter's wheel +741:n03995372:power drill +742:n03998194:prayer rug, prayer mat +743:n04004767:printer +744:n04005630:prison, prison house +745:n04008634:projectile, missile +746:n04009552:projector +747:n04019541:puck, hockey puck +748:n04023962:punching bag, punch bag, punching ball, punchball +749:n04026417:purse +750:n04033901:quill, quill pen +751:n04033995:quilt, comforter, comfort, puff +752:n04037443:racer, race car, racing car +753:n04039381:racket, racquet +754:n04040759:radiator +755:n04041544:radio, wireless +756:n04044716:radio telescope, radio reflector +757:n04049303:rain barrel +758:n04065272:recreational vehicle, RV, R.V. +759:n04067472:reel +760:n04069434:reflex camera +761:n04070727:refrigerator, icebox +762:n04074963:remote control, remote +763:n04081281:restaurant, eating house, eating place, eatery +764:n04086273:revolver, six-gun, six-shooter +765:n04090263:rifle +766:n04099969:rocking chair, rocker +767:n04111531:rotisserie +768:n04116512:rubber eraser, rubber, pencil eraser +769:n04118538:rugby ball +770:n04118776:rule, ruler +771:n04120489:running shoe +772:n04125021:safe +773:n04127249:safety pin +774:n04131690:saltshaker, salt shaker +775:n04133789:sandal +776:n04136333:sarong +777:n04141076:sax, saxophone +778:n04141327:scabbard +779:n04141975:scale, weighing machine +780:n04146614:school bus +781:n04147183:schooner +782:n04149813:scoreboard +783:n04152593:screen, CRT screen +784:n04153751:screw +785:n04154565:screwdriver +786:n04162706:seat belt, seatbelt +787:n04179913:sewing machine +788:n04192698:shield, buckler +789:n04200800:shoe shop, shoe-shop, shoe store +790:n04201297:shoji +791:n04204238:shopping basket +792:n04204347:shopping cart +793:n04208210:shovel +794:n04209133:shower cap +795:n04209239:shower curtain +796:n04228054:ski +797:n04229816:ski mask +798:n04235860:sleeping bag +799:n04238763:slide rule, slipstick +800:n04239074:sliding door +801:n04243546:slot, one-armed bandit +802:n04251144:snorkel +803:n04252077:snowmobile +804:n04252225:snowplow, snowplough +805:n04254120:soap dispenser +806:n04254680:soccer ball +807:n04254777:sock +808:n04258138:solar dish, solar collector, solar furnace +809:n04259630:sombrero +810:n04263257:soup bowl +811:n04264628:space bar +812:n04265275:space heater +813:n04266014:space shuttle +814:n04270147:spatula +815:n04273569:speedboat +816:n04275548:spider web, spider's web +817:n04277352:spindle +818:n04285008:sports car, sport car +819:n04286575:spotlight, spot +820:n04296562:stage +821:n04310018:steam locomotive +822:n04311004:steel arch bridge +823:n04311174:steel drum +824:n04317175:stethoscope +825:n04325704:stole +826:n04326547:stone wall +827:n04328186:stopwatch, stop watch +828:n04330267:stove +829:n04332243:strainer +830:n04335435:streetcar, tram, tramcar, trolley, trolley car +831:n04336792:stretcher +832:n04344873:studio couch, day bed +833:n04346328:stupa, tope +834:n04347754:submarine, pigboat, sub, U-boat +835:n04350905:suit, suit of clothes +836:n04355338:sundial +837:n04355933:sunglass +838:n04356056:sunglasses, dark glasses, shades +839:n04357314:sunscreen, sunblock, sun blocker +840:n04366367:suspension bridge +841:n04367480:swab, swob, mop +842:n04370456:sweatshirt +843:n04371430:swimming trunks, bathing trunks +844:n04371774:swing +845:n04372370:switch, electric switch, electrical switch +846:n04376876:syringe +847:n04380533:table lamp +848:n04389033:tank, army tank, armored combat vehicle, armoured combat vehicle +849:n04392985:tape player +850:n04398044:teapot +851:n04399382:teddy, teddy bear +852:n04404412:television, television system +853:n04409515:tennis ball +854:n04417672:thatch, thatched roof +855:n04418357:theater curtain, theatre curtain +856:n04423845:thimble +857:n04428191:thresher, thrasher, threshing machine +858:n04429376:throne +859:n04435653:tile roof +860:n04442312:toaster +861:n04443257:tobacco shop, tobacconist shop, tobacconist +862:n04447861:toilet seat +863:n04456115:torch +864:n04458633:totem pole +865:n04461696:tow truck, tow car, wrecker +866:n04462240:toyshop +867:n04465501:tractor +868:n04467665:trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +869:n04476259:tray +870:n04479046:trench coat +871:n04482393:tricycle, trike, velocipede +872:n04483307:trimaran +873:n04485082:tripod +874:n04486054:triumphal arch +875:n04487081:trolleybus, trolley coach, trackless trolley +876:n04487394:trombone +877:n04493381:tub, vat +878:n04501370:turnstile +879:n04505470:typewriter keyboard +880:n04507155:umbrella +881:n04509417:unicycle, monocycle +882:n04515003:upright, upright piano +883:n04517823:vacuum, vacuum cleaner +884:n04522168:vase +885:n04523525:vault +886:n04525038:velvet +887:n04525305:vending machine +888:n04532106:vestment +889:n04532670:viaduct +890:n04536866:violin, fiddle +891:n04540053:volleyball +892:n04542943:waffle iron +893:n04548280:wall clock +894:n04548362:wallet, billfold, notecase, pocketbook +895:n04550184:wardrobe, closet, press +896:n04552348:warplane, military plane +897:n04553703:washbasin, handbasin, washbowl, lavabo, wash-hand basin +898:n04554684:washer, automatic washer, washing machine +899:n04557648:water bottle +900:n04560804:water jug +901:n04562935:water tower +902:n04579145:whiskey jug +903:n04579432:whistle +904:n04584207:wig +905:n04589890:window screen +906:n04590129:window shade +907:n04591157:Windsor tie +908:n04591713:wine bottle +909:n04592741:wing +910:n04596742:wok +911:n04597913:wooden spoon +912:n04599235:wool, woolen, woollen +913:n04604644:worm fence, snake fence, snake-rail fence, Virginia fence +914:n04606251:wreck +915:n04612504:yawl +916:n04613696:yurt +917:n06359193:web site, website, internet site, site +918:n06596364:comic book +919:n06785654:crossword puzzle, crossword +920:n06794110:street sign +921:n06874185:traffic light, traffic signal, stoplight +922:n07248320:book jacket, dust cover, dust jacket, dust wrapper +923:n07565083:menu +924:n07579787:plate +925:n07583066:guacamole +926:n07584110:consomme +927:n07590611:hot pot, hotpot +928:n07613480:trifle +929:n07614500:ice cream, icecream +930:n07615774:ice lolly, lolly, lollipop, popsicle +931:n07684084:French loaf +932:n07693725:bagel, beigel +933:n07695742:pretzel +934:n07697313:cheeseburger +935:n07697537:hotdog, hot dog, red hot +936:n07711569:mashed potato +937:n07714571:head cabbage +938:n07714990:broccoli +939:n07715103:cauliflower +940:n07716358:zucchini, courgette +941:n07716906:spaghetti squash +942:n07717410:acorn squash +943:n07717556:butternut squash +944:n07718472:cucumber, cuke +945:n07718747:artichoke, globe artichoke +946:n07720875:bell pepper +947:n07730033:cardoon +948:n07734744:mushroom +949:n07742313:Granny Smith +950:n07745940:strawberry +951:n07747607:orange +952:n07749582:lemon +953:n07753113:fig +954:n07753275:pineapple, ananas +955:n07753592:banana +956:n07754684:jackfruit, jak, jack +957:n07760859:custard apple +958:n07768694:pomegranate +959:n07802026:hay +960:n07831146:carbonara +961:n07836838:chocolate sauce, chocolate syrup +962:n07860988:dough +963:n07871810:meat loaf, meatloaf +964:n07873807:pizza, pizza pie +965:n07875152:potpie +966:n07880968:burrito +967:n07892512:red wine +968:n07920052:espresso +969:n07930864:cup +970:n07932039:eggnog +971:n09193705:alp +972:n09229709:bubble +973:n09246464:cliff, drop, drop-off +974:n09256479:coral reef +975:n09288635:geyser +976:n09332890:lakeside, lakeshore +977:n09399592:promontory, headland, head, foreland +978:n09421951:sandbar, sand bar +979:n09428293:seashore, coast, seacoast, sea-coast +980:n09468604:valley, vale +981:n09472597:volcano +982:n09835506:ballplayer, baseball player +983:n10148035:groom, bridegroom +984:n10565667:scuba diver +985:n11879895:rapeseed +986:n11939491:daisy +987:n12057211:yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum +988:n12144580:corn +989:n12267677:acorn +990:n12620546:hip, rose hip, rosehip +991:n12768682:buckeye, horse chestnut, conker +992:n12985857:coral fungus +993:n12998815:agaric +994:n13037406:gyromitra +995:n13040303:stinkhorn, carrion fungus +996:n13044778:earthstar +997:n13052670:hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +998:n13054560:bolete +999:n13133613:ear, spike, capitulum +1000:n15075141:toilet tissue, toilet paper, bathroom tissue diff --git a/Robustness_Eval/CLEVER/labels/labels.py b/Robustness_Eval/CLEVER/labels/labels.py new file mode 100644 index 0000000..5543811 --- /dev/null +++ b/Robustness_Eval/CLEVER/labels/labels.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +new_label = "labels.txt" +imagenet_map = "synset_words.txt" +label2num = "label2num.txt" + +num_dict = {} + +out_file = open(label2num, 'w') + +with open(new_label, 'r') as f: + for line in f: + if line: + num = int(line.split(':')[0]) + lab = line.split(':')[1].strip() + num_dict[lab] = num + +print(len(num_dict)) + +with open(imagenet_map, 'r') as f: + for line in f: + if line: + nn = line[:9] + lab = line[10:].strip() + print(nn, lab) + num = num_dict[lab] + out_file.write("{}:{}:{}\n".format(num, nn, lab)) + diff --git a/Robustness_Eval/CLEVER/labels/labels.txt b/Robustness_Eval/CLEVER/labels/labels.txt new file mode 100644 index 0000000..d74ff55 --- /dev/null +++ b/Robustness_Eval/CLEVER/labels/labels.txt @@ -0,0 +1,1001 @@ +0:background +1:tench, Tinca tinca +2:goldfish, Carassius auratus +3:great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias +4:tiger shark, Galeocerdo cuvieri +5:hammerhead, hammerhead shark +6:electric ray, crampfish, numbfish, torpedo +7:stingray +8:cock +9:hen +10:ostrich, Struthio camelus +11:brambling, Fringilla montifringilla +12:goldfinch, Carduelis carduelis +13:house finch, linnet, Carpodacus mexicanus +14:junco, snowbird +15:indigo bunting, indigo finch, indigo bird, Passerina cyanea +16:robin, American robin, Turdus migratorius +17:bulbul +18:jay +19:magpie +20:chickadee +21:water ouzel, dipper +22:kite +23:bald eagle, American eagle, Haliaeetus leucocephalus +24:vulture +25:great grey owl, great gray owl, Strix nebulosa +26:European fire salamander, Salamandra salamandra +27:common newt, Triturus vulgaris +28:eft +29:spotted salamander, Ambystoma maculatum +30:axolotl, mud puppy, Ambystoma mexicanum +31:bullfrog, Rana catesbeiana +32:tree frog, tree-frog +33:tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +34:loggerhead, loggerhead turtle, Caretta caretta +35:leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +36:mud turtle +37:terrapin +38:box turtle, box tortoise +39:banded gecko +40:common iguana, iguana, Iguana iguana +41:American chameleon, anole, Anolis carolinensis +42:whiptail, whiptail lizard +43:agama +44:frilled lizard, Chlamydosaurus kingi +45:alligator lizard +46:Gila monster, Heloderma suspectum +47:green lizard, Lacerta viridis +48:African chameleon, Chamaeleo chamaeleon +49:Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +50:African crocodile, Nile crocodile, Crocodylus niloticus +51:American alligator, Alligator mississipiensis +52:triceratops +53:thunder snake, worm snake, Carphophis amoenus +54:ringneck snake, ring-necked snake, ring snake +55:hognose snake, puff adder, sand viper +56:green snake, grass snake +57:king snake, kingsnake +58:garter snake, grass snake +59:water snake +60:vine snake +61:night snake, Hypsiglena torquata +62:boa constrictor, Constrictor constrictor +63:rock python, rock snake, Python sebae +64:Indian cobra, Naja naja +65:green mamba +66:sea snake +67:horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +68:diamondback, diamondback rattlesnake, Crotalus adamanteus +69:sidewinder, horned rattlesnake, Crotalus cerastes +70:trilobite +71:harvestman, daddy longlegs, Phalangium opilio +72:scorpion +73:black and gold garden spider, Argiope aurantia +74:barn spider, Araneus cavaticus +75:garden spider, Aranea diademata +76:black widow, Latrodectus mactans +77:tarantula +78:wolf spider, hunting spider +79:tick +80:centipede +81:black grouse +82:ptarmigan +83:ruffed grouse, partridge, Bonasa umbellus +84:prairie chicken, prairie grouse, prairie fowl +85:peacock +86:quail +87:partridge +88:African grey, African gray, Psittacus erithacus +89:macaw +90:sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +91:lorikeet +92:coucal +93:bee eater +94:hornbill +95:hummingbird +96:jacamar +97:toucan +98:drake +99:red-breasted merganser, Mergus serrator +100:goose +101:black swan, Cygnus atratus +102:tusker +103:echidna, spiny anteater, anteater +104:platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus +105:wallaby, brush kangaroo +106:koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +107:wombat +108:jellyfish +109:sea anemone, anemone +110:brain coral +111:flatworm, platyhelminth +112:nematode, nematode worm, roundworm +113:conch +114:snail +115:slug +116:sea slug, nudibranch +117:chiton, coat-of-mail shell, sea cradle, polyplacophore +118:chambered nautilus, pearly nautilus, nautilus +119:Dungeness crab, Cancer magister +120:rock crab, Cancer irroratus +121:fiddler crab +122:king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica +123:American lobster, Northern lobster, Maine lobster, Homarus americanus +124:spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +125:crayfish, crawfish, crawdad, crawdaddy +126:hermit crab +127:isopod +128:white stork, Ciconia ciconia +129:black stork, Ciconia nigra +130:spoonbill +131:flamingo +132:little blue heron, Egretta caerulea +133:American egret, great white heron, Egretta albus +134:bittern +135:crane +136:limpkin, Aramus pictus +137:European gallinule, Porphyrio porphyrio +138:American coot, marsh hen, mud hen, water hen, Fulica americana +139:bustard +140:ruddy turnstone, Arenaria interpres +141:red-backed sandpiper, dunlin, Erolia alpina +142:redshank, Tringa totanus +143:dowitcher +144:oystercatcher, oyster catcher +145:pelican +146:king penguin, Aptenodytes patagonica +147:albatross, mollymawk +148:grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus +149:killer whale, killer, orca, grampus, sea wolf, Orcinus orca +150:dugong, Dugong dugon +151:sea lion +152:Chihuahua +153:Japanese spaniel +154:Maltese dog, Maltese terrier, Maltese +155:Pekinese, Pekingese, Peke +156:Shih-Tzu +157:Blenheim spaniel +158:papillon +159:toy terrier +160:Rhodesian ridgeback +161:Afghan hound, Afghan +162:basset, basset hound +163:beagle +164:bloodhound, sleuthhound +165:bluetick +166:black-and-tan coonhound +167:Walker hound, Walker foxhound +168:English foxhound +169:redbone +170:borzoi, Russian wolfhound +171:Irish wolfhound +172:Italian greyhound +173:whippet +174:Ibizan hound, Ibizan Podenco +175:Norwegian elkhound, elkhound +176:otterhound, otter hound +177:Saluki, gazelle hound +178:Scottish deerhound, deerhound +179:Weimaraner +180:Staffordshire bullterrier, Staffordshire bull terrier +181:American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier +182:Bedlington terrier +183:Border terrier +184:Kerry blue terrier +185:Irish terrier +186:Norfolk terrier +187:Norwich terrier +188:Yorkshire terrier +189:wire-haired fox terrier +190:Lakeland terrier +191:Sealyham terrier, Sealyham +192:Airedale, Airedale terrier +193:cairn, cairn terrier +194:Australian terrier +195:Dandie Dinmont, Dandie Dinmont terrier +196:Boston bull, Boston terrier +197:miniature schnauzer +198:giant schnauzer +199:standard schnauzer +200:Scotch terrier, Scottish terrier, Scottie +201:Tibetan terrier, chrysanthemum dog +202:silky terrier, Sydney silky +203:soft-coated wheaten terrier +204:West Highland white terrier +205:Lhasa, Lhasa apso +206:flat-coated retriever +207:curly-coated retriever +208:golden retriever +209:Labrador retriever +210:Chesapeake Bay retriever +211:German short-haired pointer +212:vizsla, Hungarian pointer +213:English setter +214:Irish setter, red setter +215:Gordon setter +216:Brittany spaniel +217:clumber, clumber spaniel +218:English springer, English springer spaniel +219:Welsh springer spaniel +220:cocker spaniel, English cocker spaniel, cocker +221:Sussex spaniel +222:Irish water spaniel +223:kuvasz +224:schipperke +225:groenendael +226:malinois +227:briard +228:kelpie +229:komondor +230:Old English sheepdog, bobtail +231:Shetland sheepdog, Shetland sheep dog, Shetland +232:collie +233:Border collie +234:Bouvier des Flandres, Bouviers des Flandres +235:Rottweiler +236:German shepherd, German shepherd dog, German police dog, alsatian +237:Doberman, Doberman pinscher +238:miniature pinscher +239:Greater Swiss Mountain dog +240:Bernese mountain dog +241:Appenzeller +242:EntleBucher +243:boxer +244:bull mastiff +245:Tibetan mastiff +246:French bulldog +247:Great Dane +248:Saint Bernard, St Bernard +249:Eskimo dog, husky +250:malamute, malemute, Alaskan malamute +251:Siberian husky +252:dalmatian, coach dog, carriage dog +253:affenpinscher, monkey pinscher, monkey dog +254:basenji +255:pug, pug-dog +256:Leonberg +257:Newfoundland, Newfoundland dog +258:Great Pyrenees +259:Samoyed, Samoyede +260:Pomeranian +261:chow, chow chow +262:keeshond +263:Brabancon griffon +264:Pembroke, Pembroke Welsh corgi +265:Cardigan, Cardigan Welsh corgi +266:toy poodle +267:miniature poodle +268:standard poodle +269:Mexican hairless +270:timber wolf, grey wolf, gray wolf, Canis lupus +271:white wolf, Arctic wolf, Canis lupus tundrarum +272:red wolf, maned wolf, Canis rufus, Canis niger +273:coyote, prairie wolf, brush wolf, Canis latrans +274:dingo, warrigal, warragal, Canis dingo +275:dhole, Cuon alpinus +276:African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +277:hyena, hyaena +278:red fox, Vulpes vulpes +279:kit fox, Vulpes macrotis +280:Arctic fox, white fox, Alopex lagopus +281:grey fox, gray fox, Urocyon cinereoargenteus +282:tabby, tabby cat +283:tiger cat +284:Persian cat +285:Siamese cat, Siamese +286:Egyptian cat +287:cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +288:lynx, catamount +289:leopard, Panthera pardus +290:snow leopard, ounce, Panthera uncia +291:jaguar, panther, Panthera onca, Felis onca +292:lion, king of beasts, Panthera leo +293:tiger, Panthera tigris +294:cheetah, chetah, Acinonyx jubatus +295:brown bear, bruin, Ursus arctos +296:American black bear, black bear, Ursus americanus, Euarctos americanus +297:ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +298:sloth bear, Melursus ursinus, Ursus ursinus +299:mongoose +300:meerkat, mierkat +301:tiger beetle +302:ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +303:ground beetle, carabid beetle +304:long-horned beetle, longicorn, longicorn beetle +305:leaf beetle, chrysomelid +306:dung beetle +307:rhinoceros beetle +308:weevil +309:fly +310:bee +311:ant, emmet, pismire +312:grasshopper, hopper +313:cricket +314:walking stick, walkingstick, stick insect +315:cockroach, roach +316:mantis, mantid +317:cicada, cicala +318:leafhopper +319:lacewing, lacewing fly +320:dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk +321:damselfly +322:admiral +323:ringlet, ringlet butterfly +324:monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +325:cabbage butterfly +326:sulphur butterfly, sulfur butterfly +327:lycaenid, lycaenid butterfly +328:starfish, sea star +329:sea urchin +330:sea cucumber, holothurian +331:wood rabbit, cottontail, cottontail rabbit +332:hare +333:Angora, Angora rabbit +334:hamster +335:porcupine, hedgehog +336:fox squirrel, eastern fox squirrel, Sciurus niger +337:marmot +338:beaver +339:guinea pig, Cavia cobaya +340:sorrel +341:zebra +342:hog, pig, grunter, squealer, Sus scrofa +343:wild boar, boar, Sus scrofa +344:warthog +345:hippopotamus, hippo, river horse, Hippopotamus amphibius +346:ox +347:water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +348:bison +349:ram, tup +350:bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis +351:ibex, Capra ibex +352:hartebeest +353:impala, Aepyceros melampus +354:gazelle +355:Arabian camel, dromedary, Camelus dromedarius +356:llama +357:weasel +358:mink +359:polecat, fitch, foulmart, foumart, Mustela putorius +360:black-footed ferret, ferret, Mustela nigripes +361:otter +362:skunk, polecat, wood pussy +363:badger +364:armadillo +365:three-toed sloth, ai, Bradypus tridactylus +366:orangutan, orang, orangutang, Pongo pygmaeus +367:gorilla, Gorilla gorilla +368:chimpanzee, chimp, Pan troglodytes +369:gibbon, Hylobates lar +370:siamang, Hylobates syndactylus, Symphalangus syndactylus +371:guenon, guenon monkey +372:patas, hussar monkey, Erythrocebus patas +373:baboon +374:macaque +375:langur +376:colobus, colobus monkey +377:proboscis monkey, Nasalis larvatus +378:marmoset +379:capuchin, ringtail, Cebus capucinus +380:howler monkey, howler +381:titi, titi monkey +382:spider monkey, Ateles geoffroyi +383:squirrel monkey, Saimiri sciureus +384:Madagascar cat, ring-tailed lemur, Lemur catta +385:indri, indris, Indri indri, Indri brevicaudatus +386:Indian elephant, Elephas maximus +387:African elephant, Loxodonta africana +388:lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +389:giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +390:barracouta, snoek +391:eel +392:coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +393:rock beauty, Holocanthus tricolor +394:anemone fish +395:sturgeon +396:gar, garfish, garpike, billfish, Lepisosteus osseus +397:lionfish +398:puffer, pufferfish, blowfish, globefish +399:abacus +400:abaya +401:academic gown, academic robe, judge's robe +402:accordion, piano accordion, squeeze box +403:acoustic guitar +404:aircraft carrier, carrier, flattop, attack aircraft carrier +405:airliner +406:airship, dirigible +407:altar +408:ambulance +409:amphibian, amphibious vehicle +410:analog clock +411:apiary, bee house +412:apron +413:ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin +414:assault rifle, assault gun +415:backpack, back pack, knapsack, packsack, rucksack, haversack +416:bakery, bakeshop, bakehouse +417:balance beam, beam +418:balloon +419:ballpoint, ballpoint pen, ballpen, Biro +420:Band Aid +421:banjo +422:bannister, banister, balustrade, balusters, handrail +423:barbell +424:barber chair +425:barbershop +426:barn +427:barometer +428:barrel, cask +429:barrow, garden cart, lawn cart, wheelbarrow +430:baseball +431:basketball +432:bassinet +433:bassoon +434:bathing cap, swimming cap +435:bath towel +436:bathtub, bathing tub, bath, tub +437:beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon +438:beacon, lighthouse, beacon light, pharos +439:beaker +440:bearskin, busby, shako +441:beer bottle +442:beer glass +443:bell cote, bell cot +444:bib +445:bicycle-built-for-two, tandem bicycle, tandem +446:bikini, two-piece +447:binder, ring-binder +448:binoculars, field glasses, opera glasses +449:birdhouse +450:boathouse +451:bobsled, bobsleigh, bob +452:bolo tie, bolo, bola tie, bola +453:bonnet, poke bonnet +454:bookcase +455:bookshop, bookstore, bookstall +456:bottlecap +457:bow +458:bow tie, bow-tie, bowtie +459:brass, memorial tablet, plaque +460:brassiere, bra, bandeau +461:breakwater, groin, groyne, mole, bulwark, seawall, jetty +462:breastplate, aegis, egis +463:broom +464:bucket, pail +465:buckle +466:bulletproof vest +467:bullet train, bullet +468:butcher shop, meat market +469:cab, hack, taxi, taxicab +470:caldron, cauldron +471:candle, taper, wax light +472:cannon +473:canoe +474:can opener, tin opener +475:cardigan +476:car mirror +477:carousel, carrousel, merry-go-round, roundabout, whirligig +478:carpenter's kit, tool kit +479:carton +480:car wheel +481:cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM +482:cassette +483:cassette player +484:castle +485:catamaran +486:CD player +487:cello, violoncello +488:cellular telephone, cellular phone, cellphone, cell, mobile phone +489:chain +490:chainlink fence +491:chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour +492:chain saw, chainsaw +493:chest +494:chiffonier, commode +495:chime, bell, gong +496:china cabinet, china closet +497:Christmas stocking +498:church, church building +499:cinema, movie theater, movie theatre, movie house, picture palace +500:cleaver, meat cleaver, chopper +501:cliff dwelling +502:cloak +503:clog, geta, patten, sabot +504:cocktail shaker +505:coffee mug +506:coffeepot +507:coil, spiral, volute, whorl, helix +508:combination lock +509:computer keyboard, keypad +510:confectionery, confectionary, candy store +511:container ship, containership, container vessel +512:convertible +513:corkscrew, bottle screw +514:cornet, horn, trumpet, trump +515:cowboy boot +516:cowboy hat, ten-gallon hat +517:cradle +518:crane +519:crash helmet +520:crate +521:crib, cot +522:Crock Pot +523:croquet ball +524:crutch +525:cuirass +526:dam, dike, dyke +527:desk +528:desktop computer +529:dial telephone, dial phone +530:diaper, nappy, napkin +531:digital clock +532:digital watch +533:dining table, board +534:dishrag, dishcloth +535:dishwasher, dish washer, dishwashing machine +536:disk brake, disc brake +537:dock, dockage, docking facility +538:dogsled, dog sled, dog sleigh +539:dome +540:doormat, welcome mat +541:drilling platform, offshore rig +542:drum, membranophone, tympan +543:drumstick +544:dumbbell +545:Dutch oven +546:electric fan, blower +547:electric guitar +548:electric locomotive +549:entertainment center +550:envelope +551:espresso maker +552:face powder +553:feather boa, boa +554:file, file cabinet, filing cabinet +555:fireboat +556:fire engine, fire truck +557:fire screen, fireguard +558:flagpole, flagstaff +559:flute, transverse flute +560:folding chair +561:football helmet +562:forklift +563:fountain +564:fountain pen +565:four-poster +566:freight car +567:French horn, horn +568:frying pan, frypan, skillet +569:fur coat +570:garbage truck, dustcart +571:gasmask, respirator, gas helmet +572:gas pump, gasoline pump, petrol pump, island dispenser +573:goblet +574:go-kart +575:golf ball +576:golfcart, golf cart +577:gondola +578:gong, tam-tam +579:gown +580:grand piano, grand +581:greenhouse, nursery, glasshouse +582:grille, radiator grille +583:grocery store, grocery, food market, market +584:guillotine +585:hair slide +586:hair spray +587:half track +588:hammer +589:hamper +590:hand blower, blow dryer, blow drier, hair dryer, hair drier +591:hand-held computer, hand-held microcomputer +592:handkerchief, hankie, hanky, hankey +593:hard disc, hard disk, fixed disk +594:harmonica, mouth organ, harp, mouth harp +595:harp +596:harvester, reaper +597:hatchet +598:holster +599:home theater, home theatre +600:honeycomb +601:hook, claw +602:hoopskirt, crinoline +603:horizontal bar, high bar +604:horse cart, horse-cart +605:hourglass +606:iPod +607:iron, smoothing iron +608:jack-o'-lantern +609:jean, blue jean, denim +610:jeep, landrover +611:jersey, T-shirt, tee shirt +612:jigsaw puzzle +613:jinrikisha, ricksha, rickshaw +614:joystick +615:kimono +616:knee pad +617:knot +618:lab coat, laboratory coat +619:ladle +620:lampshade, lamp shade +621:laptop, laptop computer +622:lawn mower, mower +623:lens cap, lens cover +624:letter opener, paper knife, paperknife +625:library +626:lifeboat +627:lighter, light, igniter, ignitor +628:limousine, limo +629:liner, ocean liner +630:lipstick, lip rouge +631:Loafer +632:lotion +633:loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +634:loupe, jeweler's loupe +635:lumbermill, sawmill +636:magnetic compass +637:mailbag, postbag +638:mailbox, letter box +639:maillot +640:maillot, tank suit +641:manhole cover +642:maraca +643:marimba, xylophone +644:mask +645:matchstick +646:maypole +647:maze, labyrinth +648:measuring cup +649:medicine chest, medicine cabinet +650:megalith, megalithic structure +651:microphone, mike +652:microwave, microwave oven +653:military uniform +654:milk can +655:minibus +656:miniskirt, mini +657:minivan +658:missile +659:mitten +660:mixing bowl +661:mobile home, manufactured home +662:Model T +663:modem +664:monastery +665:monitor +666:moped +667:mortar +668:mortarboard +669:mosque +670:mosquito net +671:motor scooter, scooter +672:mountain bike, all-terrain bike, off-roader +673:mountain tent +674:mouse, computer mouse +675:mousetrap +676:moving van +677:muzzle +678:nail +679:neck brace +680:necklace +681:nipple +682:notebook, notebook computer +683:obelisk +684:oboe, hautboy, hautbois +685:ocarina, sweet potato +686:odometer, hodometer, mileometer, milometer +687:oil filter +688:organ, pipe organ +689:oscilloscope, scope, cathode-ray oscilloscope, CRO +690:overskirt +691:oxcart +692:oxygen mask +693:packet +694:paddle, boat paddle +695:paddlewheel, paddle wheel +696:padlock +697:paintbrush +698:pajama, pyjama, pj's, jammies +699:palace +700:panpipe, pandean pipe, syrinx +701:paper towel +702:parachute, chute +703:parallel bars, bars +704:park bench +705:parking meter +706:passenger car, coach, carriage +707:patio, terrace +708:pay-phone, pay-station +709:pedestal, plinth, footstall +710:pencil box, pencil case +711:pencil sharpener +712:perfume, essence +713:Petri dish +714:photocopier +715:pick, plectrum, plectron +716:pickelhaube +717:picket fence, paling +718:pickup, pickup truck +719:pier +720:piggy bank, penny bank +721:pill bottle +722:pillow +723:ping-pong ball +724:pinwheel +725:pirate, pirate ship +726:pitcher, ewer +727:plane, carpenter's plane, woodworking plane +728:planetarium +729:plastic bag +730:plate rack +731:plow, plough +732:plunger, plumber's helper +733:Polaroid camera, Polaroid Land camera +734:pole +735:police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +736:poncho +737:pool table, billiard table, snooker table +738:pop bottle, soda bottle +739:pot, flowerpot +740:potter's wheel +741:power drill +742:prayer rug, prayer mat +743:printer +744:prison, prison house +745:projectile, missile +746:projector +747:puck, hockey puck +748:punching bag, punch bag, punching ball, punchball +749:purse +750:quill, quill pen +751:quilt, comforter, comfort, puff +752:racer, race car, racing car +753:racket, racquet +754:radiator +755:radio, wireless +756:radio telescope, radio reflector +757:rain barrel +758:recreational vehicle, RV, R.V. +759:reel +760:reflex camera +761:refrigerator, icebox +762:remote control, remote +763:restaurant, eating house, eating place, eatery +764:revolver, six-gun, six-shooter +765:rifle +766:rocking chair, rocker +767:rotisserie +768:rubber eraser, rubber, pencil eraser +769:rugby ball +770:rule, ruler +771:running shoe +772:safe +773:safety pin +774:saltshaker, salt shaker +775:sandal +776:sarong +777:sax, saxophone +778:scabbard +779:scale, weighing machine +780:school bus +781:schooner +782:scoreboard +783:screen, CRT screen +784:screw +785:screwdriver +786:seat belt, seatbelt +787:sewing machine +788:shield, buckler +789:shoe shop, shoe-shop, shoe store +790:shoji +791:shopping basket +792:shopping cart +793:shovel +794:shower cap +795:shower curtain +796:ski +797:ski mask +798:sleeping bag +799:slide rule, slipstick +800:sliding door +801:slot, one-armed bandit +802:snorkel +803:snowmobile +804:snowplow, snowplough +805:soap dispenser +806:soccer ball +807:sock +808:solar dish, solar collector, solar furnace +809:sombrero +810:soup bowl +811:space bar +812:space heater +813:space shuttle +814:spatula +815:speedboat +816:spider web, spider's web +817:spindle +818:sports car, sport car +819:spotlight, spot +820:stage +821:steam locomotive +822:steel arch bridge +823:steel drum +824:stethoscope +825:stole +826:stone wall +827:stopwatch, stop watch +828:stove +829:strainer +830:streetcar, tram, tramcar, trolley, trolley car +831:stretcher +832:studio couch, day bed +833:stupa, tope +834:submarine, pigboat, sub, U-boat +835:suit, suit of clothes +836:sundial +837:sunglass +838:sunglasses, dark glasses, shades +839:sunscreen, sunblock, sun blocker +840:suspension bridge +841:swab, swob, mop +842:sweatshirt +843:swimming trunks, bathing trunks +844:swing +845:switch, electric switch, electrical switch +846:syringe +847:table lamp +848:tank, army tank, armored combat vehicle, armoured combat vehicle +849:tape player +850:teapot +851:teddy, teddy bear +852:television, television system +853:tennis ball +854:thatch, thatched roof +855:theater curtain, theatre curtain +856:thimble +857:thresher, thrasher, threshing machine +858:throne +859:tile roof +860:toaster +861:tobacco shop, tobacconist shop, tobacconist +862:toilet seat +863:torch +864:totem pole +865:tow truck, tow car, wrecker +866:toyshop +867:tractor +868:trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +869:tray +870:trench coat +871:tricycle, trike, velocipede +872:trimaran +873:tripod +874:triumphal arch +875:trolleybus, trolley coach, trackless trolley +876:trombone +877:tub, vat +878:turnstile +879:typewriter keyboard +880:umbrella +881:unicycle, monocycle +882:upright, upright piano +883:vacuum, vacuum cleaner +884:vase +885:vault +886:velvet +887:vending machine +888:vestment +889:viaduct +890:violin, fiddle +891:volleyball +892:waffle iron +893:wall clock +894:wallet, billfold, notecase, pocketbook +895:wardrobe, closet, press +896:warplane, military plane +897:washbasin, handbasin, washbowl, lavabo, wash-hand basin +898:washer, automatic washer, washing machine +899:water bottle +900:water jug +901:water tower +902:whiskey jug +903:whistle +904:wig +905:window screen +906:window shade +907:Windsor tie +908:wine bottle +909:wing +910:wok +911:wooden spoon +912:wool, woolen, woollen +913:worm fence, snake fence, snake-rail fence, Virginia fence +914:wreck +915:yawl +916:yurt +917:web site, website, internet site, site +918:comic book +919:crossword puzzle, crossword +920:street sign +921:traffic light, traffic signal, stoplight +922:book jacket, dust cover, dust jacket, dust wrapper +923:menu +924:plate +925:guacamole +926:consomme +927:hot pot, hotpot +928:trifle +929:ice cream, icecream +930:ice lolly, lolly, lollipop, popsicle +931:French loaf +932:bagel, beigel +933:pretzel +934:cheeseburger +935:hotdog, hot dog, red hot +936:mashed potato +937:head cabbage +938:broccoli +939:cauliflower +940:zucchini, courgette +941:spaghetti squash +942:acorn squash +943:butternut squash +944:cucumber, cuke +945:artichoke, globe artichoke +946:bell pepper +947:cardoon +948:mushroom +949:Granny Smith +950:strawberry +951:orange +952:lemon +953:fig +954:pineapple, ananas +955:banana +956:jackfruit, jak, jack +957:custard apple +958:pomegranate +959:hay +960:carbonara +961:chocolate sauce, chocolate syrup +962:dough +963:meat loaf, meatloaf +964:pizza, pizza pie +965:potpie +966:burrito +967:red wine +968:espresso +969:cup +970:eggnog +971:alp +972:bubble +973:cliff, drop, drop-off +974:coral reef +975:geyser +976:lakeside, lakeshore +977:promontory, headland, head, foreland +978:sandbar, sand bar +979:seashore, coast, seacoast, sea-coast +980:valley, vale +981:volcano +982:ballplayer, baseball player +983:groom, bridegroom +984:scuba diver +985:rapeseed +986:daisy +987:yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum +988:corn +989:acorn +990:hip, rose hip, rosehip +991:buckeye, horse chestnut, conker +992:coral fungus +993:agaric +994:gyromitra +995:stinkhorn, carrion fungus +996:earthstar +997:hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +998:bolete +999:ear, spike, capitulum +1000:toilet tissue, toilet paper, bathroom tissue diff --git a/Robustness_Eval/CLEVER/labels/synset_words.txt b/Robustness_Eval/CLEVER/labels/synset_words.txt new file mode 100644 index 0000000..a9e8c7f --- /dev/null +++ b/Robustness_Eval/CLEVER/labels/synset_words.txt @@ -0,0 +1,1000 @@ +n01440764 tench, Tinca tinca +n01443537 goldfish, Carassius auratus +n01484850 great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias +n01491361 tiger shark, Galeocerdo cuvieri +n01494475 hammerhead, hammerhead shark +n01496331 electric ray, crampfish, numbfish, torpedo +n01498041 stingray +n01514668 cock +n01514859 hen +n01518878 ostrich, Struthio camelus +n01530575 brambling, Fringilla montifringilla +n01531178 goldfinch, Carduelis carduelis +n01532829 house finch, linnet, Carpodacus mexicanus +n01534433 junco, snowbird +n01537544 indigo bunting, indigo finch, indigo bird, Passerina cyanea +n01558993 robin, American robin, Turdus migratorius +n01560419 bulbul +n01580077 jay +n01582220 magpie +n01592084 chickadee +n01601694 water ouzel, dipper +n01608432 kite +n01614925 bald eagle, American eagle, Haliaeetus leucocephalus +n01616318 vulture +n01622779 great grey owl, great gray owl, Strix nebulosa +n01629819 European fire salamander, Salamandra salamandra +n01630670 common newt, Triturus vulgaris +n01631663 eft +n01632458 spotted salamander, Ambystoma maculatum +n01632777 axolotl, mud puppy, Ambystoma mexicanum +n01641577 bullfrog, Rana catesbeiana +n01644373 tree frog, tree-frog +n01644900 tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +n01664065 loggerhead, loggerhead turtle, Caretta caretta +n01665541 leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +n01667114 mud turtle +n01667778 terrapin +n01669191 box turtle, box tortoise +n01675722 banded gecko +n01677366 common iguana, iguana, Iguana iguana +n01682714 American chameleon, anole, Anolis carolinensis +n01685808 whiptail, whiptail lizard +n01687978 agama +n01688243 frilled lizard, Chlamydosaurus kingi +n01689811 alligator lizard +n01692333 Gila monster, Heloderma suspectum +n01693334 green lizard, Lacerta viridis +n01694178 African chameleon, Chamaeleo chamaeleon +n01695060 Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +n01697457 African crocodile, Nile crocodile, Crocodylus niloticus +n01698640 American alligator, Alligator mississipiensis +n01704323 triceratops +n01728572 thunder snake, worm snake, Carphophis amoenus +n01728920 ringneck snake, ring-necked snake, ring snake +n01729322 hognose snake, puff adder, sand viper +n01729977 green snake, grass snake +n01734418 king snake, kingsnake +n01735189 garter snake, grass snake +n01737021 water snake +n01739381 vine snake +n01740131 night snake, Hypsiglena torquata +n01742172 boa constrictor, Constrictor constrictor +n01744401 rock python, rock snake, Python sebae +n01748264 Indian cobra, Naja naja +n01749939 green mamba +n01751748 sea snake +n01753488 horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +n01755581 diamondback, diamondback rattlesnake, Crotalus adamanteus +n01756291 sidewinder, horned rattlesnake, Crotalus cerastes +n01768244 trilobite +n01770081 harvestman, daddy longlegs, Phalangium opilio +n01770393 scorpion +n01773157 black and gold garden spider, Argiope aurantia +n01773549 barn spider, Araneus cavaticus +n01773797 garden spider, Aranea diademata +n01774384 black widow, Latrodectus mactans +n01774750 tarantula +n01775062 wolf spider, hunting spider +n01776313 tick +n01784675 centipede +n01795545 black grouse +n01796340 ptarmigan +n01797886 ruffed grouse, partridge, Bonasa umbellus +n01798484 prairie chicken, prairie grouse, prairie fowl +n01806143 peacock +n01806567 quail +n01807496 partridge +n01817953 African grey, African gray, Psittacus erithacus +n01818515 macaw +n01819313 sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +n01820546 lorikeet +n01824575 coucal +n01828970 bee eater +n01829413 hornbill +n01833805 hummingbird +n01843065 jacamar +n01843383 toucan +n01847000 drake +n01855032 red-breasted merganser, Mergus serrator +n01855672 goose +n01860187 black swan, Cygnus atratus +n01871265 tusker +n01872401 echidna, spiny anteater, anteater +n01873310 platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus +n01877812 wallaby, brush kangaroo +n01882714 koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +n01883070 wombat +n01910747 jellyfish +n01914609 sea anemone, anemone +n01917289 brain coral +n01924916 flatworm, platyhelminth +n01930112 nematode, nematode worm, roundworm +n01943899 conch +n01944390 snail +n01945685 slug +n01950731 sea slug, nudibranch +n01955084 chiton, coat-of-mail shell, sea cradle, polyplacophore +n01968897 chambered nautilus, pearly nautilus, nautilus +n01978287 Dungeness crab, Cancer magister +n01978455 rock crab, Cancer irroratus +n01980166 fiddler crab +n01981276 king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica +n01983481 American lobster, Northern lobster, Maine lobster, Homarus americanus +n01984695 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +n01985128 crayfish, crawfish, crawdad, crawdaddy +n01986214 hermit crab +n01990800 isopod +n02002556 white stork, Ciconia ciconia +n02002724 black stork, Ciconia nigra +n02006656 spoonbill +n02007558 flamingo +n02009229 little blue heron, Egretta caerulea +n02009912 American egret, great white heron, Egretta albus +n02011460 bittern +n02012849 crane +n02013706 limpkin, Aramus pictus +n02017213 European gallinule, Porphyrio porphyrio +n02018207 American coot, marsh hen, mud hen, water hen, Fulica americana +n02018795 bustard +n02025239 ruddy turnstone, Arenaria interpres +n02027492 red-backed sandpiper, dunlin, Erolia alpina +n02028035 redshank, Tringa totanus +n02033041 dowitcher +n02037110 oystercatcher, oyster catcher +n02051845 pelican +n02056570 king penguin, Aptenodytes patagonica +n02058221 albatross, mollymawk +n02066245 grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus +n02071294 killer whale, killer, orca, grampus, sea wolf, Orcinus orca +n02074367 dugong, Dugong dugon +n02077923 sea lion +n02085620 Chihuahua +n02085782 Japanese spaniel +n02085936 Maltese dog, Maltese terrier, Maltese +n02086079 Pekinese, Pekingese, Peke +n02086240 Shih-Tzu +n02086646 Blenheim spaniel +n02086910 papillon +n02087046 toy terrier +n02087394 Rhodesian ridgeback +n02088094 Afghan hound, Afghan +n02088238 basset, basset hound +n02088364 beagle +n02088466 bloodhound, sleuthhound +n02088632 bluetick +n02089078 black-and-tan coonhound +n02089867 Walker hound, Walker foxhound +n02089973 English foxhound +n02090379 redbone +n02090622 borzoi, Russian wolfhound +n02090721 Irish wolfhound +n02091032 Italian greyhound +n02091134 whippet +n02091244 Ibizan hound, Ibizan Podenco +n02091467 Norwegian elkhound, elkhound +n02091635 otterhound, otter hound +n02091831 Saluki, gazelle hound +n02092002 Scottish deerhound, deerhound +n02092339 Weimaraner +n02093256 Staffordshire bullterrier, Staffordshire bull terrier +n02093428 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier +n02093647 Bedlington terrier +n02093754 Border terrier +n02093859 Kerry blue terrier +n02093991 Irish terrier +n02094114 Norfolk terrier +n02094258 Norwich terrier +n02094433 Yorkshire terrier +n02095314 wire-haired fox terrier +n02095570 Lakeland terrier +n02095889 Sealyham terrier, Sealyham +n02096051 Airedale, Airedale terrier +n02096177 cairn, cairn terrier +n02096294 Australian terrier +n02096437 Dandie Dinmont, Dandie Dinmont terrier +n02096585 Boston bull, Boston terrier +n02097047 miniature schnauzer +n02097130 giant schnauzer +n02097209 standard schnauzer +n02097298 Scotch terrier, Scottish terrier, Scottie +n02097474 Tibetan terrier, chrysanthemum dog +n02097658 silky terrier, Sydney silky +n02098105 soft-coated wheaten terrier +n02098286 West Highland white terrier +n02098413 Lhasa, Lhasa apso +n02099267 flat-coated retriever +n02099429 curly-coated retriever +n02099601 golden retriever +n02099712 Labrador retriever +n02099849 Chesapeake Bay retriever +n02100236 German short-haired pointer +n02100583 vizsla, Hungarian pointer +n02100735 English setter +n02100877 Irish setter, red setter +n02101006 Gordon setter +n02101388 Brittany spaniel +n02101556 clumber, clumber spaniel +n02102040 English springer, English springer spaniel +n02102177 Welsh springer spaniel +n02102318 cocker spaniel, English cocker spaniel, cocker +n02102480 Sussex spaniel +n02102973 Irish water spaniel +n02104029 kuvasz +n02104365 schipperke +n02105056 groenendael +n02105162 malinois +n02105251 briard +n02105412 kelpie +n02105505 komondor +n02105641 Old English sheepdog, bobtail +n02105855 Shetland sheepdog, Shetland sheep dog, Shetland +n02106030 collie +n02106166 Border collie +n02106382 Bouvier des Flandres, Bouviers des Flandres +n02106550 Rottweiler +n02106662 German shepherd, German shepherd dog, German police dog, alsatian +n02107142 Doberman, Doberman pinscher +n02107312 miniature pinscher +n02107574 Greater Swiss Mountain dog +n02107683 Bernese mountain dog +n02107908 Appenzeller +n02108000 EntleBucher +n02108089 boxer +n02108422 bull mastiff +n02108551 Tibetan mastiff +n02108915 French bulldog +n02109047 Great Dane +n02109525 Saint Bernard, St Bernard +n02109961 Eskimo dog, husky +n02110063 malamute, malemute, Alaskan malamute +n02110185 Siberian husky +n02110341 dalmatian, coach dog, carriage dog +n02110627 affenpinscher, monkey pinscher, monkey dog +n02110806 basenji +n02110958 pug, pug-dog +n02111129 Leonberg +n02111277 Newfoundland, Newfoundland dog +n02111500 Great Pyrenees +n02111889 Samoyed, Samoyede +n02112018 Pomeranian +n02112137 chow, chow chow +n02112350 keeshond +n02112706 Brabancon griffon +n02113023 Pembroke, Pembroke Welsh corgi +n02113186 Cardigan, Cardigan Welsh corgi +n02113624 toy poodle +n02113712 miniature poodle +n02113799 standard poodle +n02113978 Mexican hairless +n02114367 timber wolf, grey wolf, gray wolf, Canis lupus +n02114548 white wolf, Arctic wolf, Canis lupus tundrarum +n02114712 red wolf, maned wolf, Canis rufus, Canis niger +n02114855 coyote, prairie wolf, brush wolf, Canis latrans +n02115641 dingo, warrigal, warragal, Canis dingo +n02115913 dhole, Cuon alpinus +n02116738 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +n02117135 hyena, hyaena +n02119022 red fox, Vulpes vulpes +n02119789 kit fox, Vulpes macrotis +n02120079 Arctic fox, white fox, Alopex lagopus +n02120505 grey fox, gray fox, Urocyon cinereoargenteus +n02123045 tabby, tabby cat +n02123159 tiger cat +n02123394 Persian cat +n02123597 Siamese cat, Siamese +n02124075 Egyptian cat +n02125311 cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +n02127052 lynx, catamount +n02128385 leopard, Panthera pardus +n02128757 snow leopard, ounce, Panthera uncia +n02128925 jaguar, panther, Panthera onca, Felis onca +n02129165 lion, king of beasts, Panthera leo +n02129604 tiger, Panthera tigris +n02130308 cheetah, chetah, Acinonyx jubatus +n02132136 brown bear, bruin, Ursus arctos +n02133161 American black bear, black bear, Ursus americanus, Euarctos americanus +n02134084 ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +n02134418 sloth bear, Melursus ursinus, Ursus ursinus +n02137549 mongoose +n02138441 meerkat, mierkat +n02165105 tiger beetle +n02165456 ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +n02167151 ground beetle, carabid beetle +n02168699 long-horned beetle, longicorn, longicorn beetle +n02169497 leaf beetle, chrysomelid +n02172182 dung beetle +n02174001 rhinoceros beetle +n02177972 weevil +n02190166 fly +n02206856 bee +n02219486 ant, emmet, pismire +n02226429 grasshopper, hopper +n02229544 cricket +n02231487 walking stick, walkingstick, stick insect +n02233338 cockroach, roach +n02236044 mantis, mantid +n02256656 cicada, cicala +n02259212 leafhopper +n02264363 lacewing, lacewing fly +n02268443 dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk +n02268853 damselfly +n02276258 admiral +n02277742 ringlet, ringlet butterfly +n02279972 monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +n02280649 cabbage butterfly +n02281406 sulphur butterfly, sulfur butterfly +n02281787 lycaenid, lycaenid butterfly +n02317335 starfish, sea star +n02319095 sea urchin +n02321529 sea cucumber, holothurian +n02325366 wood rabbit, cottontail, cottontail rabbit +n02326432 hare +n02328150 Angora, Angora rabbit +n02342885 hamster +n02346627 porcupine, hedgehog +n02356798 fox squirrel, eastern fox squirrel, Sciurus niger +n02361337 marmot +n02363005 beaver +n02364673 guinea pig, Cavia cobaya +n02389026 sorrel +n02391049 zebra +n02395406 hog, pig, grunter, squealer, Sus scrofa +n02396427 wild boar, boar, Sus scrofa +n02397096 warthog +n02398521 hippopotamus, hippo, river horse, Hippopotamus amphibius +n02403003 ox +n02408429 water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +n02410509 bison +n02412080 ram, tup +n02415577 bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis +n02417914 ibex, Capra ibex +n02422106 hartebeest +n02422699 impala, Aepyceros melampus +n02423022 gazelle +n02437312 Arabian camel, dromedary, Camelus dromedarius +n02437616 llama +n02441942 weasel +n02442845 mink +n02443114 polecat, fitch, foulmart, foumart, Mustela putorius +n02443484 black-footed ferret, ferret, Mustela nigripes +n02444819 otter +n02445715 skunk, polecat, wood pussy +n02447366 badger +n02454379 armadillo +n02457408 three-toed sloth, ai, Bradypus tridactylus +n02480495 orangutan, orang, orangutang, Pongo pygmaeus +n02480855 gorilla, Gorilla gorilla +n02481823 chimpanzee, chimp, Pan troglodytes +n02483362 gibbon, Hylobates lar +n02483708 siamang, Hylobates syndactylus, Symphalangus syndactylus +n02484975 guenon, guenon monkey +n02486261 patas, hussar monkey, Erythrocebus patas +n02486410 baboon +n02487347 macaque +n02488291 langur +n02488702 colobus, colobus monkey +n02489166 proboscis monkey, Nasalis larvatus +n02490219 marmoset +n02492035 capuchin, ringtail, Cebus capucinus +n02492660 howler monkey, howler +n02493509 titi, titi monkey +n02493793 spider monkey, Ateles geoffroyi +n02494079 squirrel monkey, Saimiri sciureus +n02497673 Madagascar cat, ring-tailed lemur, Lemur catta +n02500267 indri, indris, Indri indri, Indri brevicaudatus +n02504013 Indian elephant, Elephas maximus +n02504458 African elephant, Loxodonta africana +n02509815 lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +n02510455 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +n02514041 barracouta, snoek +n02526121 eel +n02536864 coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +n02606052 rock beauty, Holocanthus tricolor +n02607072 anemone fish +n02640242 sturgeon +n02641379 gar, garfish, garpike, billfish, Lepisosteus osseus +n02643566 lionfish +n02655020 puffer, pufferfish, blowfish, globefish +n02666196 abacus +n02667093 abaya +n02669723 academic gown, academic robe, judge's robe +n02672831 accordion, piano accordion, squeeze box +n02676566 acoustic guitar +n02687172 aircraft carrier, carrier, flattop, attack aircraft carrier +n02690373 airliner +n02692877 airship, dirigible +n02699494 altar +n02701002 ambulance +n02704792 amphibian, amphibious vehicle +n02708093 analog clock +n02727426 apiary, bee house +n02730930 apron +n02747177 ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin +n02749479 assault rifle, assault gun +n02769748 backpack, back pack, knapsack, packsack, rucksack, haversack +n02776631 bakery, bakeshop, bakehouse +n02777292 balance beam, beam +n02782093 balloon +n02783161 ballpoint, ballpoint pen, ballpen, Biro +n02786058 Band Aid +n02787622 banjo +n02788148 bannister, banister, balustrade, balusters, handrail +n02790996 barbell +n02791124 barber chair +n02791270 barbershop +n02793495 barn +n02794156 barometer +n02795169 barrel, cask +n02797295 barrow, garden cart, lawn cart, wheelbarrow +n02799071 baseball +n02802426 basketball +n02804414 bassinet +n02804610 bassoon +n02807133 bathing cap, swimming cap +n02808304 bath towel +n02808440 bathtub, bathing tub, bath, tub +n02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon +n02814860 beacon, lighthouse, beacon light, pharos +n02815834 beaker +n02817516 bearskin, busby, shako +n02823428 beer bottle +n02823750 beer glass +n02825657 bell cote, bell cot +n02834397 bib +n02835271 bicycle-built-for-two, tandem bicycle, tandem +n02837789 bikini, two-piece +n02840245 binder, ring-binder +n02841315 binoculars, field glasses, opera glasses +n02843684 birdhouse +n02859443 boathouse +n02860847 bobsled, bobsleigh, bob +n02865351 bolo tie, bolo, bola tie, bola +n02869837 bonnet, poke bonnet +n02870880 bookcase +n02871525 bookshop, bookstore, bookstall +n02877765 bottlecap +n02879718 bow +n02883205 bow tie, bow-tie, bowtie +n02892201 brass, memorial tablet, plaque +n02892767 brassiere, bra, bandeau +n02894605 breakwater, groin, groyne, mole, bulwark, seawall, jetty +n02895154 breastplate, aegis, egis +n02906734 broom +n02909870 bucket, pail +n02910353 buckle +n02916936 bulletproof vest +n02917067 bullet train, bullet +n02927161 butcher shop, meat market +n02930766 cab, hack, taxi, taxicab +n02939185 caldron, cauldron +n02948072 candle, taper, wax light +n02950826 cannon +n02951358 canoe +n02951585 can opener, tin opener +n02963159 cardigan +n02965783 car mirror +n02966193 carousel, carrousel, merry-go-round, roundabout, whirligig +n02966687 carpenter's kit, tool kit +n02971356 carton +n02974003 car wheel +n02977058 cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM +n02978881 cassette +n02979186 cassette player +n02980441 castle +n02981792 catamaran +n02988304 CD player +n02992211 cello, violoncello +n02992529 cellular telephone, cellular phone, cellphone, cell, mobile phone +n02999410 chain +n03000134 chainlink fence +n03000247 chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour +n03000684 chain saw, chainsaw +n03014705 chest +n03016953 chiffonier, commode +n03017168 chime, bell, gong +n03018349 china cabinet, china closet +n03026506 Christmas stocking +n03028079 church, church building +n03032252 cinema, movie theater, movie theatre, movie house, picture palace +n03041632 cleaver, meat cleaver, chopper +n03042490 cliff dwelling +n03045698 cloak +n03047690 clog, geta, patten, sabot +n03062245 cocktail shaker +n03063599 coffee mug +n03063689 coffeepot +n03065424 coil, spiral, volute, whorl, helix +n03075370 combination lock +n03085013 computer keyboard, keypad +n03089624 confectionery, confectionary, candy store +n03095699 container ship, containership, container vessel +n03100240 convertible +n03109150 corkscrew, bottle screw +n03110669 cornet, horn, trumpet, trump +n03124043 cowboy boot +n03124170 cowboy hat, ten-gallon hat +n03125729 cradle +n03126707 crane +n03127747 crash helmet +n03127925 crate +n03131574 crib, cot +n03133878 Crock Pot +n03134739 croquet ball +n03141823 crutch +n03146219 cuirass +n03160309 dam, dike, dyke +n03179701 desk +n03180011 desktop computer +n03187595 dial telephone, dial phone +n03188531 diaper, nappy, napkin +n03196217 digital clock +n03197337 digital watch +n03201208 dining table, board +n03207743 dishrag, dishcloth +n03207941 dishwasher, dish washer, dishwashing machine +n03208938 disk brake, disc brake +n03216828 dock, dockage, docking facility +n03218198 dogsled, dog sled, dog sleigh +n03220513 dome +n03223299 doormat, welcome mat +n03240683 drilling platform, offshore rig +n03249569 drum, membranophone, tympan +n03250847 drumstick +n03255030 dumbbell +n03259280 Dutch oven +n03271574 electric fan, blower +n03272010 electric guitar +n03272562 electric locomotive +n03290653 entertainment center +n03291819 envelope +n03297495 espresso maker +n03314780 face powder +n03325584 feather boa, boa +n03337140 file, file cabinet, filing cabinet +n03344393 fireboat +n03345487 fire engine, fire truck +n03347037 fire screen, fireguard +n03355925 flagpole, flagstaff +n03372029 flute, transverse flute +n03376595 folding chair +n03379051 football helmet +n03384352 forklift +n03388043 fountain +n03388183 fountain pen +n03388549 four-poster +n03393912 freight car +n03394916 French horn, horn +n03400231 frying pan, frypan, skillet +n03404251 fur coat +n03417042 garbage truck, dustcart +n03424325 gasmask, respirator, gas helmet +n03425413 gas pump, gasoline pump, petrol pump, island dispenser +n03443371 goblet +n03444034 go-kart +n03445777 golf ball +n03445924 golfcart, golf cart +n03447447 gondola +n03447721 gong, tam-tam +n03450230 gown +n03452741 grand piano, grand +n03457902 greenhouse, nursery, glasshouse +n03459775 grille, radiator grille +n03461385 grocery store, grocery, food market, market +n03467068 guillotine +n03476684 hair slide +n03476991 hair spray +n03478589 half track +n03481172 hammer +n03482405 hamper +n03483316 hand blower, blow dryer, blow drier, hair dryer, hair drier +n03485407 hand-held computer, hand-held microcomputer +n03485794 handkerchief, hankie, hanky, hankey +n03492542 hard disc, hard disk, fixed disk +n03494278 harmonica, mouth organ, harp, mouth harp +n03495258 harp +n03496892 harvester, reaper +n03498962 hatchet +n03527444 holster +n03529860 home theater, home theatre +n03530642 honeycomb +n03532672 hook, claw +n03534580 hoopskirt, crinoline +n03535780 horizontal bar, high bar +n03538406 horse cart, horse-cart +n03544143 hourglass +n03584254 iPod +n03584829 iron, smoothing iron +n03590841 jack-o'-lantern +n03594734 jean, blue jean, denim +n03594945 jeep, landrover +n03595614 jersey, T-shirt, tee shirt +n03598930 jigsaw puzzle +n03599486 jinrikisha, ricksha, rickshaw +n03602883 joystick +n03617480 kimono +n03623198 knee pad +n03627232 knot +n03630383 lab coat, laboratory coat +n03633091 ladle +n03637318 lampshade, lamp shade +n03642806 laptop, laptop computer +n03649909 lawn mower, mower +n03657121 lens cap, lens cover +n03658185 letter opener, paper knife, paperknife +n03661043 library +n03662601 lifeboat +n03666591 lighter, light, igniter, ignitor +n03670208 limousine, limo +n03673027 liner, ocean liner +n03676483 lipstick, lip rouge +n03680355 Loafer +n03690938 lotion +n03691459 loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +n03692522 loupe, jeweler's loupe +n03697007 lumbermill, sawmill +n03706229 magnetic compass +n03709823 mailbag, postbag +n03710193 mailbox, letter box +n03710637 maillot +n03710721 maillot, tank suit +n03717622 manhole cover +n03720891 maraca +n03721384 marimba, xylophone +n03724870 mask +n03729826 matchstick +n03733131 maypole +n03733281 maze, labyrinth +n03733805 measuring cup +n03742115 medicine chest, medicine cabinet +n03743016 megalith, megalithic structure +n03759954 microphone, mike +n03761084 microwave, microwave oven +n03763968 military uniform +n03764736 milk can +n03769881 minibus +n03770439 miniskirt, mini +n03770679 minivan +n03773504 missile +n03775071 mitten +n03775546 mixing bowl +n03776460 mobile home, manufactured home +n03777568 Model T +n03777754 modem +n03781244 monastery +n03782006 monitor +n03785016 moped +n03786901 mortar +n03787032 mortarboard +n03788195 mosque +n03788365 mosquito net +n03791053 motor scooter, scooter +n03792782 mountain bike, all-terrain bike, off-roader +n03792972 mountain tent +n03793489 mouse, computer mouse +n03794056 mousetrap +n03796401 moving van +n03803284 muzzle +n03804744 nail +n03814639 neck brace +n03814906 necklace +n03825788 nipple +n03832673 notebook, notebook computer +n03837869 obelisk +n03838899 oboe, hautboy, hautbois +n03840681 ocarina, sweet potato +n03841143 odometer, hodometer, mileometer, milometer +n03843555 oil filter +n03854065 organ, pipe organ +n03857828 oscilloscope, scope, cathode-ray oscilloscope, CRO +n03866082 overskirt +n03868242 oxcart +n03868863 oxygen mask +n03871628 packet +n03873416 paddle, boat paddle +n03874293 paddlewheel, paddle wheel +n03874599 padlock +n03876231 paintbrush +n03877472 pajama, pyjama, pj's, jammies +n03877845 palace +n03884397 panpipe, pandean pipe, syrinx +n03887697 paper towel +n03888257 parachute, chute +n03888605 parallel bars, bars +n03891251 park bench +n03891332 parking meter +n03895866 passenger car, coach, carriage +n03899768 patio, terrace +n03902125 pay-phone, pay-station +n03903868 pedestal, plinth, footstall +n03908618 pencil box, pencil case +n03908714 pencil sharpener +n03916031 perfume, essence +n03920288 Petri dish +n03924679 photocopier +n03929660 pick, plectrum, plectron +n03929855 pickelhaube +n03930313 picket fence, paling +n03930630 pickup, pickup truck +n03933933 pier +n03935335 piggy bank, penny bank +n03937543 pill bottle +n03938244 pillow +n03942813 ping-pong ball +n03944341 pinwheel +n03947888 pirate, pirate ship +n03950228 pitcher, ewer +n03954731 plane, carpenter's plane, woodworking plane +n03956157 planetarium +n03958227 plastic bag +n03961711 plate rack +n03967562 plow, plough +n03970156 plunger, plumber's helper +n03976467 Polaroid camera, Polaroid Land camera +n03976657 pole +n03977966 police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +n03980874 poncho +n03982430 pool table, billiard table, snooker table +n03983396 pop bottle, soda bottle +n03991062 pot, flowerpot +n03992509 potter's wheel +n03995372 power drill +n03998194 prayer rug, prayer mat +n04004767 printer +n04005630 prison, prison house +n04008634 projectile, missile +n04009552 projector +n04019541 puck, hockey puck +n04023962 punching bag, punch bag, punching ball, punchball +n04026417 purse +n04033901 quill, quill pen +n04033995 quilt, comforter, comfort, puff +n04037443 racer, race car, racing car +n04039381 racket, racquet +n04040759 radiator +n04041544 radio, wireless +n04044716 radio telescope, radio reflector +n04049303 rain barrel +n04065272 recreational vehicle, RV, R.V. +n04067472 reel +n04069434 reflex camera +n04070727 refrigerator, icebox +n04074963 remote control, remote +n04081281 restaurant, eating house, eating place, eatery +n04086273 revolver, six-gun, six-shooter +n04090263 rifle +n04099969 rocking chair, rocker +n04111531 rotisserie +n04116512 rubber eraser, rubber, pencil eraser +n04118538 rugby ball +n04118776 rule, ruler +n04120489 running shoe +n04125021 safe +n04127249 safety pin +n04131690 saltshaker, salt shaker +n04133789 sandal +n04136333 sarong +n04141076 sax, saxophone +n04141327 scabbard +n04141975 scale, weighing machine +n04146614 school bus +n04147183 schooner +n04149813 scoreboard +n04152593 screen, CRT screen +n04153751 screw +n04154565 screwdriver +n04162706 seat belt, seatbelt +n04179913 sewing machine +n04192698 shield, buckler +n04200800 shoe shop, shoe-shop, shoe store +n04201297 shoji +n04204238 shopping basket +n04204347 shopping cart +n04208210 shovel +n04209133 shower cap +n04209239 shower curtain +n04228054 ski +n04229816 ski mask +n04235860 sleeping bag +n04238763 slide rule, slipstick +n04239074 sliding door +n04243546 slot, one-armed bandit +n04251144 snorkel +n04252077 snowmobile +n04252225 snowplow, snowplough +n04254120 soap dispenser +n04254680 soccer ball +n04254777 sock +n04258138 solar dish, solar collector, solar furnace +n04259630 sombrero +n04263257 soup bowl +n04264628 space bar +n04265275 space heater +n04266014 space shuttle +n04270147 spatula +n04273569 speedboat +n04275548 spider web, spider's web +n04277352 spindle +n04285008 sports car, sport car +n04286575 spotlight, spot +n04296562 stage +n04310018 steam locomotive +n04311004 steel arch bridge +n04311174 steel drum +n04317175 stethoscope +n04325704 stole +n04326547 stone wall +n04328186 stopwatch, stop watch +n04330267 stove +n04332243 strainer +n04335435 streetcar, tram, tramcar, trolley, trolley car +n04336792 stretcher +n04344873 studio couch, day bed +n04346328 stupa, tope +n04347754 submarine, pigboat, sub, U-boat +n04350905 suit, suit of clothes +n04355338 sundial +n04355933 sunglass +n04356056 sunglasses, dark glasses, shades +n04357314 sunscreen, sunblock, sun blocker +n04366367 suspension bridge +n04367480 swab, swob, mop +n04370456 sweatshirt +n04371430 swimming trunks, bathing trunks +n04371774 swing +n04372370 switch, electric switch, electrical switch +n04376876 syringe +n04380533 table lamp +n04389033 tank, army tank, armored combat vehicle, armoured combat vehicle +n04392985 tape player +n04398044 teapot +n04399382 teddy, teddy bear +n04404412 television, television system +n04409515 tennis ball +n04417672 thatch, thatched roof +n04418357 theater curtain, theatre curtain +n04423845 thimble +n04428191 thresher, thrasher, threshing machine +n04429376 throne +n04435653 tile roof +n04442312 toaster +n04443257 tobacco shop, tobacconist shop, tobacconist +n04447861 toilet seat +n04456115 torch +n04458633 totem pole +n04461696 tow truck, tow car, wrecker +n04462240 toyshop +n04465501 tractor +n04467665 trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +n04476259 tray +n04479046 trench coat +n04482393 tricycle, trike, velocipede +n04483307 trimaran +n04485082 tripod +n04486054 triumphal arch +n04487081 trolleybus, trolley coach, trackless trolley +n04487394 trombone +n04493381 tub, vat +n04501370 turnstile +n04505470 typewriter keyboard +n04507155 umbrella +n04509417 unicycle, monocycle +n04515003 upright, upright piano +n04517823 vacuum, vacuum cleaner +n04522168 vase +n04523525 vault +n04525038 velvet +n04525305 vending machine +n04532106 vestment +n04532670 viaduct +n04536866 violin, fiddle +n04540053 volleyball +n04542943 waffle iron +n04548280 wall clock +n04548362 wallet, billfold, notecase, pocketbook +n04550184 wardrobe, closet, press +n04552348 warplane, military plane +n04553703 washbasin, handbasin, washbowl, lavabo, wash-hand basin +n04554684 washer, automatic washer, washing machine +n04557648 water bottle +n04560804 water jug +n04562935 water tower +n04579145 whiskey jug +n04579432 whistle +n04584207 wig +n04589890 window screen +n04590129 window shade +n04591157 Windsor tie +n04591713 wine bottle +n04592741 wing +n04596742 wok +n04597913 wooden spoon +n04599235 wool, woolen, woollen +n04604644 worm fence, snake fence, snake-rail fence, Virginia fence +n04606251 wreck +n04612504 yawl +n04613696 yurt +n06359193 web site, website, internet site, site +n06596364 comic book +n06785654 crossword puzzle, crossword +n06794110 street sign +n06874185 traffic light, traffic signal, stoplight +n07248320 book jacket, dust cover, dust jacket, dust wrapper +n07565083 menu +n07579787 plate +n07583066 guacamole +n07584110 consomme +n07590611 hot pot, hotpot +n07613480 trifle +n07614500 ice cream, icecream +n07615774 ice lolly, lolly, lollipop, popsicle +n07684084 French loaf +n07693725 bagel, beigel +n07695742 pretzel +n07697313 cheeseburger +n07697537 hotdog, hot dog, red hot +n07711569 mashed potato +n07714571 head cabbage +n07714990 broccoli +n07715103 cauliflower +n07716358 zucchini, courgette +n07716906 spaghetti squash +n07717410 acorn squash +n07717556 butternut squash +n07718472 cucumber, cuke +n07718747 artichoke, globe artichoke +n07720875 bell pepper +n07730033 cardoon +n07734744 mushroom +n07742313 Granny Smith +n07745940 strawberry +n07747607 orange +n07749582 lemon +n07753113 fig +n07753275 pineapple, ananas +n07753592 banana +n07754684 jackfruit, jak, jack +n07760859 custard apple +n07768694 pomegranate +n07802026 hay +n07831146 carbonara +n07836838 chocolate sauce, chocolate syrup +n07860988 dough +n07871810 meat loaf, meatloaf +n07873807 pizza, pizza pie +n07875152 potpie +n07880968 burrito +n07892512 red wine +n07920052 espresso +n07930864 cup +n07932039 eggnog +n09193705 alp +n09229709 bubble +n09246464 cliff, drop, drop-off +n09256479 coral reef +n09288635 geyser +n09332890 lakeside, lakeshore +n09399592 promontory, headland, head, foreland +n09421951 sandbar, sand bar +n09428293 seashore, coast, seacoast, sea-coast +n09468604 valley, vale +n09472597 volcano +n09835506 ballplayer, baseball player +n10148035 groom, bridegroom +n10565667 scuba diver +n11879895 rapeseed +n11939491 daisy +n12057211 yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum +n12144580 corn +n12267677 acorn +n12620546 hip, rose hip, rosehip +n12768682 buckeye, horse chestnut, conker +n12985857 coral fungus +n12998815 agaric +n13037406 gyromitra +n13040303 stinkhorn, carrion fungus +n13044778 earthstar +n13052670 hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +n13054560 bolete +n13133613 ear, spike, capitulum +n15075141 toilet tissue, toilet paper, bathroom tissue diff --git a/Robustness_Eval/CLEVER/nlayer_model.py b/Robustness_Eval/CLEVER/nlayer_model.py new file mode 100644 index 0000000..6da963e --- /dev/null +++ b/Robustness_Eval/CLEVER/nlayer_model.py @@ -0,0 +1,105 @@ +## setup_mnist.py -- mnist data and model loading code +## +## Copyright (C) 2017-2018, IBM Corp. +## Copyright (C) 2017, Huan Zhang . +## +## This program is licenced under the Apache 2.0 licence, +## contained in the LICENCE file in this directory. + +import numpy as np +import os +import pickle +import gzip +import argparse +import urllib.request + +from tensorflow.contrib.keras.api.keras.models import Sequential +from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout, Activation, Flatten +from tensorflow.contrib.keras.api.keras.layers import Conv2D, MaxPooling2D +from tensorflow.contrib.keras.api.keras.models import load_model +from tensorflow.contrib.keras.api.keras import backend as K + + +class NLayerModel: + def __init__(self, params, restore = None, session=None, use_log=False, image_size=28, image_channel=1): + + self.image_size = image_size + self.num_channels = image_channel + self.num_labels = 10 + + model = Sequential() + model.add(Flatten(input_shape=(image_size, image_size, image_channel))) + # list of all hidden units weights + self.U = [] + for param in params: + # add each dense layer, and save a reference to list U + self.U.append(Dense(param)) + model.add(self.U[-1]) + # ReLU activation + model.add(Activation('relu')) + self.W = Dense(10) + model.add(self.W) + # output log probability, used for black-box attack + if use_log: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.layer_outputs = layer_outputs + self.model = model + + def predict(self, data): + return self.model(data) + + +if __name__ == "__main__": + import scipy.io as sio + parser = argparse.ArgumentParser(description='save n-layer MNIST and CIFAR weights') + parser.add_argument('--model', + default="mnist", + choices=["mnist", "cifar"], + help='model name') + parser.add_argument('--modelfile', + default="", + help='override the model filename, use user specied one') + parser.add_argument('layer_parameters', + nargs='+', + help='number of hidden units per layer') + args = parser.parse_args() + nlayers = len(args.layer_parameters) + 1 + + import tensorflow as tf + with tf.Session() as sess: + # if a model file is not specified, use a manual override + if not args.modelfile: + args.modelfile = "models/"+args.model+"_"+str(nlayers)+"layer_relu" + if args.model == "mnist": + model = NLayerModel(args.layer_parameters, args.modelfile, sess) + #model = NLayerModel(args.layer_parameters, "models/mnist_"+str(nlayers)+"layer_relu") + elif args.model == "cifar": + model = NLayerModel(args.layer_parameters, args.modelfile, sess, image_size=32, image_channel=3) + else: + raise(RuntimeError("Unknow model")) + + + [W, bias_W] = model.W.get_weights() + save_dict = {'W': W, 'bias_W': bias_W} + print("Output layer shape:", W.shape) + U = model.U + for i, Ui in enumerate(U): + # save hidden layer weights, layer by layer + [weight_Ui, bias_Ui] = Ui.get_weights() + print("Hidden layer {} shape: {}".format(i, weight_Ui.shape)) + save_dict['U'+str(i+1)] = weight_Ui + save_dict['bias_U'+str(i+1)] = bias_Ui + + save_name = args.model + "_" + str(nlayers) + "layers.mat" + print('saving to {} with matrices {}'.format(save_name, save_dict.keys())) + # results saved to mnist.mat or cifar.mat + sio.savemat(save_name, save_dict) + diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_cnn.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_cnn.csv new file mode 100644 index 0000000..0e49f59 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_cnn.csv @@ -0,0 +1,101 @@ +id,target +6159 +6279 +4663 +3394 +4292 +2819 +6701 +4278 +7516 +7363 +4425 +6104 +9179 +540 +1378 +3001 +4912 +4532 +2484 +8839 +7876 +2947 +5604 +8623 +1412 +3904 +8215 +4379 +4773 +3058 +493 +9592 +2541 +3632 +8772 +1301 +990 +4207 +2238 +6522 +1313 +1614 +2439 +2450 +8744 +104 +513 +5323 +7792 +940 +7483 +7715 +987 +3389 +4543 +490 +9763 +915 +1564 +8663 +4855 +4848 +3162 +1500 +5439 +8155 +2220 +8579 +7008 +9824 +4314 +3583 +8564 +1631 +6058 +6585 +3661 +1229 +3175 +9662 +8880 +2345 +9974 +626 +6048 +6789 +9618 +7936 +4466 +1270 +1048 +1810 +4736 +5918 +4371 +726 +6301 +3039 +6555 +4688 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_dd.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_dd.csv new file mode 100644 index 0000000..00508e6 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_dd.csv @@ -0,0 +1,101 @@ +id,target +8744 +540 +4425 +9592 +6159 +4379 +4532 +9179 +8772 +4207 +7516 +7234 +4371 +4278 +4773 +3001 +7363 +293 +7282 +2819 +6104 +871 +1313 +1500 +7715 +8215 +6701 +6080 +4848 +3661 +940 +1158 +231 +2484 +1301 +4999 +513 +987 +4663 +973 +4912 +5918 +490 +4314 +6814 +493 +4713 +3372 +7156 +390 +1190 +5604 +7876 +4284 +3058 +8623 +5439 +9824 +6672 +7792 +2645 +3904 +8579 +4292 +9974 +2220 +1378 +4890 +7451 +1810 +604 +3691 +6788 +9662 +7113 +9909 +1631 +5801 +5472 +3394 +9763 +5875 +2439 +5887 +9351 +5741 +2932 +7013 +5905 +5321 +6585 +9645 +2693 +626 +2947 +5480 +9865 +9634 +7484 +3632 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_mlp.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_mlp.csv new file mode 100644 index 0000000..f363e31 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_mlp.csv @@ -0,0 +1,101 @@ +id,target +1300 +7999 +9930 +8272 +9466 +6252 +9369 +1770 +5183 +9294 +5717 +4903 +8086 +5912 +9097 +8609 +6812 +7626 +7258 +4282 +5725 +1657 +3393 +8734 +1656 +3551 +3183 +104 +1376 +2202 +3312 +7222 +9610 +1033 +9504 +9609 +2315 +7945 +9408 +5052 +7205 +575 +5839 +2786 +4244 +3597 +9732 +1766 +7406 +1306 +7802 +3885 +817 +7437 +2912 +3095 +3456 +9306 +8104 +3387 +783 +178 +4661 +6051 +1759 +7982 +3967 +1438 +4614 +9003 +3390 +5490 +8610 +2690 +3772 +3991 +5365 +7134 +3757 +6308 +5187 +2018 +6907 +9415 +6148 +6917 +5854 +195 +3327 +8889 +8209 +9551 +8175 +6687 +3636 +8290 +1982 +5955 +7646 +9921 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_0.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_0.csv new file mode 100644 index 0000000..ba3cd15 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_0.csv @@ -0,0 +1,101 @@ +id,target +4317 +4567 +412 +565 +9944 +5821 +3898 +1970 +2832 +2952 +1594 +5936 +951 +1270 +8520 +5054 +1331 +2618 +2445 +1531 +2394 +2291 +3410 +8272 +367 +1640 +6019 +4379 +2084 +5913 +1998 +2998 +449 +3985 +4194 +5972 +4382 +1571 +250 +4695 +827 +3776 +4000 +3235 +1272 +2559 +5832 +2587 +2447 +3133 +125 +417 +9423 +671 +3145 +4511 +1782 +8186 +890 +1516 +447 +5842 +4974 +2898 +2339 +3437 +5867 +9656 +3995 +740 +333 +24 +5239 +3718 +4963 +5922 +3146 +3702 +6053 +2151 +7643 +4323 +1343 +4263 +3943 +5926 +4808 +8146 +62 +2105 +389 +1299 +4120 +8624 +2068 +2858 +3490 +9941 +4575 +4271 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_03.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_03.csv new file mode 100644 index 0000000..391926d --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_03.csv @@ -0,0 +1,101 @@ +id,target +2607 +1782 +3751 +3767 +2437 +7432 +4966 +1754 +9696 +2325 +1500 +4248 +3946 +8416 +1184 +4860 +1640 +3410 +2063 +1516 +1283 +175 +4000 +1270 +9613 +4886 +6019 +5268 +3808 +774 +2394 +691 +24 +4695 +6912 +9686 +542 +6700 +3333 +1192 +7226 +1718 +1429 +3133 +1551 +5832 +898 +3796 +2998 +5239 +4379 +2457 +2245 +3761 +1412 +7447 +4433 +3337 +523 +4511 +3429 +3838 +3618 +667 +827 +1594 +9544 +3196 +2366 +417 +6026 +4856 +1571 +2917 +551 +363 +9649 +716 +7402 +9700 +4498 +4816 +9538 +3851 +9831 +990 +1998 +565 +2426 +3976 +1096 +43 +4563 +2519 +8231 +9733 +7232 +671 +3850 +5979 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_cnn.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_cnn.csv new file mode 100644 index 0000000..833f49e --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_cnn.csv @@ -0,0 +1,101 @@ +id,target +3926 +3333 +606 +3206 +578 +628 +1718 +7130 +9944 +5835 +9914 +4785 +9733 +9905 +9544 +4075 +1096 +3618 +591 +4068 +2851 +2024 +4306 +565 +3716 +1173 +8358 +1594 +1012 +9217 +8272 +417 +73 +1204 +3477 +5654 +2393 +2921 +1200 +5757 +4567 +5299 +5649 +2229 +1743 +488 +335 +876 +8520 +3597 +4341 +5936 +1270 +2780 +3160 +4856 +5961 +2832 +2438 +8146 +1234 +80 +5268 +1601 +9925 +3376 +1970 +5714 +3475 +5634 +5821 +2858 +4294 +1597 +2618 +1640 +3856 +8952 +349 +4657 +5600 +1384 +9943 +4695 +9769 +2853 +5866 +5054 +3410 +3898 +9655 +464 +2394 +2424 +4256 +119 +2589 +1531 +3062 +5887 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_dd.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_dd.csv new file mode 100644 index 0000000..1be5593 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_dd.csv @@ -0,0 +1,101 @@ +id,target +2607 +1594 +565 +5936 +417 +4567 +2394 +1184 +2437 +9696 +3838 +4695 +3410 +2998 +6019 +3767 +3437 +6700 +8520 +3133 +7432 +827 +5832 +1270 +3235 +9613 +1571 +250 +3808 +4382 +9634 +1998 +1192 +1754 +8416 +8146 +4379 +9538 +24 +4000 +4966 +9423 +7402 +2898 +3333 +3985 +1640 +2858 +2084 +4511 +389 +3337 +4886 +760 +2457 +1500 +1343 +5268 +1718 +1283 +4194 +671 +1516 +4963 +4974 +2361 +951 +7332 +7447 +3718 +3580 +7226 +9544 +1924 +3946 +2325 +3796 +4507 +8312 +9686 +7680 +9733 +3477 +1865 +5561 +3136 +9208 +5926 +3821 +523 +4856 +3086 +5068 +3288 +789 +3490 +5001 +5239 +9211 +1412 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_mlp.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_mlp.csv new file mode 100644 index 0000000..ccd50d3 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_mlp.csv @@ -0,0 +1,101 @@ +id,target +2437 +1184 +9696 +3767 +9613 +1754 +6700 +3808 +3337 +565 +9686 +3946 +1551 +523 +9634 +7432 +1968 +1838 +5677 +1594 +320 +9538 +7402 +4194 +8416 +1998 +2063 +3132 +990 +3796 +4379 +9831 +6912 +7447 +3751 +2325 +2394 +2189 +4695 +1412 +363 +542 +5832 +5068 +300 +3207 +3580 +1640 +9649 +1270 +3333 +3133 +2998 +5691 +551 +1571 +4384 +8320 +898 +9587 +9612 +3005 +3976 +4974 +43 +4567 +417 +740 +3437 +7226 +8095 +1429 +1433 +4498 +4425 +2245 +6743 +1283 +4507 +4433 +9544 +5268 +4995 +125 +1320 +3850 +3477 +3410 +250 +2366 +6445 +5617 +1096 +3030 +5926 +3851 +4350 +4265 +1226 +154 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_cnn.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_cnn.csv new file mode 100644 index 0000000..b9d3cdd --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_cnn.csv @@ -0,0 +1,101 @@ +id,target +8477 +8988 +3541 +5606 +6297 +7334 +8235 +2165 +3598 +6711 +832 +5725 +9025 +248 +7511 +33 +584 +3508 +3650 +8017 +2801 +5055 +1685 +3714 +1964 +6323 +7807 +8446 +7165 +7776 +9354 +8526 +8684 +9196 +5124 +8776 +1935 +1081 +2468 +7985 +4453 +9809 +6120 +2997 +3654 +5682 +2949 +8757 +5617 +91 +9082 +1105 +2010 +1650 +5312 +5714 +3251 +6210 +3446 +2106 +4860 +4140 +3725 +7475 +4457 +9017 +338 +3830 +8649 +1782 +6756 +2157 +8806 +945 +5228 +8324 +7374 +9164 +8661 +9992 +4089 +5845 +1067 +5193 +9495 +7397 +1252 +2674 +2691 +6106 +4476 +835 +4699 +7430 +5450 +8697 +7885 +6002 +4885 +4352 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_dd.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_dd.csv new file mode 100644 index 0000000..903a81f --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_dd.csv @@ -0,0 +1,101 @@ +id,target +5312 +6339 +3830 +1976 +9590 +1303 +205 +5183 +8684 +4176 +9448 +8477 +5606 +5178 +3381 +2516 +8410 +8020 +6592 +3945 +9157 +8070 +7618 +3598 +8235 +4259 +9017 +705 +5876 +9809 +4674 +6278 +998 +2803 +9928 +5450 +2133 +7019 +7265 +1112 +7165 +2358 +2698 +7291 +5828 +1513 +8889 +8096 +6605 +2453 +2214 +2768 +854 +67 +2044 +1163 +8789 +6297 +3541 +474 +3455 +6323 +232 +3063 +832 +6727 +9779 +460 +1070 +6546 +190 +4778 +1992 +7528 +6151 +8365 +3650 +9503 +2468 +8651 +8832 +8192 +5370 +8597 +4624 +1373 +6206 +6873 +6120 +6002 +7475 +187 +8401 +5116 +8847 +1558 +4886 +7244 +4155 +8428 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_mlp.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_mlp.csv new file mode 100644 index 0000000..6c50ce5 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_mlp.csv @@ -0,0 +1,101 @@ +id,target +942 +7244 +1479 +4394 +1327 +7438 +8684 +5606 +8697 +5312 +9596 +8365 +426 +1494 +8245 +1373 +9385 +1252 +2746 +7334 +7118 +9477 +474 +8627 +2850 +5728 +6261 +1976 +8587 +6297 +3784 +1560 +7968 +6026 +2653 +147 +3860 +3927 +3454 +7985 +371 +9164 +1555 +1992 +7291 +6920 +5617 +7211 +8458 +3830 +7540 +7704 +8743 +4248 +3455 +6860 +3000 +8810 +9530 +3771 +4275 +1067 +2037 +6853 +7153 +7202 +3672 +4307 +5743 +4623 +9431 +7262 +9745 +4130 +3570 +3209 +7536 +6019 +9295 +9180 +7612 +5837 +8189 +4088 +444 +8232 +8477 +9071 +9213 +3532 +4900 +6054 +273 +21 +5349 +3323 +5115 +9508 +6873 +7460 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_0.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_0.csv new file mode 100644 index 0000000..0ab8672 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_0.csv @@ -0,0 +1,101 @@ +id,target +9129 +4981 +9082 +3142 +4418 +9072 +4905 +9111 +2679 +503 +4402 +4999 +9154 +236 +9480 +5716 +2354 +6132 +5498 +5216 +2967 +7308 +5267 +514 +9131 +3772 +4474 +2482 +5024 +3933 +3904 +4666 +3775 +3057 +5287 +4965 +9820 +1056 +2864 +7163 +6309 +5629 +5541 +9285 +4387 +5402 +2977 +7977 +3668 +5785 +6111 +5684 +8455 +8460 +5436 +2108 +8691 +331 +5465 +5192 +9995 +6828 +6307 +9573 +1915 +3703 +7528 +8734 +4961 +291 +911 +7321 +5863 +1771 +7825 +816 +467 +5385 +2837 +2327 +9032 +4070 +5829 +643 +8482 +4241 +5157 +82 +4274 +3013 +5883 +9985 +921 +7518 +1432 +7744 +6234 +6345 +1799 +5108 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_03.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_03.csv new file mode 100644 index 0000000..fdbdb97 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_03.csv @@ -0,0 +1,101 @@ +id,target +183 +7792 +9498 +9818 +7225 +9066 +7705 +5030 +8869 +1231 +8464 +2337 +8474 +3768 +2466 +9475 +8446 +8637 +5031 +9352 +9004 +3660 +2352 +7712 +6082 +5574 +8123 +3447 +7415 +546 +246 +7667 +5421 +8179 +1808 +4348 +4999 +5411 +2328 +7709 +2217 +8448 +9753 +9297 +9457 +665 +1589 +2060 +5116 +5686 +5810 +7953 +9801 +6130 +1457 +5990 +6429 +3614 +6167 +3827 +2714 +3849 +514 +8338 +3602 +8307 +192 +5259 +8657 +2294 +1061 +545 +6325 +3545 +9178 +9865 +8579 +8554 +7485 +8528 +8929 +5133 +5684 +5497 +6526 +8513 +7335 +3540 +311 +123 +5834 +5415 +9761 +5568 +1362 +8117 +9952 +6114 +6016 +1698 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_cnn.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_cnn.csv new file mode 100644 index 0000000..e574e4c --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_cnn.csv @@ -0,0 +1,101 @@ +id,target +8697 +5497 +6258 +5884 +8789 +7122 +4115 +4499 +5441 +6933 +3712 +6350 +6804 +2231 +5030 +3603 +7667 +3886 +9178 +6315 +8884 +592 +3545 +8554 +8211 +384 +8148 +8739 +9138 +6402 +5810 +9221 +8474 +8446 +5481 +5353 +8399 +6130 +6325 +5385 +2737 +8870 +8625 +2791 +6633 +4418 +7576 +8869 +4875 +1590 +9355 +7681 +5337 +7705 +4008 +5694 +5484 +5581 +7485 +6685 +5898 +6355 +7787 +2486 +6328 +2170 +6207 +8077 +5568 +9066 +9555 +5844 +4336 +7593 +9148 +1203 +7939 +9312 +392 +8454 +8622 +8828 +503 +732 +6104 +5938 +5276 +6261 +9457 +1977 +517 +973 +6354 +2458 +2826 +3036 +353 +8990 +1662 +7778 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_dd.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_dd.csv new file mode 100644 index 0000000..6b08c48 --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_dd.csv @@ -0,0 +1,101 @@ +id,target +9178 +54 +8454 +7485 +7712 +2337 +6354 +6633 +2791 +4999 +9424 +6399 +8622 +8548 +5884 +7709 +7705 +6425 +6355 +7613 +5695 +5660 +7787 +9504 +3603 +1061 +6933 +9066 +973 +2873 +2231 +9130 +6130 +5938 +2535 +6406 +9555 +3849 +6096 +5295 +183 +8614 +732 +8697 +7346 +1733 +8179 +5684 +2321 +2518 +4416 +9457 +7699 +7761 +5497 +2486 +4686 +7681 +5481 +7122 +6104 +8629 +3220 +8668 +5276 +7792 +10 +9360 +6177 +5694 +9647 +586 +2703 +8595 +7576 +8117 +246 +1698 +1898 +3036 +8206 +2170 +4499 +1977 +7295 +5844 +4008 +7723 +2644 +6319 +2996 +8572 +8619 +1293 +2657 +8448 +8587 +517 +3058 +9148 diff --git a/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_mlp.csv b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_mlp.csv new file mode 100644 index 0000000..924c0fa --- /dev/null +++ b/Robustness_Eval/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_mlp.csv @@ -0,0 +1,101 @@ +id,target +6932 +592 +2376 +6798 +296 +6760 +7566 +4278 +4241 +9414 +5686 +5337 +2592 +7846 +3052 +1610 +9060 +1457 +3849 +7882 +4227 +9129 +4992 +3359 +3179 +1535 +7516 +8194 +5990 +3170 +3624 +4348 +7780 +8595 +8514 +5531 +3982 +1203 +5259 +4070 +440 +4959 +8858 +7913 +8481 +1558 +6259 +5006 +8023 +5031 +7933 +7526 +6779 +8921 +5079 +157 +4604 +6766 +6489 +136 +3768 +148 +1261 +6342 +9757 +4641 +9004 +9223 +4320 +3178 +7835 +5684 +2568 +7168 +2384 +7703 +3830 +6097 +5435 +4965 +8179 +6127 +8528 +5592 +3445 +172 +6280 +1723 +6181 +9387 +6016 +7269 +4611 +5088 +7337 +7129 +3325 +2894 +6333 +7485 diff --git a/Robustness_Eval/CLEVER/process_log.py b/Robustness_Eval/CLEVER/process_log.py new file mode 100644 index 0000000..0827740 --- /dev/null +++ b/Robustness_Eval/CLEVER/process_log.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +process_log.py + +process log files generated by clever.py + +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang + +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" + +import os, glob +import sys +import scipy +import scipy.io as sio +from scipy.stats import weibull_min +import numpy as np +import pandas as pd +import argparse +import matplotlib.pyplot as plt + + +def readDebugLog2array(filename): + f = open(filename) + + # last 3 lines are L0 verbosity, the first line is comment + #lines = f.readlines()[0:-3] + lines = f.readlines()[0:] # read all the lines and check if belongs to [DEBUG][L1] below + # if the files are ended with "_grep" + #lines = f.readlines() + f.close() + + data_arr = {"[DEBUG][L1] id":[],"true_label":[],"target_label":[],"info":[],"bnd_norm":[], + "bnd":[],"ks":[],"pVal":[],"shape":[],"loc":[],"scale":[], "g_x0":[]} + + #Ncols = len(lines[0].strip().split(',')); + + for line in lines: + # split ',' into columns + subline = line.strip().split(',') + + #print(subline) + #print('-reading lines-') + + # only save the info when the line is [DEBUG][L1] + if subline[0].split('=')[0] == '[DEBUG][L1] id ': + for elems in subline: + temp = elems.split('='); + key = temp[0].strip() + val = temp[1].strip() + # save key and val to array + data_arr[key].append(val) + + return data_arr + + +table_results = {} + +if __name__ == "__main__": + # parse command line parameters + parser = argparse.ArgumentParser(description='Process experiment data.') + parser.add_argument('data_folder', nargs='+', help='log file(s) or directory') + parser.add_argument('--save_pickle', + action='store_true', + help='save result to pickle') + args = vars(parser.parse_args()) + print(args) + # process all the log files in the folder + flag_process_all_dir = False + # save result to pickle + is_save2pickle = args['save_pickle'] + + files = [] + # the argument is a list of paths + for path in args['data_folder']: + # if the path is a directory, look for all log files inside + if os.path.isdir(path): + files.extend(glob.glob(os.path.join(path, "*.log"))) + # if the path is a file, include it directly + else: + files.append(path) + print(files) + + for file in files: + # datas is a dictionary + datas = readDebugLog2array(file) + + # convert the dictionary to dataframe df + df = pd.DataFrame.from_dict(datas) + + # convert the string type columns to numeric + df['bnd'] = pd.to_numeric(df['bnd']) + df['g_x0'] = pd.to_numeric(df['g_x0']) + df['ks'] = pd.to_numeric(df['ks']) + df['loc'] = pd.to_numeric(df['loc']) + df['pVal'] = pd.to_numeric(df['pVal']) + df['scale'] = pd.to_numeric(df['scale']) + df['shape'] = pd.to_numeric(df['shape']) + + tag1 = os.path.basename(file).split('_')[1] + + # cifar, mnist, imagenet + if (tag1 == 'cifar') or (tag1 == 'mnist'): + modelName = tag1 + '_' + os.path.basename(file).split('_')[2].split('.')[0] + else: + modelName = tag1 + + if modelName not in table_results: + nan = float('nan') + table_results[modelName] = {'least': {'1': [nan,nan], '2': [nan,nan], 'i': [nan,nan]}, + 'random':{'1': [nan,nan], '2': [nan,nan], 'i': [nan,nan]}, + 'top2': {'1': [nan,nan], '2': [nan,nan], 'i': [nan,nan]}} + + for label in ['least','random','top2']: + for bnd_norm in ['1','2','i']: + df_out = df[(df["info"]==label) & (df["bnd_norm"]==bnd_norm)] + + if not df_out.empty: + out_name = 'pickle_'+modelName+'_'+label+'_norm'+bnd_norm + + + if is_save2pickle: + # save selected df to pickle files + df_out.to_pickle(out_name) + + # obtain statistics and print out + descrb_0 = df_out.describe() + descrb_1 = df_out[(df_out["pVal"]>0.05)&(df_out["shape"]<1000)].describe() + + + bnd_0 = descrb_0["bnd"]["mean"] + count_0 = descrb_0["bnd"]["count"] + + bnd_1 = descrb_1["bnd"]["mean"] + count_1 = descrb_1["bnd"]["count"] + + table_results[modelName][label][bnd_norm][0] = bnd_1 + table_results[modelName][label][bnd_norm][1] = count_1 * 100.0 / count_0 + print("[L0] model = {}, Nimg = {}, bnd_avg = {:.5g}, pVal>0.05 & shape<1000 gives" + "Nimg = {}, bnd_avg = {:.5g}, useable = {:.1f} %".format(out_name,count_0,bnd_0,count_1,bnd_1,count_1/count_0*100)) + # print out table for easy pasting to LaTeX + output = sys.stdout + print('Generating LaTeX table...') + order=['mnist_2-layer', 'mnist_normal', 'mnist_distilled', 'mnist_brelu', 'cifar_2-layer', 'cifar_normal', 'cifar_distilled', 'cifar_brelu', 'inception', 'resnet', 'mobilenet'] + + def gen_table(elem_index, precision=3): + for label in ['least','random','top2']: + output.write("{:15s} &\t{:7s} &\t{:7s} &\n".format(label, '2', 'i')) + for model in order: + if model in table_results: + output.write("{:15s} &\t".format(model)) + for bnd_norm in ['2','i']: + output.write(("{:7."+str(precision)+"f} &\t").format(table_results[model][label][bnd_norm][elem_index])) + output.write("\n") + + print('\n%%% Table for bounds %%%') + gen_table(0) + print('\n%%% Table for p-values %%%') + gen_table(1,1) + diff --git a/Robustness_Eval/CLEVER/randsphere.py b/Robustness_Eval/CLEVER/randsphere.py new file mode 100644 index 0000000..76bb45b --- /dev/null +++ b/Robustness_Eval/CLEVER/randsphere.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import numpy as np +import time +from shmemarray import ShmemRawArray, NpShmemArray +from scipy.special import gammainc +from defense import defend_reduce, defend_jpeg, defend_tv, defend_none, defend_png +from functools import partial + +""" +Original Matlab code (for L2 sampling): + +function X = randsphere(m,n,r) + +X = randn(m,n); +s2 = sum(X.^2,2); +X = X.*repmat(r*(gammainc(s2/2,n/2).^(1/n))./sqrt(s2),1,n); +""" + +# generate random signs +def randsign(N): + n_bytes = (N + 7) // 8 + rbytes = np.random.randint(0, 255, dtype=np.uint8, size=n_bytes) + return (np.unpackbits(rbytes)[:N] - 0.5) * 2 + +def l2_samples(m,n): + X = np.random.randn(m, n) + s2 = np.sum(X * X, axis = 1) + return X * (np.tile(1.0*np.power(gammainc(n/2,s2/2), 1/n) / np.sqrt(s2), (n,1))).T + +def linf_samples(m, n): + return np.random.uniform(-1.0, 1.0, (m,n)) + +def l1_samples(m, n): + # U is uniform random between 0, 1 + U = np.random.uniform(0, 1.0, (m,n-1)) + V = np.empty(shape=(m,n+1)) + # V is sorted U, with 0 and 1 added to the begin and the end + V[:,0] = 0.0 + V[:,-1] = 1.0 + V[:,1:-1] = np.sort(U) + # X is the interval between each V_i + X = V[:,1:] - V[:,:-1] + # randomly flip the sign of each X + s = randsign(m * n).reshape(m,n) + return X * s + +def randsphere(idx, n, r, total_size, scale_size, tag_prefix, input_shape, norm, transform = None): + # currently we assume r = 1.0 and rescale using the array "scale" + assert r == 1.0 + result_arr = NpShmemArray(np.float32, (total_size, n), tag_prefix + "randsphere", False) + # for scale, we may want a different starting point for imagenet, which is scale_start + scale = NpShmemArray(np.float32, (scale_size, 1), tag_prefix + "scale", False) + input_example = NpShmemArray(np.float32, input_shape, tag_prefix + "input_example", False) + all_inputs = NpShmemArray(np.float32, (total_size,) + input_example.shape, tag_prefix + "all_inputs", False) + # m is the number of items, off is the offset + m, offset, scale_start = idx + # n is the dimension + if norm == "l2": + samples = l2_samples(m, n) + if norm == "l1": + samples = l1_samples(m, n) + if norm == "li": + samples = linf_samples(m, n) + result_arr[offset : offset + m] = samples + # make a scaling + result_arr[offset : offset + m] *= scale[offset + scale_start : offset + scale_start + m] + # add to input example + all_inputs[offset : offset + m] = input_example + result_arr = result_arr.reshape(-1, *input_shape) + all_inputs[offset : offset + m] += result_arr[offset : offset + m] + # apply an transformation + if transform: + transform_new = lambda x: (eval(transform)(np.squeeze(x) + 0.5) - 0.5).reshape(all_inputs[offset].shape) + # before transformation we need to clip first + np.clip(all_inputs[offset : offset + m], -0.5, 0.5, out = all_inputs[offset : offset + m]) + # run transformation + for i in range(offset, offset + m): + all_inputs[i] = transform_new(all_inputs[i]) + # we need to clip again after transformation (this is in the caller) + return + diff --git a/Robustness_Eval/CLEVER/run.sh b/Robustness_Eval/CLEVER/run.sh new file mode 100644 index 0000000..ed52b46 --- /dev/null +++ b/Robustness_Eval/CLEVER/run.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +""" +run.sh + +Run file: collect the samples of gradient norms + +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang + +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" +# run file for running 1st order and 2nd order clever + +if [ "$#" -le 6 ]; then + echo "Usage: $0 model modeltype nsamp niters activation order target gpuNum" + echo "model={mnist, cifar}; modeltype={2-layer, normal}; activation={tanh, sigmoid, softplus}; order={1,2}" + echo "target={top2, rand, least, all}, gpuNum={0,1,2} on frigg, {0,1} on apollo, {0} local" + exit 1 +fi + +## set up parameters +# mnist or cifar +model=$1 +# 2-layer, normal(7-layer) +modeltype=$2 +# 200 +nsamp=$3 +# 10, 20, 50, 100, 200, 300 +niters=$4 +# tanh, sigmoid, softplus +activation=$5 +# 1, 2 +order=$6 +# 1 (top2), 2 (rand), 4 (least), 15 (all) +target=$7 + +if [ "$#" -le 7 ]; then + gpuNum="0" + #echo "Number of args = $#" + echo "Using GPU 0" +else + gpuNum=$8 + #echo "Number of args = $#" + echo "Using GPU $gpuNum" +fi + +if [ "$target" == "top2" ]; then + target_type="1" +elif [ "$target" == "rand" ]; then + target_type="2" +elif [ "$target" == "least" ]; then + target_type="4" +elif [ "$target" == "all" ]; then + target_type="15" +else + echo "Wrong target $target: should be one in {top2, rand, least, all}" + exit 1 +fi + +output="${model}_${modeltype}_${activation}_ord${order}_ns${nsamp}_nit${niters}_${target}_$(date +%m%d_%H%M%S)" +dir="logs/$model/$modeltype" +mkdir -p $dir +logfile=$dir/$output.log +echo $logfile + +CMD="python3 collect_gradients.py --numimg 20 -d $model --model_name $modeltype --batch_size $nsamp --Nsamps $nsamp --Niters $niters --activation $activation --target_type $target_type --order $order" +echo $CMD + +## run on frigg +###CUDA_VISIBLE_DEVICES=$gpuNum +CUDA_VISIBLE_DEVICES=$gpuNum $CMD 2>&1 | tee $logfile +##$CMD 2>&1 | tee $logfile + +#NUMBA_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 $CMD 2>&1 | tee $logfile +# NUMBA_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 $CMD >$logfile 2>$logfile.err +echo "Done $logfile" + + + diff --git a/Robustness_Eval/CLEVER/run_clever_16.sh b/Robustness_Eval/CLEVER/run_clever_16.sh new file mode 100644 index 0000000..6b2207e --- /dev/null +++ b/Robustness_Eval/CLEVER/run_clever_16.sh @@ -0,0 +1,52 @@ +declare -A model_list1 +declare -A model_list2 +declare -A categ_list +categ_list=(["vanilla"]="" ["vanilla_reverse"]="reverse_") +dataset_list1="mnist" +model_list1=(["mnist_0"]="mnist_0" ["mnist_03"]="mnist_03") +dataset_list2="mnist cifar" +model_list2=(["2-layer"]="mlp" ["normal"]="cnn" ["distilled"]="dd") +lines=1 +curtime=`date +"%Y-%m-%d-%I-%M-%p"` +numimg=20 +tempfname="tempfile_`echo $curtime`.txt" + +for categ in ${!categ_list[@]}; do + for dataset in $dataset_list1; do + for model in ${!model_list1[@]}; do + python3 collect_gradients.py --numimg $numimg --target_type 16 --dataset $dataset --model_name $model --save ./lipschitz_mat/$categ --ids cknn_spade_nodelist/`echo ${model_list1[$model]}`_`echo ${categ_list[$categ]}`TopNode.csv + done + done + + for dataset in $dataset_list2; do + for model in ${!model_list2[@]}; do + python3 collect_gradients.py --numimg $numimg --target_type 16 --dataset $dataset --model_name $model --save ./lipschitz_mat/$categ --ids cknn_spade_nodelist/`echo $dataset`_`echo ${model_list2[$model]}`_`echo ${categ_list[$categ]}`TopNode.csv + done + done +done + + +echo "">$tempfname +for categ in ${!categ_list[@]}; do + fname=clever_score/clever_score_`echo $categ`_num`echo $numimg`_`echo $curtime`.txt + echo "">$fname + for dataset in $dataset_list1; do + for model in ${!model_list1[@]}; do + echo $dataset $categ $model >> $fname + python3 clever.py lipschitz_mat/$categ/`echo $dataset`_`echo $model` > $tempfname + echo "$(tail -$lines $tempfname)">>$fname + # echo lipschitz_mat/$categ/`echo $dataset`_`echo $model` + done + done + + for dataset in $dataset_list2; do + for model in ${!model_list2[@]}; do + echo $dataset $categ $model >> $fname + python3 clever.py lipschitz_mat/$categ/`echo $dataset`_`echo $model` > $tempfname + echo "$(tail -$lines $tempfname)">>$fname + # echo lipschitz_mat/$categ/`echo $dataset`_`echo $model` + done + done +done + +rm $tempfname \ No newline at end of file diff --git a/Robustness_Eval/CLEVER/setup_cifar.py b/Robustness_Eval/CLEVER/setup_cifar.py new file mode 100644 index 0000000..66fe648 --- /dev/null +++ b/Robustness_Eval/CLEVER/setup_cifar.py @@ -0,0 +1,173 @@ +## setup_cifar.py -- cifar data and model loading code +## +## Copyright (C) 2017-2018, IBM Corp. +## Copyright (C) 2017, Lily Weng +## and Huan Zhang +## Copyright (C) 2016, Nicholas Carlini . +## +## This program is licenced under the Apache 2.0 licence, +## contained in the LICENCE file in this directory. + + +import tensorflow as tf +import numpy as np +import os +import pickle +import gzip +import pickle +import urllib.request + +from tensorflow.contrib.keras.api.keras.models import Sequential +from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout, Activation, Flatten +from tensorflow.contrib.keras.api.keras.layers import Conv2D, MaxPooling2D +from tensorflow.contrib.keras.api.keras.layers import Lambda +from tensorflow.contrib.keras.api.keras.models import load_model +from tensorflow.contrib.keras.api.keras import backend as K + +def load_batch(fpath, label_key='labels'): + f = open(fpath, 'rb') + d = pickle.load(f, encoding="bytes") + for k, v in d.items(): + del(d[k]) + d[k.decode("utf8")] = v + f.close() + data = d["data"] + labels = d[label_key] + + data = data.reshape(data.shape[0], 3, 32, 32) + final = np.zeros((data.shape[0], 32, 32, 3),dtype=np.float32) + final[:,:,:,0] = data[:,0,:,:] + final[:,:,:,1] = data[:,1,:,:] + final[:,:,:,2] = data[:,2,:,:] + + final /= 255 + final -= .5 + labels2 = np.zeros((len(labels), 10)) + labels2[np.arange(len(labels2)), labels] = 1 + + return final, labels + +def load_batch(fpath): + f = open(fpath,"rb").read() + size = 32*32*3+1 + labels = [] + images = [] + for i in range(10000): + arr = np.fromstring(f[i*size:(i+1)*size],dtype=np.uint8) + lab = np.identity(10)[arr[0]] + img = arr[1:].reshape((3,32,32)).transpose((1,2,0)) + + labels.append(lab) + images.append((img/255)-.5) + return np.array(images),np.array(labels) + + +class CIFAR: + def __init__(self): + train_data = [] + train_labels = [] + + if not os.path.exists("cifar-10-batches-bin"): + urllib.request.urlretrieve("https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz", + "cifar-data.tar.gz") + os.popen("tar -xzf cifar-data.tar.gz").read() + + + for i in range(5): + r,s = load_batch("cifar-10-batches-bin/data_batch_"+str(i+1)+".bin") + train_data.extend(r) + train_labels.extend(s) + + train_data = np.array(train_data,dtype=np.float32) + train_labels = np.array(train_labels) + + self.test_data, self.test_labels = load_batch("cifar-10-batches-bin/test_batch.bin") + + VALIDATION_SIZE = 5000 + + self.validation_data = train_data[:VALIDATION_SIZE, :, :, :] + self.validation_labels = train_labels[:VALIDATION_SIZE] + self.train_data = train_data[VALIDATION_SIZE:, :, :, :] + self.train_labels = train_labels[VALIDATION_SIZE:] + +class CIFARModel: + def __init__(self, restore=None, session=None, use_softmax=False, use_brelu = False, activation = "relu"): + def bounded_relu(x): + return K.relu(x, max_value=1) + if use_brelu: + activation = bounded_relu + else: + activation = activation + + print("inside CIFARModel: activation = {}".format(activation)) + + self.num_channels = 3 + self.image_size = 32 + self.num_labels = 10 + + model = Sequential() + + model.add(Conv2D(64, (3, 3), + input_shape=(32, 32, 3))) + model.add(Activation(activation)) + model.add(Conv2D(64, (3, 3))) + model.add(Activation(activation)) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Conv2D(128, (3, 3))) + model.add(Activation(activation)) + model.add(Conv2D(128, (3, 3))) + model.add(Activation(activation)) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Flatten()) + model.add(Dense(256)) + model.add(Activation(activation)) + model.add(Dense(256)) + model.add(Activation(activation)) + model.add(Dense(10)) + if use_softmax: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.layer_outputs = layer_outputs + self.model = model + + def predict(self, data): + return self.model(data) + + +class TwoLayerCIFARModel: + def __init__(self, restore = None, session=None, use_softmax=False): + self.num_channels = 3 + self.image_size = 32 + self.num_labels = 10 + + model = Sequential() + model.add(Flatten(input_shape=(32, 32, 3))) + model.add(Dense(1024)) + model.add(Activation('softplus')) + model.add(Dense(10)) + # output log probability, used for black-box attack + if use_softmax: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.layer_outputs = layer_outputs + self.model = model + + def predict(self, data): + + return self.model(data) diff --git a/Robustness_Eval/CLEVER/setup_imagenet.py b/Robustness_Eval/CLEVER/setup_imagenet.py new file mode 100644 index 0000000..6c8bbe5 --- /dev/null +++ b/Robustness_Eval/CLEVER/setup_imagenet.py @@ -0,0 +1,498 @@ +## Modified by Huan Zhang for ResNet, Inception v1, v2, v4, VGG, MobileNet, Densenet, Alexnet and NASnet +## Modified by Huan Zhang for the updated Inception-v3 model (inception_v3_2016_08_28.tar.gz) +## Modified by Nicholas Carlini to match model structure for attack code. +## Original copyright license follows. + + +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Simple image classification with an ImageNet Classifier. + +Run image classification with an ImageNet Classifier (Inception, ResNet, AlexNet, etc) trained on ImageNet 2012 Challenge data +set. + +This program creates a graph from a saved GraphDef protocol buffer, +and runs inference on an input JPEG image. It outputs human readable +strings of the top 5 predictions along with their probabilities. + +Change the --image_file argument to any jpg image to compute a +classification of that image. + +Please see the tutorial and website for a detailed description of how +to use this script to perform image recognition. + +https://tensorflow.org/tutorials/image_recognition/ +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os.path +import re +import sys +from functools import partial +import random +import tarfile +import scipy.misc + +import numpy as np +from six.moves import urllib +import tensorflow as tf + +import PIL +from PIL import Image + + +model_params = {} + + +"""Add a new new entry to ImageNet models + +Parameters: +name: name of the new model, like "resnet" +url: URL to download the model +image_size: image size, usually 224 or 299 +model_filename: model protobuf file name (.pb) +label_filename: a text file contains the mapping from class ID to human readable string +input_tensor: input tensor of the network defined by protobuf, like "input:0" +logit: logit output tensor of the network, like "resnet_v2_50/predictions/Reshape:0" +prob: probability output tensor of the network, like "resnet_v2_50/predictions/Reshape_1:0" +shape: tensor for reshaping the final output, like "resnet_v2_50/predictions/Shape:0". + Set to None if no reshape needed. + +All the tensor names can be viewed and found in TensorBoard. +""" +def AddModel(name, url, model_filename, image_size, label_filename, input_tensor, logit, prob, shape): + global model_params + param = {} + param['url'] = url + param['model_filename'] = model_filename + param['size'] = image_size + param['input'] = input_tensor + param['logit'] = logit + param['prob'] = prob + param['shape'] = shape + param['label_filename'] = label_filename + param['name'] = name + model_params[name] = param + +# pylint: disable=line-too-long +AddModel('resnet_v2_50', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_resnet_v2_50.pb', 299, 'labels.txt', 'input:0', + 'resnet_v2_50/predictions/Reshape:0', 'resnet_v2_50/predictions/Reshape_1:0', 'resnet_v2_50/predictions/Shape:0') +AddModel('resnet_v2_101', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_resnet_v2_101.pb', 299, 'labels.txt', 'input:0', + 'resnet_v2_101/predictions/Reshape:0', 'resnet_v2_101/predictions/Reshape_1:0', 'resnet_v2_101/predictions/Shape:0') +AddModel('resnet_v2_152', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_resnet_v2_152.pb', 299, 'labels.txt', 'input:0', + 'resnet_v2_152/predictions/Reshape:0', 'resnet_v2_152/predictions/Reshape_1:0', 'resnet_v2_152/predictions/Shape:0') +AddModel('inception_v1', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_v1.pb', 224, 'labels.txt', 'input:0', + 'InceptionV1/Logits/Predictions/Reshape:0', 'InceptionV1/Logits/Predictions/Reshape_1:0', 'InceptionV1/Logits/Predictions/Shape:0') +AddModel('inception_v2', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_v2.pb', 224, 'labels.txt', 'input:0', + 'InceptionV2/Predictions/Reshape:0', 'InceptionV2/Predictions/Reshape_1:0', 'InceptionV2/Predictions/Shape:0') +AddModel('inception_v3', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_v3.pb', 299, 'labels.txt', 'input:0', + 'InceptionV3/Predictions/Reshape:0', 'InceptionV3/Predictions/Softmax:0', 'InceptionV3/Predictions/Shape:0') +AddModel('inception_v4', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_v4.pb', 299, 'labels.txt', 'input:0', + 'InceptionV4/Logits/Logits/BiasAdd:0', 'InceptionV4/Logits/Predictions:0', '') +AddModel('inception_resnet_v2', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_resnet_v2.pb', 299, 'labels.txt', 'input:0', + 'InceptionResnetV2/Logits/Logits/BiasAdd:0', 'InceptionResnetV2/Logits/Predictions:0', '') +AddModel('vgg_16', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_vgg_16.pb', 224, 'labels.txt', 'input:0', + 'vgg_16/fc8/squeezed:0', 'vgg_16/fc8/squeezed:0', '') +AddModel('vgg_19', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_vgg_19.pb', 224, 'labels.txt', 'input:0', + 'vgg_19/fc8/squeezed:0', 'vgg_19/fc8/squeezed:0', '') +AddModel('mobilenet_v1_025', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_mobilenet_v1_025.pb', 224, 'labels.txt', 'input:0', + 'MobilenetV1/Predictions/Reshape:0', 'MobilenetV1/Predictions/Reshape_1:0', 'MobilenetV1/Predictions/Shape:0') +AddModel('mobilenet_v1_050', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_mobilenet_v1_050.pb', 224, 'labels.txt', 'input:0', + 'MobilenetV1/Predictions/Reshape:0', 'MobilenetV1/Predictions/Reshape_1:0', 'MobilenetV1/Predictions/Shape:0') +AddModel('mobilenet_v1_100', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_mobilenet_v1_100.pb', 224, 'labels.txt', 'input:0', + 'MobilenetV1/Predictions/Reshape:0', 'MobilenetV1/Predictions/Reshape_1:0', 'MobilenetV1/Predictions/Shape:0') +AddModel('nasnet_large', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_nasnet_large.pb', 331, 'labels.txt', 'input:0', + 'final_layer/FC/BiasAdd:0', 'final_layer/predictions:0', '') +AddModel('densenet121_k32', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'densenet121_k32_frozen.pb', 224, 'labels.txt', 'input:0', + 'densenet121/predictions/Reshape:0', 'densenet121/predictions/Reshape_1:0', 'densenet121/predictions/Shape:0') +AddModel('densenet169_k32', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'densenet169_k32_frozen.pb', 224, 'labels.txt', 'input:0', + 'densenet169/predictions/Reshape:0', 'densenet169/predictions/Reshape_1:0', 'densenet169/predictions/Shape:0') +AddModel('densenet161_k48', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'densenet161_k48_frozen.pb', 224, 'labels.txt', 'input:0', + 'densenet161/predictions/Reshape:0', 'densenet161/predictions/Reshape_1:0', 'densenet161/predictions/Shape:0') +AddModel('alexnet', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'alexnet_frozen.pb', 227, 'labels.txt', 'Placeholder:0', + 'fc8/fc8:0', 'Softmax:0', '') + +# pylint: enable=line-too-long + + +class NodeLookup(object): + """Converts integer node ID's to human readable labels.""" + + def __init__(self, + label_lookup_path=None): + if not label_lookup_path: + label_lookup_path = os.path.join( + FLAGS.model_dir, 'labels.txt') + self.node_lookup = self.load(label_lookup_path) + + def load(self, label_lookup_path): + """Loads a human readable English name for each softmax node. + + Args: + label_lookup_path: string UID to integer node ID. + uid_lookup_path: string UID to human-readable string. + + Returns: + dict from integer node ID to human-readable string. + """ + if not tf.gfile.Exists(label_lookup_path): + tf.logging.fatal('File does not exist %s', label_lookup_path) + + # Loads mapping from string UID to integer node ID. + node_id_to_name = {} + proto_as_ascii = tf.gfile.GFile(label_lookup_path).readlines() + for line in proto_as_ascii: + if line: + words = line.split(':') + target_class = int(words[0]) + name = words[1] + node_id_to_name[target_class] = name + + return node_id_to_name + + def id_to_string(self, node_id): + if node_id not in self.node_lookup: + return '' + return self.node_lookup[node_id] + +LOADED_GRAPH = None + +def create_graph(model_param): + """Creates a graph from saved GraphDef file and returns a saver.""" + # Creates graph from saved graph_def.pb. + global LOADED_GRAPH + with tf.gfile.FastGFile(os.path.join( + # FLAGS.model_dir, 'classify_image_graph_def.pb'), 'rb') as f: + FLAGS.model_dir, model_param['model_filename']), 'rb') as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + #for line in repr(graph_def).split("\n"): + # if "tensor_content" not in line: + # print(line) + LOADED_GRAPH = graph_def + + +class ImageNetModelPrediction: + def __init__(self, sess, use_softmax = False, model_name = "resnet_v2_50", softmax_tensor = None): + self.sess = sess + self.use_softmax = use_softmax + model_param = model_params[model_name] + self.output_name = model_param['prob'] if self.use_softmax else model_param['logit'] + self.input_name = model_param['input'] + self.shape_name = model_param['shape'] + self.model_name = model_param['name'] + self.image_size = model_param['size'] + self.img = tf.placeholder(tf.float32, (None, self.image_size, self.image_size, 3)) + if not softmax_tensor: + # no existing graph + self.softmax_tensor = tf.import_graph_def( + LOADED_GRAPH, + # sess.graph.as_graph_def(), + input_map={self.input_name: self.img}, + return_elements=[self.output_name]) + if 'vgg' in self.model_name and use_softmax == True: + # the pretrained VGG network output is logits, need an extra softmax + self.softmax_tensor = tf.nn.softmax(self.softmax_tensor) + else: + # use an existing graph + self.softmax_tensor = softmax_tensor + print("GraphDef Size:", self.sess.graph_def.ByteSize()) + + def predict(self, dat): + dat = np.squeeze(dat) + if 'vgg' in self.model_name: + # VGG uses 0 - 255 image as input + dat = (0.5 + dat) * 255.0 + imagenet_mean = np.array([123.68, 116.78, 103.94], dtype=np.float32) + dat -= imagenet_mean + elif 'alexnet' in self.model_name: + if dat.ndim == 3: + dat = dat[:,:,::-1] + else: + dat = dat[:,:,:,::-1] # change RGB to BGR + dat = (0.5 + dat) * 255.0 + imagenet_mean = np.array([104., 117., 124.], dtype=np.float32) + dat -= imagenet_mean + elif 'densenet' in self.model_name: + dat = (0.5 + dat) * 255.0 + imagenet_mean = np.array([123.68, 116.78, 103.94], dtype=np.float32) + dat -= imagenet_mean + dat = dat * 0.017 + else: + dat = dat * 2.0 + + + if dat.ndim == 3: + scaled = dat.reshape((1,) + dat.shape) + else: + scaled = dat + # print(scaled.shape) + predictions = self.sess.run(self.softmax_tensor, + {self.img: scaled}) + predictions = np.squeeze(predictions) + return predictions + # Creates node ID --> English string lookup. + node_lookup = NodeLookup() + top_k = predictions.argsort()#[-FLAGS.num_top_predictions:][::-1] + for node_id in top_k: + print('id',node_id) + human_string = node_lookup.id_to_string(node_id) + score = predictions[node_id] + print('%s (score = %.5f)' % (human_string, score)) + return top_k[-1] + + +CREATED_GRAPH = False +class ImageNetModel: + def __init__(self, sess, use_softmax = False, model_name = "resnet_v2_50", create_prediction = True): + global CREATED_GRAPH + self.sess = sess + self.use_softmax = use_softmax + model_param = model_params[model_name] + maybe_download_and_extract(model_param) + + if not CREATED_GRAPH: + create_graph(model_param) + CREATED_GRAPH = True + self.num_channels = 3 + self.output_name = model_param['prob'] if self.use_softmax else model_param['logit'] + self.input_name = model_param['input'] + self.shape_name = model_param['shape'] + self.model_name = model_param['name'] + self.num_labels = 1000 if 'vgg' in self.model_name or 'densenet' in self.model_name or 'alexnet' in self.model_name else 1001 + self.image_size = model_param['size'] + self.use_softmax = use_softmax + if create_prediction: + self.model = ImageNetModelPrediction(sess, use_softmax, model_name) + + def predict(self, img): + if 'vgg' in self.model_name: + # VGG uses 0 - 255 image as input + img = (0.5 + img) * 255.0 + imagenet_mean = np.array([123.68, 116.78, 103.94], dtype=np.float32) + img -= imagenet_mean + elif 'alexnet' in self.model_name: + img = tf.reverse(img,axis=[-1])# change RGB to BGR + img = (0.5 + img) * 255.0 + imagenet_mean = np.array([104., 117., 124.], dtype=np.float32) + img -= imagenet_mean + elif 'densenet' in self.model_name: + # convert to 0 - 255 image as input + img = (0.5 + img) * 255.0 + imagenet_mean = np.array([123.68, 116.78, 103.94], dtype=np.float32) + img -= imagenet_mean + img = img * 0.017 + else: + img = img * 2.0 + + if img.shape.is_fully_defined() and img.shape.as_list()[0] and self.shape_name: + # check if a shape has been specified explicitly + shape = (int(img.shape[0]), self.num_labels) + self.softmax_tensor = tf.import_graph_def( + LOADED_GRAPH, + # self.sess.graph.as_graph_def(), + input_map={self.input_name: img, self.shape_name: shape}, + return_elements=[self.output_name]) + if 'vgg' in self.model_name and self.use_softmax == True: + # the pretrained VGG network output is logitimport_graph_defs, need an extra softmax + self.softmax_tensor = tf.nn.softmax(self.softmax_tensor) + else: + # placeholder shape + self.softmax_tensor = tf.import_graph_def( + LOADED_GRAPH, + # self.sess.graph.as_graph_def(), + input_map={self.input_name: img}, + return_elements=[self.output_name]) + if 'vgg' in self.model_name and self.use_softmax == True: + # the pretrained VGG network output is logits, need an extra softmax + self.softmax_tensor = tf.nn.softmax(self.softmax_tensor) + print("GraphDef Size:", self.sess.graph_def.ByteSize()) + return self.softmax_tensor[0] + + +def maybe_download_and_extract(model_param): + """Download and extract model tar file.""" + dest_directory = FLAGS.model_dir + if not os.path.exists(dest_directory): + os.makedirs(dest_directory) + filename = model_param['url'].split('/')[-1] + filepath = os.path.join(dest_directory, filename) + modelname = model_param['model_filename'].split('/')[-1] + modelpath = os.path.join(dest_directory, modelname) + if not os.path.exists(modelpath): + def _progress(count, block_size, total_size): + sys.stdout.write('\r>> Downloading %s %.1f%%' % ( + filename, float(count * block_size) / float(total_size) * 100.0)) + sys.stdout.flush() + filepath, _ = urllib.request.urlretrieve(model_param['url'], filepath, _progress) + print() + statinfo = os.stat(filepath) + print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.') + if os.path.splitext(filename)[1] != '.pb': + tarfile.open(filepath, 'r:gz').extractall(dest_directory) + + +def main(_): + param = model_params[FLAGS.model_name] + maybe_download_and_extract(param) + image = (FLAGS.image_file if FLAGS.image_file else + os.path.join(FLAGS.model_dir, 'cropped_panda.jpg')) + # run_inference_on_image(image) + create_graph(param) + image_size = param['size'] + with tf.Session() as sess: + dat = np.array(scipy.misc.imresize(scipy.misc.imread(image),(image_size, image_size)), dtype = np.float32) + dat /= 255.0 + dat -= 0.5 + # print(dat) + model = ImageNetModelPrediction(sess, True, FLAGS.model_name) + predictions = model.predict(dat) + # Creates node ID --> English string lookup. + node_lookup = NodeLookup() + top_k = predictions.argsort()#[-FLAGS.num_top_predictions:][::-1] + for node_id in top_k: + score = predictions[node_id] + if 'vgg' in FLAGS.model_name or 'densenet' in FLAGS.model_name or 'alexnet' in FLAGS.model_name: + node_id += 1 + print('id',node_id) + human_string = node_lookup.id_to_string(node_id) + print('%s (score = %.5f)' % (human_string, score)) + + +def keep_aspect_ratio_transform(img, img_size): + + s_0, s_1 = img.size + if s_0 < s_1: + ratio = (img_size / float(s_0)) + size_1 = int((float(img.size[1]) * float(ratio))) + img = img.resize((img_size, size_1), PIL.Image.ANTIALIAS) + else: + ratio = (img_size / float(s_1)) + size_0 = int((float(img.size[0]) * float(ratio))) + img = img.resize((size_0, img_size), PIL.Image.ANTIALIAS) + + c_0 = img.size[0] // 2 + c_1 = img.size[1] // 2 + + if img_size % 2 == 0: + w_left = h_top = img_size // 2 + w_right = h_bottom = img_size // 2 + else: + w_left = h_top = img_size // 2 + w_right = h_bottom = img_size // 2 + 1 + + transformed_img = img.crop( + ( + c_0 - w_left, + c_1 - h_top, + c_0 + w_right, + c_1 + h_bottom + ) + ) + + return transformed_img + +def readimg(ff, img_size): + f = "../imagenetdata/imgs/"+ff + # img = scipy.misc.imread(f) + # skip small images (image should be at least img_size X img_size) + + # if img.shape[0] < img_size or img.shape[1] < img_size: + # return None + + # img = np.array(scipy.misc.imresize(img,(img_size, img_size)),dtype=np.float32)/255.0-.5 + img = Image.open(f) + transformed_img = keep_aspect_ratio_transform(img, img_size) + + img = np.array(transformed_img)/255.0-.5 + if img.shape != (img_size, img_size, 3): + # grayscale image + if img.shape == (img_size, img_size): + img = np.repeat(np.expand_dims(img, axis = 2), 3, axis = 2) + return [img, int(ff.split(".")[0])] + return None + return [img, int(ff.split(".")[0])] + +class ImageNet: + def __init__(self, img_size, load_total_imgs = 1000): + from multiprocessing import Pool, cpu_count + pool = Pool(cpu_count()) + file_list = sorted(os.listdir("../imagenetdata/imgs/")) + random.shuffle(file_list) + # for efficiency, we only load first 1000 images + # You can pass load_total_imgs to load all images + short_file_list = file_list[:load_total_imgs] + r = pool.map(partial(readimg, img_size=img_size), short_file_list) + print(short_file_list) + print("Loaded imagenet", len(short_file_list), "of", len(file_list), "images") + + r = [x for x in r if x != None] + test_data, test_labels = zip(*r) + self.test_data = np.array(test_data) + self.test_labels = np.zeros((len(test_labels), 1001)) + self.test_labels[np.arange(len(test_labels)), test_labels] = 1 + + pool.close() + pool.join() + +if __name__ == '__main__': + FLAGS = tf.app.flags.FLAGS + # classify_image_graph_def.pb: + # Binary representation of the GraphDef protocol buffer. + # imagenet_synset_to_human_label_map.txt: + # Map from synset ID to a human readable string. + # imagenet_2012_challenge_label_map_proto.pbtxt: + # Text representation of a protocol buffer mapping a label to synset ID. + tf.app.flags.DEFINE_string( + 'model_dir', 'tmp/imagenet', + """Path to classify_image_graph_def.pb, """ + """imagenet_synset_to_human_label_map.txt, and """ + """imagenet_2012_challenge_label_map_proto.pbtxt.""") + tf.app.flags.DEFINE_string('image_file', '', + """Absolute path to image file.""") + tf.app.flags.DEFINE_string('model_name', 'resnet_v2_101', + """Absolute path to image file.""") + tf.app.flags.DEFINE_integer('num_top_predictions', 5, + """Display this many predictions.""") + tf.app.run() +else: + # starting from TF 1.5, an parameter unkown by tf.app.flags will raise an error + # so we cannot use tf.app.flags when loading this file as a module, because the + # main program may define other options. + from argparse import Namespace + FLAGS = Namespace(model_dir="tmp/imagenet") + diff --git a/Robustness_Eval/CLEVER/setup_mnist.py b/Robustness_Eval/CLEVER/setup_mnist.py new file mode 100644 index 0000000..ed9ef63 --- /dev/null +++ b/Robustness_Eval/CLEVER/setup_mnist.py @@ -0,0 +1,238 @@ +## setup_mnist.py -- mnist data and model loading code +## +## Copyright (C) 2017-2018, IBM Corp. +## Copyright (C) 2017, Lily Weng +## and Huan Zhang +## Copyright (C) 2016, Nicholas Carlini . +## +## This program is licenced under the Apache 2.0 licence, +## contained in the LICENCE file in this directory. + +import tensorflow as tf +import numpy as np +import os +import pickle +import gzip +import urllib.request + +from tensorflow.contrib.keras.api.keras.models import Sequential +from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout, Activation, Flatten +from tensorflow.contrib.keras.api.keras.layers import Conv2D, MaxPooling2D +from tensorflow.contrib.keras.api.keras.layers import Lambda +from tensorflow.contrib.keras.api.keras.models import load_model +from tensorflow.contrib.keras.api.keras import backend as K + +def extract_data(filename, num_images): + with gzip.open(filename) as bytestream: + bytestream.read(16) + buf = bytestream.read(num_images*28*28) + data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32) + data = (data / 255) - 0.5 + data = data.reshape(num_images, 28, 28, 1) + return data + +def extract_labels(filename, num_images): + with gzip.open(filename) as bytestream: + bytestream.read(8) + buf = bytestream.read(1 * num_images) + labels = np.frombuffer(buf, dtype=np.uint8) + return (np.arange(10) == labels[:, None]).astype(np.float32) + +class MNIST: + def __init__(self): + if not os.path.exists("data"): + os.mkdir("data") + files = ["train-images-idx3-ubyte.gz", + "t10k-images-idx3-ubyte.gz", + "train-labels-idx1-ubyte.gz", + "t10k-labels-idx1-ubyte.gz"] + for name in files: + + urllib.request.urlretrieve('http://yann.lecun.com/exdb/mnist/' + name, "data/"+name) + + train_data = extract_data("data/train-images-idx3-ubyte.gz", 60000) + train_labels = extract_labels("data/train-labels-idx1-ubyte.gz", 60000) + self.test_data = extract_data("data/t10k-images-idx3-ubyte.gz", 10000) + self.test_labels = extract_labels("data/t10k-labels-idx1-ubyte.gz", 10000) + + VALIDATION_SIZE = 5000 + + self.validation_data = train_data[:VALIDATION_SIZE, :, :, :] + self.validation_labels = train_labels[:VALIDATION_SIZE] + self.train_data = train_data[VALIDATION_SIZE:, :, :, :] + self.train_labels = train_labels[VALIDATION_SIZE:] + + +class MNISTModel: + def __init__(self, restore = None, session=None, use_softmax=False, use_brelu = False, activation = "relu"): + def bounded_relu(x): + return K.relu(x, max_value=1) + if use_brelu: + activation = bounded_relu + + print("inside MNISTModel: activation = {}".format(activation)) + + self.num_channels = 1 + self.image_size = 28 + self.num_labels = 10 + + model = Sequential() + + model.add(Conv2D(32, (3, 3), + input_shape=(28, 28, 1))) + model.add(Activation(activation)) + model.add(Conv2D(32, (3, 3))) + model.add(Activation(activation)) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Conv2D(64, (3, 3))) + model.add(Activation(activation)) + model.add(Conv2D(64, (3, 3))) + model.add(Activation(activation)) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Flatten()) + model.add(Dense(200)) + model.add(Activation(activation)) + model.add(Dense(200)) + model.add(Activation(activation)) + model.add(Dense(10)) + # output log probability, used for black-box attack + if use_softmax: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.model = model + self.layer_outputs = layer_outputs + + def predict(self, data): + return self.model(data) + +class TwoLayerMNISTModel: + def __init__(self, restore = None, session=None, use_softmax=False): + self.num_channels = 1 + self.image_size = 28 + self.num_labels = 10 + + model = Sequential() + model.add(Flatten(input_shape=(28, 28, 1))) + model.add(Dense(1024)) + model.add(Lambda(lambda x: x * 10)) + model.add(Activation('softplus')) + model.add(Lambda(lambda x: x * 0.1)) + model.add(Dense(10)) + # output log probability, used for black-box attack + if use_softmax: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.layer_outputs = layer_outputs + self.model = model + + def predict(self, data): + + return self.model(data) + + +class mnist_0: + def __init__(self, restore = None, session=None, use_softmax=False, use_brelu = False, activation = "relu"): + def bounded_relu(x): + return K.relu(x, max_value=1) + if use_brelu: + activation = bounded_relu + + print("inside MNISTModel: activation = {}".format(activation)) + + self.num_channels = 1 + self.image_size = 28 + self.num_labels = 10 + + labels = tf.placeholder(tf.int64, shape=[None], name='labels') + logits = tf.placeholder(tf.float32, shape=[None, 10], name='logits') + + model = Sequential() + conv1 = Conv2D(32, (5,5), input_shape=(28,28,1), activation='relu', padding='SAME', use_bias=True) + model.add(conv1) + model.add(MaxPooling2D(pool_size=(2,2),padding='SAME')) + conv2 = Conv2D(64, (5,5), activation='relu', padding='SAME', use_bias=True) + model.add(conv2) + model.add(MaxPooling2D(pool_size=(2,2),padding='SAME')) + model.add(Flatten()) + + fc1 = Dense(1024, activation='relu', use_bias=True) + model.add(fc1) + fc2 = Dense(10, activation='softmax', use_bias=True) + model.add(fc2) + + # output log probability, used for black-box attack + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.model = model + self.layer_outputs = layer_outputs + + def predict(self, data): + return self.model(data) + + +class mnist_03: + def __init__(self, restore = None, session=None, use_softmax=False, use_brelu = False, activation = "relu"): + def bounded_relu(x): + return K.relu(x, max_value=1) + if use_brelu: + activation = bounded_relu + + print("inside MNISTModel: activation = {}".format(activation)) + + self.num_channels = 1 + self.image_size = 28 + self.num_labels = 10 + + labels = tf.placeholder(tf.int64, shape=[None], name='labels') + logits = tf.placeholder(tf.float32, shape=[None, 10], name='logits') + + model = Sequential() + conv1 = Conv2D(32, (5,5), input_shape=(28,28,1), activation='relu', padding='SAME', use_bias=True) + model.add(conv1) + model.add(MaxPooling2D(pool_size=(2,2),padding='SAME')) + conv2 = Conv2D(64, (5,5), activation='relu', padding='SAME', use_bias=True) + model.add(conv2) + model.add(MaxPooling2D(pool_size=(2,2),padding='SAME')) + model.add(Flatten()) + + fc1 = Dense(1024, activation='relu', use_bias=True) + model.add(fc1) + fc2 = Dense(10, activation='softmax', use_bias=True) + model.add(fc2) + + # output log probability, used for black-box attack + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.model = model + self.layer_outputs = layer_outputs + + def predict(self, data): + return self.model(data) diff --git a/Robustness_Eval/CLEVER/shmemarray.py b/Robustness_Eval/CLEVER/shmemarray.py new file mode 100644 index 0000000..aa561a6 --- /dev/null +++ b/Robustness_Eval/CLEVER/shmemarray.py @@ -0,0 +1,104 @@ +# +# Based on multiprocessing.sharedctypes.RawArray +# +# Uses posix_ipc (http://semanchuk.com/philip/posix_ipc/) to allow shared ctypes arrays +# among unrelated processors +# +# Usage Notes: +# * The first two args (typecode_or_type and size_or_initializer) should work the same as with RawArray. +# * The shared array is accessible by any process, as long as tag matches. +# * The shared memory segment is unlinked when the origin array (that returned +# by ShmemRawArray(..., create=True)) is deleted/gc'ed +# * Creating an shared array using a tag that currently exists will raise an ExistentialError +# * Accessing a shared array using a tag that doesn't exist (or one that has been unlinked) will also +# raise an ExistentialError +# +# Author: Shawn Chin (http://shawnchin.github.com) +# +# Edited for python 3 by: Adam Stooke +# + +import numpy as np +# import os +import sys +import mmap +import ctypes +import posix_ipc +# from _multiprocessing import address_of_buffer # (not in python 3) +from string import ascii_letters, digits + +valid_chars = frozenset("/-_. %s%s" % (ascii_letters, digits)) + +typecode_to_type = { + 'c': ctypes.c_char, 'u': ctypes.c_wchar, + 'b': ctypes.c_byte, 'B': ctypes.c_ubyte, + 'h': ctypes.c_short, 'H': ctypes.c_ushort, + 'i': ctypes.c_int, 'I': ctypes.c_uint, + 'l': ctypes.c_long, 'L': ctypes.c_ulong, + 'f': ctypes.c_float, 'd': ctypes.c_double +} + + +def address_of_buffer(buf): # (python 3) + return ctypes.addressof(ctypes.c_char.from_buffer(buf)) + + +class ShmemBufferWrapper(object): + + def __init__(self, tag, size, create=True): + # default vals so __del__ doesn't fail if __init__ fails to complete + self._mem = None + self._map = None + self._owner = create + self.size = size + + assert 0 <= size < sys.maxsize # sys.maxint (python 3) + flag = (0, posix_ipc.O_CREX)[create] + self._mem = posix_ipc.SharedMemory(tag, flags=flag, size=size) + self._map = mmap.mmap(self._mem.fd, self._mem.size) + self._mem.close_fd() + + def get_address(self): + # addr, size = address_of_buffer(self._map) + # assert size == self.size + assert self._map.size() == self.size # (changed for python 3) + addr = address_of_buffer(self._map) + return addr + + def __del__(self): + if self._map is not None: + self._map.close() + if self._mem is not None and self._owner: + self._mem.unlink() + + +def ShmemRawArray(typecode_or_type, size_or_initializer, tag, create=True): + assert frozenset(tag).issubset(valid_chars) + if tag[0] != "/": + tag = "/%s" % (tag,) + + type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) + if isinstance(size_or_initializer, int): + type_ = type_ * size_or_initializer + else: + type_ = type_ * len(size_or_initializer) + + buffer = ShmemBufferWrapper(tag, ctypes.sizeof(type_), create=create) + obj = type_.from_address(buffer.get_address()) + obj._buffer = buffer + + if not isinstance(size_or_initializer, int): + obj.__init__(*size_or_initializer) + + return obj + + +############################################################################### +# New Additions (by Adam) # + + +def NpShmemArray(dtype, shape, tag, create=True): + size = int(np.prod(shape)) + nbytes = size * np.dtype(dtype).itemsize + shmem = ShmemRawArray(ctypes.c_char, nbytes, tag, create) + return np.frombuffer(shmem, dtype=dtype, count=size).reshape(shape) diff --git a/Robustness_Eval/GAIRAT/GAIR.py b/Robustness_Eval/GAIRAT/GAIR.py new file mode 100644 index 0000000..caaf792 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR.py @@ -0,0 +1,17 @@ +from models import * +import torch +import numpy as np + +def GAIR(num_steps, Kappa, Lambda, func): + # Weight assign + if func == "Tanh": + reweight = ((Lambda+(int(num_steps/2)-Kappa)*5/(int(num_steps/2))).tanh()+1)/2 + normalized_reweight = reweight * len(reweight) / reweight.sum() + elif func == "Sigmoid": + reweight = (Lambda+(int(num_steps/2)-Kappa)*5/(int(num_steps/2))).sigmoid() + normalized_reweight = reweight * len(reweight) / reweight.sum() + elif func == "Discrete": + reweight = ((num_steps+1)-Kappa)/(num_steps+1) + normalized_reweight = reweight * len(reweight) / reweight.sum() + + return normalized_reweight diff --git a/Robustness_Eval/GAIRAT/GAIRAT.py b/Robustness_Eval/GAIRAT/GAIRAT.py new file mode 100644 index 0000000..d1840d7 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIRAT.py @@ -0,0 +1,359 @@ +""" +The code is adapted from the Jingfeng Zhang's code: +https://github.com/zjfheart/Geometry-aware-Instance-reweighted-Adversarial-Training/blob/main/GAIRAT.py +""" + + + + +import os +import argparse +import torchvision +import torch.optim as optim +from torchvision import transforms +from models import * +from GAIR import GAIR +import numpy as np +import attack_generator as attack +from utils import Logger + +parser = argparse.ArgumentParser(description='GAIRAT: Geometry-aware instance-dependent adversarial training') +parser.add_argument('--epochs', type=int, default=120, metavar='N', help='number of epochs to train') +parser.add_argument('--weight-decay', '--wd', default=2e-4, type=float, metavar='W') +parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='SGD momentum') +parser.add_argument('--epsilon', type=float, default=0.031, help='perturbation bound') +parser.add_argument('--num-steps', type=int, default=10, help='maximum perturbation step K') +parser.add_argument('--step-size', type=float, default=0.007, help='step size') +parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed') +parser.add_argument('--net', type=str, default="WRN",help="decide which network to use,choose from smallcnn,resnet18,WRN") +parser.add_argument('--dataset', type=str, default="cifar10", help="choose from cifar10,svhn,cifar100,mnist") +parser.add_argument('--random',type=bool,default=True,help="whether to initiat adversarial sample with random noise") +parser.add_argument('--depth',type=int,default=32,help='WRN depth') +parser.add_argument('--width-factor',type=int,default=10,help='WRN width factor') +parser.add_argument('--drop-rate',type=float,default=0.0, help='WRN drop rate') +parser.add_argument('--resume',type=str,default=None,help='whether to resume training') +parser.add_argument('--out-dir',type=str,default='./GAIRAT_result',help='dir of output') +parser.add_argument('--lr-schedule', default='piecewise', choices=['superconverge', 'piecewise', 'linear', 'onedrop', 'multipledecay', 'cosine']) +parser.add_argument('--lr-max', default=0.1, type=float) +parser.add_argument('--lr-one-drop', default=0.01, type=float) +parser.add_argument('--lr-drop-epoch', default=100, type=int) +parser.add_argument('--Lambda',type=str, default='-1.0', help='parameter for GAIR') +parser.add_argument('--Lambda_max',type=float, default=float('inf'), help='max Lambda') +parser.add_argument('--Lambda_schedule', default='fixed', choices=['linear', 'piecewise', 'fixed']) +parser.add_argument('--weight_assignment_function', default='Tanh', choices=['Discrete','Sigmoid','Tanh']) +parser.add_argument('--begin_epoch', type=int, default=60, help='when to use GAIR') +args = parser.parse_args() + +# Training settings +seed = args.seed +momentum = args.momentum +weight_decay = args.weight_decay +depth = args.depth +width_factor = args.width_factor +drop_rate = args.drop_rate +resume = args.resume +out_dir = args.out_dir + +torch.manual_seed(seed) +np.random.seed(seed) +torch.cuda.manual_seed_all(seed) +torch.backends.cudnn.benchmark = True +torch.backends.cudnn.deterministic = True + +# Models and optimizer +if args.net == "smallcnn": + model = SmallCNN().cuda() + net = "smallcnn" +if args.net == "resnet18": + model = ResNet18().cuda() + net = "resnet18" +if args.net == "preactresnet18": + model = PreActResNet18().cuda() + net = "preactresnet18" +if args.net == "WRN": + model = Wide_ResNet_Madry(depth=depth, num_classes=10, widen_factor=width_factor, dropRate=drop_rate).cuda() + net = "WRN{}-{}-dropout{}".format(depth,width_factor,drop_rate) + +model = torch.nn.DataParallel(model) +optimizer = optim.SGD(model.parameters(), lr=args.lr_max, momentum=momentum, weight_decay=weight_decay) + +# Learning schedules +if args.lr_schedule == 'superconverge': + lr_schedule = lambda t: np.interp([t], [0, args.epochs * 2 // 5, args.epochs], [0, args.lr_max, 0])[0] +elif args.lr_schedule == 'piecewise': + def lr_schedule(t): + if args.epochs >= 110: + # Train Wide-ResNet + if t / args.epochs < 0.5: + return args.lr_max + elif t / args.epochs < 0.75: + return args.lr_max / 10. + elif t / args.epochs < (11/12): + return args.lr_max / 100. + else: + return args.lr_max / 200. + else: + # Train ResNet + if t / args.epochs < 0.3: + return args.lr_max + elif t / args.epochs < 0.6: + return args.lr_max / 10. + else: + return args.lr_max / 100. +elif args.lr_schedule == 'linear': + lr_schedule = lambda t: np.interp([t], [0, args.epochs // 3, args.epochs * 2 // 3, args.epochs], [args.lr_max, args.lr_max, args.lr_max / 10, args.lr_max / 100])[0] +elif args.lr_schedule == 'onedrop': + def lr_schedule(t): + if t < args.lr_drop_epoch: + return args.lr_max + else: + return args.lr_one_drop +elif args.lr_schedule == 'multipledecay': + def lr_schedule(t): + return args.lr_max - (t//(args.epochs//10))*(args.lr_max/10) +elif args.lr_schedule == 'cosine': + def lr_schedule(t): + return args.lr_max * 0.5 * (1 + np.cos(t / args.epochs * np.pi)) + +# Store path +if not os.path.exists(out_dir): + os.makedirs(out_dir) + +# Save checkpoint +def save_checkpoint(state, checkpoint=out_dir, filename='checkpoint.pth.tar'): + filepath = os.path.join(checkpoint, filename) + torch.save(state, filepath) + +# Get adversarially robust network +def train(epoch, end_epoch, model, train_loader, test_loader, optimizer, Lambda): + + lr = 0 + num_data = 0 + train_robust_loss = 0 + + x_adv_res = np.zeros((1,3*32*32)) + data_res = np.zeros((1,3*32*32)) + kappa_res = np.zeros((1)) + logit_res = np.zeros((1,10)) + logit_clean_res = np.zeros((1,10)) + weight_res = np.zeros((1)) + labels_res = np.zeros((1)) + + for batch_idx, (data, target) in enumerate(train_loader): + loss = 0 + data, target = data.cuda(), target.cuda() + + # Get adversarial data and geometry value + x_adv, Kappa = attack.GA_PGD(model,data,target,args.epsilon,args.step_size,args.num_steps,loss_fn="cent",category="Madry",rand_init=True) + + model.train() + lr = lr_schedule(epoch + 1) + optimizer.param_groups[0].update(lr=lr) + optimizer.zero_grad() + + logit = model(x_adv) + logit_clean = model(data) + + shp = x_adv.shape + if len(shp) == 2: + x_adv_out = x_adv + data_out = data + else: + x_adv_out = x_adv.reshape(shp[0], shp[1]*shp[2]*shp[3]) + data_out = data.reshape(shp[0], shp[1]*shp[2]*shp[3]) + x_adv_res = np.vstack((x_adv_res, x_adv_out.detach().cpu().numpy())) + kappa_res = np.append(kappa_res, Kappa.cpu().numpy()) + logit_res = np.vstack((logit_res, logit.detach().cpu().numpy())) + data_res = np.vstack((data_res, data_out.detach().cpu().numpy())) + logit_clean_res = np.vstack((logit_clean_res, logit_clean.detach().cpu().numpy())) + labels_res = np.append(labels_res,target.cpu().numpy()) + + if (epoch + 1) >= args.begin_epoch: + Kappa = Kappa.cuda() + loss = nn.CrossEntropyLoss(reduce=False)(logit, target) + # Calculate weight assignment according to geometry value + normalized_reweight = GAIR(args.num_steps, Kappa, Lambda, args.weight_assignment_function) + loss = loss.mul(normalized_reweight).mean() + + weight_res = np.append(weight_res, normalized_reweight.cpu().numpy()) + + else: + loss = nn.CrossEntropyLoss(reduce="mean")(logit, target) + + train_robust_loss += loss.item() * len(x_adv) + + loss.backward() + optimizer.step() + + num_data += len(data) + + train_robust_loss = train_robust_loss / num_data + + x_adv_res = x_adv_res[1:] + kappa_res = kappa_res[1:] + logit_res = logit_res[1:] + logit_clean_res = logit_clean_res[1:] + data_res = data_res[1:] + weight_res = weight_res[1:] + labels_res = labels_res[1:] + + return train_robust_loss, lr, x_adv_res, kappa_res, logit_res, data_res, logit_clean_res, weight_res, labels_res + +# Adjust lambda for weight assignment using epoch +def adjust_Lambda(epoch): + Lam = float(args.Lambda) + if args.epochs >= 110: + # Train Wide-ResNet + Lambda = args.Lambda_max + if args.Lambda_schedule == 'linear': + if epoch >= 60: + Lambda = args.Lambda_max - (epoch/args.epochs) * (args.Lambda_max - Lam) + elif args.Lambda_schedule == 'piecewise': + if epoch >= 60: + Lambda = Lam + elif epoch >= 90: + Lambda = Lam-1.0 + elif epoch >= 110: + Lambda = Lam-1.5 + elif args.Lambda_schedule == 'fixed': + if epoch >= 60: + Lambda = Lam + else: + # Train ResNet + Lambda = args.Lambda_max + if args.Lambda_schedule == 'linear': + if epoch >= 30: + Lambda = args.Lambda_max - (epoch/args.epochs) * (args.Lambda_max - Lam) + elif args.Lambda_schedule == 'piecewise': + if epoch >= 30: + Lambda = Lam + elif epoch >= 60: + Lambda = Lam-2.0 + elif args.Lambda_schedule == 'fixed': + if epoch >= 30: + Lambda = Lam + return Lambda + +# Setup data loader +transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), +]) +transform_test = transforms.Compose([ + transforms.ToTensor(), +]) + +if args.dataset == "cifar10": + trainset = torchvision.datasets.CIFAR10(root='./data/cifar-10', train=True, download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=False, num_workers=2) + testset = torchvision.datasets.CIFAR10(root='./data/cifar-10', train=False, download=True, transform=transform_test) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2) +if args.dataset == "svhn": + trainset = torchvision.datasets.SVHN(root='./data/SVHN', split='train', download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=False, num_workers=2) + testset = torchvision.datasets.SVHN(root='./data/SVHN', split='test', download=True, transform=transform_test) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2) +if args.dataset == "mnist": + trainset = torchvision.datasets.MNIST(root='./data/MNIST', train=True, download=True, transform=transforms.ToTensor()) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=False, num_workers=1,pin_memory=True) + testset = torchvision.datasets.MNIST(root='./data/MNIST', train=False, download=True, transform=transforms.ToTensor()) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=1,pin_memory=True) + +# Resume +title = 'GAIRAT' +best_acc = 0 +start_epoch = 0 +if resume: + # Resume directly point to checkpoint.pth.tar + print ('==> GAIRAT Resuming from checkpoint ..') + print(resume) + assert os.path.isfile(resume) + out_dir = os.path.dirname(resume) + checkpoint = torch.load(resume) + start_epoch = checkpoint['epoch'] + best_acc = checkpoint['test_pgd20_acc'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + logger_test = Logger(os.path.join(out_dir, 'log_results.txt'), title=title, resume=True) +else: + print('==> GAIRAT') + logger_test = Logger(os.path.join(out_dir, 'log_results.txt'), title=title) + logger_test.set_names(['Epoch', 'Natural Test Acc', 'PGD20 Acc']) + +## Training get started +test_nat_acc = 0 +test_pgd20_acc = 0 + +best_x_adv_res = np.array([]) +best_kappa_res = np.array([]) +best_logit_res = np.array([]) +best_data_res = np.array([]) +best_logit_clean_res = np.array([]) +best_weight_res = np.array([]) +best_labels_res = np.array([]) + +output_ad = r'{}/'.format(args.out_dir) +if not os.path.exists(output_ad): + os.makedirs(output_ad) + +for epoch in range(start_epoch, args.epochs): + + # Get lambda + Lambda = adjust_Lambda(epoch + 1) + + # Adversarial training + train_robust_loss, lr, x_adv_res, kappa_res, logit_res, data_res, logit_clean_res, weight_res, labels_res = train(epoch, args.epochs, model, train_loader, test_loader, optimizer, Lambda) + + # Evalutions similar to DAT. + _, test_nat_acc = attack.eval_clean(model, test_loader) + _, test_pgd20_acc = attack.eval_robust(model, test_loader, perturb_steps=20, epsilon=0.031, step_size=0.031 / 4,loss_fn="cent", category="Madry", random=True) + + + print( + 'Epoch: [%d | %d] | Learning Rate: %f | Natural Test Acc %.2f | PGD20 Test Acc %.2f |\n' % ( + epoch, + args.epochs, + lr, + test_nat_acc, + test_pgd20_acc) + ) + + logger_test.append([epoch + 1, test_nat_acc, test_pgd20_acc]) + + # Save the best checkpoint + if test_pgd20_acc > best_acc: + best_acc = test_pgd20_acc + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'test_nat_acc': test_nat_acc, + 'test_pgd20_acc': test_pgd20_acc, + 'optimizer' : optimizer.state_dict(), + },filename='bestpoint.pth.tar') + + best_x_adv_res = x_adv_res + best_kappa_res=kappa_res + best_logit_res = logit_res + best_data_res = data_res + best_logit_clean_res = logit_clean_res + best_weight_res = weight_res + best_labels_res = labels_res + + # Save the last checkpoint + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'test_nat_acc': test_nat_acc, + 'test_pgd20_acc': test_pgd20_acc, + 'optimizer' : optimizer.state_dict(), + }) + + np.savetxt("{}best_x_adv.csv".format(output_ad),best_x_adv_res, "%.3f") + np.savetxt("{}best_kappa.csv".format(output_ad),best_kappa_res, "%.1f") + np.savetxt("{}best_logit.csv".format(output_ad),best_logit_res, "%.3f") + np.savetxt("{}best_logit_clean.csv".format(output_ad),best_logit_clean_res, "%.3f") + np.savetxt("{}best_data.csv".format(output_ad),best_data_res, "%.3f") + np.savetxt("{}best_weight_res.csv".format(output_ad),best_weight_res, "%.3f") + np.savetxt("{}best_labels_res.csv".format(output_ad),best_labels_res, "%.3f") +logger_test.close() diff --git a/Robustness_Eval/GAIRAT/GAIR_FAT.py b/Robustness_Eval/GAIRAT/GAIR_FAT.py new file mode 100644 index 0000000..0df7285 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_FAT.py @@ -0,0 +1,327 @@ +import os +import argparse +import torchvision +import torch.optim as optim +from torchvision import transforms +from models import * +from earlystop import GA_earlystop +from GAIR import GAIR +import numpy as np +import attack_generator as attack +from utils import Logger + +parser = argparse.ArgumentParser(description='GAIR-FAT') +parser.add_argument('--epochs', type=int, default=120, metavar='N', help='number of epochs to train') +parser.add_argument('--weight-decay', '--wd', default=2e-4, type=float, metavar='W') +parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='SGD momentum') +parser.add_argument('--epsilon', type=float, default=0.031, help='perturbation bound') +parser.add_argument('--num-steps', type=int, default=10, help='maximum perturbation step K') +parser.add_argument('--step-size', type=float, default=0.007, help='step size') +parser.add_argument('--seed', type=int, default=3, metavar='S', help='random seed') +parser.add_argument('--net', type=str, default="WRN",help="decide which network to use,choose from smallcnn,resnet18,WRN") +parser.add_argument('--tau',type=int,default=0,help='slippery step tau') +parser.add_argument('--dataset', type=str, default="cifar10", help="choose from cifar10,svhn,cifar100,mnist") +parser.add_argument('--random',type=bool,default=True,help="whether to initiat adversarial sample with random noise") +parser.add_argument('--omega',type=float,default=0.0,help="random sample parameter") +parser.add_argument('--dynamictau',type=bool,default=True,help='whether to use dynamic tau') +parser.add_argument('--depth',type=int,default=32,help='WRN depth') +parser.add_argument('--width-factor',type=int,default=10,help='WRN width factor') +parser.add_argument('--drop-rate',type=float,default=0.0, help='WRN drop rate') +parser.add_argument('--resume',type=str,default=None,help='whether to resume training') +parser.add_argument('--out-dir',type=str,default='./GAIR_FAT_result',help='dir of output') +parser.add_argument('--lr-schedule', default='piecewise', choices=['superconverge', 'piecewise', 'linear', 'onedrop', 'multipledecay', 'cosine']) +parser.add_argument('--lr-max', default=0.1, type=float) +parser.add_argument('--lr-one-drop', default=0.01, type=float) +parser.add_argument('--lr-drop-epoch', default=100, type=int) +parser.add_argument('--Lambda',type=str, default='0.0', help='parameter for join loss') +parser.add_argument('--Lambda_max',type=float, default=float('inf'), help='max Lambda') +parser.add_argument('--Lambda_schedule', default='fixed', choices=['linear', 'piecewise', 'fixed']) +parser.add_argument('--weight_assignment_function', default='Tanh', choices=['Discrete','Sigmoid','Tanh']) +parser.add_argument('--begin_epoch', type=int, default=60, help='when to use GAIR') +args = parser.parse_args() + +# Training settings +seed = args.seed +epsilon = args.epsilon +step_size = args.step_size +num_steps = args.num_steps +dynamictau = args.dynamictau +momentum = args.momentum +weight_decay = args.weight_decay +omega = args.omega +random = args.random +depth = args.depth +width_factor = args.width_factor +drop_rate = args.drop_rate +resume = args.resume +out_dir = args.out_dir + +torch.manual_seed(seed) +np.random.seed(seed) +torch.cuda.manual_seed_all(seed) +torch.backends.cudnn.benchmark = False +torch.backends.cudnn.deterministic = True + +# Models and optimizer +if args.net == "smallcnn": + model = SmallCNN().cuda() + net = "smallcnn" +if args.net == "resnet18": + model = ResNet18().cuda() + net = "resnet18" +if args.net == "preactresnet18": + model = PreActResNet18().cuda() + net = "preactresnet18" +if args.net == "WRN": + model = Wide_ResNet_Madry(depth=depth, num_classes=10, widen_factor=width_factor, dropRate=drop_rate).cuda() + net = "WRN{}-{}-dropout{}".format(depth,width_factor,drop_rate) + +model = torch.nn.DataParallel(model) +optimizer = optim.SGD(model.parameters(), lr=args.lr_max, momentum=momentum, weight_decay=weight_decay) + +# Learning schedules +if args.lr_schedule == 'superconverge': + lr_schedule = lambda t: np.interp([t], [0, args.epochs * 2 // 5, args.epochs], [0, args.lr_max, 0])[0] +elif args.lr_schedule == 'piecewise': + def lr_schedule(t): + if args.epochs >= 110: + # Train Wide-ResNet + if t / args.epochs < 0.5: + return args.lr_max + elif t / args.epochs < 0.75: + return args.lr_max / 10. + elif t / args.epochs < (11/12): + return args.lr_max / 100. + else: + return args.lr_max / 200. + else: + # Train ResNet + if t / args.epochs < 0.3: + return args.lr_max + elif t / args.epochs < 0.6: + return args.lr_max / 10. + else: + return args.lr_max / 100. +elif args.lr_schedule == 'linear': + lr_schedule = lambda t: np.interp([t], [0, args.epochs // 3, args.epochs * 2 // 3, args.epochs], [args.lr_max, args.lr_max, args.lr_max / 10, args.lr_max / 100])[0] +elif args.lr_schedule == 'onedrop': + def lr_schedule(t): + if t < args.lr_drop_epoch: + return args.lr_max + else: + return args.lr_one_drop +elif args.lr_schedule == 'multipledecay': + def lr_schedule(t): + return args.lr_max - (t//(args.epochs//10))*(args.lr_max/10) +elif args.lr_schedule == 'cosine': + def lr_schedule(t): + return args.lr_max * 0.5 * (1 + np.cos(t / args.epochs * np.pi)) + +# Store path +if not os.path.exists(out_dir): + os.makedirs(out_dir) + +# Save checkpoint +def save_checkpoint(state, checkpoint=out_dir, filename='checkpoint.pth.tar'): + filepath = os.path.join(checkpoint, filename) + torch.save(state, filepath) + +# Get adversarially robust network +def train(epoch, end_epoch, model, train_loader, test_loader, optimizer, tau, Lambda): + + lr = 0 + num_data = 0 + train_robust_loss = 0 + + if epoch >= end_epoch - 1: + fp_x = open("x_adv.csv","a+") + fp_k = open("kappa.csv", "a+") + + for batch_idx, (data, target) in enumerate(train_loader): + + loss = 0 + data, target = data.cuda(), target.cuda() + + # Get adversarial data and geometry value + output_adv, output_target, output_natural, count, Kappa = GA_earlystop(model,data,target,step_size=args.step_size,epsilon=args.epsilon,perturb_steps=args.num_steps,tau=tau,type="fat",random=random,omega=omega) + + + shp = output_adv.shape + print(Kappa.shape) + if epoch >= end_epoch - 1: + if shp.shape == 2: + op_adv_out = output_adv + else: + op_adv_out = np.reshape(output_adv, (shp[0], shp[1]*shp[2]*shp[3])) + np.savetxt(fp_x, op_adv_out) + np.savetxt(fp_k, Kappa) + + model.train() + lr = lr_schedule(epoch + 1) + optimizer.param_groups[0].update(lr=lr) + optimizer.zero_grad() + + logit = model(output_adv) + + if (epoch + 1) >= args.begin_epoch: + Kappa = Kappa.cuda() + loss = nn.CrossEntropyLoss(reduce=False)(logit, target) + # Calculate weight assignment according to geometry value + normalized_reweight = GAIR(args.num_steps, Kappa, Lambda, args.weight_assignment_function) + loss = loss.mul(normalized_reweight).mean() + else: + loss = nn.CrossEntropyLoss(reduce="mean")(logit, output_target) + train_robust_loss += loss.item() * len(output_adv) + + loss.backward() + optimizer.step() + + num_data += len(data) + + train_robust_loss = train_robust_loss / num_data + + return train_robust_loss, lr + +# Adjust tau +def adjust_tau(epoch, dynamictau): + tau = args.tau + if dynamictau: + if epoch <= 40: + tau = 0 + elif epoch <= 70: + tau = 3 + else: + tau = 6 + return tau + +def adjust_Lambda(epoch): + Lam = float(args.Lambda) + if args.epochs >= 110: + # Train Wide-ResNet + Lambda = args.Lambda_max + if args.Lambda_schedule == 'linear': + if epoch >= 60: + Lambda = args.Lambda_max - (epoch/args.epochs) * (args.Lambda_max - Lam) + elif args.Lambda_schedule == 'piecewise': + if epoch >= 60: + Lambda = Lam + elif epoch >= 90: + Lambda = Lam-1.0 + elif epoch >= 110: + Lambda = Lam-1.5 + elif args.Lambda_schedule == 'fixed': + if epoch >= 60: + Lambda = Lam + else: + # Train ResNet + Lambda = args.Lambda_max + if args.Lambda_schedule == 'linear': + if epoch >= 30: + Lambda = args.Lambda_max - (epoch/args.epochs) * (args.Lambda_max - Lam) + elif args.Lambda_schedule == 'piecewise': + if epoch >= 30: + Lambda = Lam + elif epoch >= 60: + Lambda = Lam-2.0 + elif args.Lambda_schedule == 'fixed': + if epoch >= 30: + Lambda = Lam + return Lambda + +# Setup data loader +transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), +]) +transform_test = transforms.Compose([ + transforms.ToTensor(), +]) + +if args.dataset == "cifar10": + trainset = torchvision.datasets.CIFAR10(root='./data/cifar-10', train=True, download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2) + testset = torchvision.datasets.CIFAR10(root='./data/cifar-10', train=False, download=True, transform=transform_test) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2) +if args.dataset == "svhn": + trainset = torchvision.datasets.SVHN(root='./data/SVHN', split='train', download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2) + testset = torchvision.datasets.SVHN(root='./data/SVHN', split='test', download=True, transform=transform_test) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2) +if args.dataset == "mnist": + trainset = torchvision.datasets.MNIST(root='./data/MNIST', train=True, download=True, transform=transforms.ToTensor()) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=1,pin_memory=True) + testset = torchvision.datasets.MNIST(root='./data/MNIST', train=False, download=True, transform=transforms.ToTensor()) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=1,pin_memory=True) + +# Resume +title = 'GAIR-FAT' +best_acc = 0 +start_epoch = 0 +if resume: + # Resume directly point to checkpoint.pth.tar + print ('==> GAIR-FAT Resuming from checkpoint ..') + print(resume) + assert os.path.isfile(resume) + out_dir = os.path.dirname(resume) + checkpoint = torch.load(resume) + start_epoch = checkpoint['epoch'] + best_acc = checkpoint['test_pgd20_acc'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + logger_test = Logger(os.path.join(out_dir, 'log_results.txt'), title=title, resume=True) +else: + print('==> GAIR-FAT') + logger_test = Logger(os.path.join(out_dir, 'log_results.txt'), title=title) + logger_test.set_names(['Epoch', 'Natural Test Acc', 'PGD20 Acc']) + +## Training get started +test_nat_acc = 0 +test_pgd20_acc = 0 + +for epoch in range(start_epoch, args.epochs): + + # Get tau and lambda + tau = adjust_tau(epoch + 1, dynamictau) + Lambda = adjust_Lambda(epoch + 1) + + # Adversarial training + train_robust_loss, lr = train(epoch, args.epochs, model, train_loader, test_loader, optimizer, tau, Lambda) + + # Evalutions similar to DAT. + _, test_nat_acc = attack.eval_clean(model, test_loader) + _, test_pgd20_acc = attack.eval_robust(model, test_loader, perturb_steps=20, epsilon=0.031, step_size=0.031 / 4,loss_fn="cent", category="Madry", random=True) + + + print( + 'Epoch: [%d | %d] | Learning Rate: %f | Natural Test Acc %.2f | PGD20 Test Acc %.2f |\n' % ( + epoch, + args.epochs, + lr, + test_nat_acc, + test_pgd20_acc) + ) + + logger_test.append([epoch + 1, test_nat_acc, test_pgd20_acc]) + + # Save the best checkpoint + if test_pgd20_acc > best_acc: + best_acc = test_pgd20_acc + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'test_nat_acc': test_nat_acc, + 'test_pgd20_acc': test_pgd20_acc, + 'optimizer' : optimizer.state_dict(), + },filename='bestpoint.pth.tar') + + # Save the last checkpoint + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'test_nat_acc': test_nat_acc, + 'test_pgd20_acc': test_pgd20_acc, + 'optimizer' : optimizer.state_dict(), + }) + +logger_test.close() \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/attack_cw.py b/Robustness_Eval/GAIRAT/GAIR_RST/attack_cw.py new file mode 100644 index 0000000..4411f8d --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/attack_cw.py @@ -0,0 +1,365 @@ +""" +Implementation of L_infty Carlini-Wagner attack based on the L2 implementation +in FoolBox v1.9 (with many dependencies on that pakage) +https://github.com/bethgelab/foolbox +""" + +try: + from foolbox.models import PyTorchModel + from foolbox.attacks import Attack + from foolbox.attacks.base import call_decorator + from foolbox.utils import onehot_like + from foolbox.distances import Linf + HAVE_FOOLBOX = True +except ImportError: + HAVE_FOOLBOX = False + Attack = object + call_decorator = lambda x: x +import numpy as np +import logging + + +def cw(model, + X, + y, + binary_search_steps=5, + max_iterations=1000, + learning_rate=5E-3, + initial_const=1E-2, + tau_decrease_factor=0.9 + ): + if not HAVE_FOOLBOX: + raise ImportError('Could not import FoolBox') + + foolbox_model = PyTorchModel(model, bounds=(0, 1), num_classes=10) + attack = CarliniWagnerLIAttack(foolbox_model, distance=Linf) + linf_distances = [] + for i in range(len(X)): + logging.info('Example: %g', i) + image = X[i, :].detach().cpu().numpy() + label = y[i].cpu().numpy() + adversarial = attack(image, label, + binary_search_steps=binary_search_steps, + max_iterations=max_iterations, + learning_rate=learning_rate, + initial_const=initial_const, + tau_decrease_factor=tau_decrease_factor) + logging.info('Linf distance: %g', np.max(np.abs(adversarial - image))) + linf_distances.append(np.max(np.abs(adversarial - image))) + return linf_distances + + +class CarliniWagnerLIAttack(Attack): + """The Linf version of the Carlini & Wagner attack. + + This attack is described in [1]_. This implementation + is based on the reference implementation by Carlini [2]_. + For bounds \ne (0, 1), it differs from [2]_ because we + normalize the squared L2 loss with the bounds. + + References + ---------- + .. [1] Nicholas Carlini, David Wagner: "Towards Evaluating the + Robustness of Neural Networks", https://arxiv.org/abs/1608.04644 + .. [2] https://github.com/carlini/nn_robust_attacks + + """ + + @call_decorator + def __call__(self, input_or_adv, label=None, unpack=True, + binary_search_steps=5, + tau_decrease_factor=0.9, + max_iterations=1000, + confidence=0, learning_rate=5e-3, + initial_const=1e-2, abort_early=True): + + """The L_infty version of the Carlini & Wagner attack. + + Parameters + ---------- + input_or_adv : `numpy.ndarray` or :class:`Adversarial` + The original, unperturbed input as a `numpy.ndarray` or + an :class:`Adversarial` instance. + label : int + The reference label of the original input. Must be passed + if `a` is a `numpy.ndarray`, must not be passed if `a` is + an :class:`Adversarial` instance. + unpack : bool + If true, returns the adversarial input, otherwise returns + the Adversarial object. + binary_search_steps : int + The number of steps for the binary search used to + find the optimal tradeoff-constant between distance and confidence. + max_iterations : int + The maximum number of iterations. Larger values are more + accurate; setting it too small will require a large learning rate + and will produce poor results. + confidence : int or float + Confidence of adversarial examples: a higher value produces + adversarials that are further away, but more strongly classified + as adversarial. + learning_rate : float + The learning rate for the attack algorithm. Smaller values + produce better results but take longer to converge. + initial_const : float + The initial tradeoff-constant to use to tune the relative + importance of distance and confidence. If `binary_search_steps` + is large, the initial constant is not important. + abort_early : bool + If True, Adam will be aborted if the loss hasn't decreased + for some time (a tenth of max_iterations). + + """ + + a = input_or_adv + + del input_or_adv + del label + del unpack + + if not a.has_gradient(): + logging.fatal('Applied gradient-based attack to model that ' + 'does not provide gradients.') + return + + min_, max_ = a.bounds() + + def to_attack_space(x): + # map from [min_, max_] to [-1, +1] + a = (min_ + max_) / 2 + b = (max_ - min_) / 2 + x = (x - a) / b + + # from [-1, +1] to approx. (-1, +1) + x = x * 0.999999 + + # from (-1, +1) to (-inf, +inf) + return np.arctanh(x) + + def to_model_space(x): + """Transforms an input from the attack space + to the model space. This transformation and + the returned gradient are elementwise.""" + + # from (-inf, +inf) to (-1, +1) + x = np.tanh(x) + + grad = 1 - np.square(x) + + # map from (-1, +1) to (min_, max_) + a = (min_ + max_) / 2 + b = (max_ - min_) / 2 + x = x * b + a + + grad = grad * b + return x, grad + + # variables representing inputs in attack space will be + # prefixed with att_ + att_original = to_attack_space(a.original_image) + + # will be close but not identical to a.original_image + reconstructed_original, _ = to_model_space(att_original) + + # the binary search finds the smallest const for which we + # find an adversarial + const = initial_const + lower_bound = 0 + upper_bound = np.inf + true_logits, _ = a.predictions(reconstructed_original) + true_label = np.argmax(true_logits) + # Binary search for constant + start_tau = 1.0 + for binary_search_step in range(binary_search_steps): + if binary_search_step == binary_search_steps - 1 and \ + binary_search_steps >= 10: + # in the last binary search step, use the upper_bound instead + # TODO: find out why... it's not obvious why this is useful + const = upper_bound + + logging.info( + 'starting optimization with const = {}, best overall distance = {}'.format( + const, a.distance)) + + # found adv with the current const + + att_warmstart = att_original + tau = start_tau + while tau > 1. / 256: + found_adv = False + att_perturbation = np.zeros_like(att_original) + # create a new optimizer to minimize the perturbation + optimizer = AdamOptimizer(att_perturbation.shape) + loss_at_previous_check = np.inf + # logging.info('Running with const:{}, tau:{}'.format(const, tau)) + for iteration in range(max_iterations): + + x, dxdp = to_model_space(att_warmstart + att_perturbation) + logits, i = a.predictions(x) + false_label = np.argmax(logits) + is_adv = not (false_label == true_label) + + loss, dldx, adv_loss, distance = self.loss_function( + const, tau, a, x, logits, reconstructed_original, + confidence, min_, max_) + + check_loss = logits[true_label] - logits[false_label] + + # logging.info('Iteration:{}, loss: {}; adv_loss:{}; distance:{}'.format( + # iteration, loss, adv_loss, distance)) + # backprop the gradient of the loss w.r.t. x further + # to get the gradient of the loss w.r.t. att_perturbation + assert dldx.shape == x.shape + assert dxdp.shape == x.shape + # we can do a simple elementwise multiplication, because + # grad_x_wrt_p is a matrix of elementwise derivatives + # (i.e. each x[i] w.r.t. p[i] only, for all i) and + # grad_loss_wrt_x is a real gradient reshaped as a matrix + gradient = dldx * dxdp + + att_perturbation += optimizer(gradient, learning_rate) + x, dxdp = to_model_space(att_warmstart + att_perturbation) + + if is_adv: + # Tau + binary search was successful but continuing opt + found_adv = True + + if abort_early and \ + iteration % (np.ceil(max_iterations / 10)) == 0: + # after each tenth of the iterations, check progress + # logging.info('Iteration:{}, loss: {}; best overall distance: {}; is_adv:{}'.format( + # iteration, loss, a.distance, is_adv)) + if not (loss_at_previous_check - loss > 0.0001): + break # stop Adam if there has not been progress + loss_at_previous_check = loss + + if not found_adv: + # This constant is fine, just that we broke out of tau + if tau < start_tau: + found_adv = True + start_tau = tau + break + + else: + actualtau = np.max(np.abs(x - reconstructed_original)) + if actualtau < tau: + tau = actualtau + tau = tau * tau_decrease_factor + att_warmstart = to_attack_space(x) + + if found_adv: + # logging.info('found adversarial with const = {}'.format(const)) + upper_bound = const + else: + # logging.info('failed to find adversarial ' + # 'with const = {}'.format(const)) + lower_bound = const + + if upper_bound == np.inf: + # exponential search + const *= 10 + else: + # binary search + const = (lower_bound + upper_bound) / 2 + + @classmethod + def loss_function(cls, const, tau, a, x, logits, reconstructed_original, + confidence, min_, max_): + """Returns the loss and the gradient of the loss w.r.t. x, + assuming that logits = model(x).""" + + targeted = a.target_class() is not None + if targeted: + c_minimize = cls.best_other_class(logits, a.target_class()) + c_maximize = a.target_class() + else: + c_minimize = a.original_class + c_maximize = cls.best_other_class(logits, a.original_class) + + is_adv_loss = logits[c_minimize] - logits[c_maximize] + + # is_adv is True as soon as the is_adv_loss goes below 0 + # but sometimes we want additional confidence + is_adv_loss += confidence + is_adv_loss = max(0, is_adv_loss) + + s = max_ - min_ + # squared_l2_distance = np.sum((x - reconstructed_original)**2) / s**2 + linf_distance = np.sum( + np.maximum(np.abs(x - reconstructed_original) - tau, 0)) + + total_loss = linf_distance + const * is_adv_loss + + # calculate the gradient of total_loss w.r.t. x + logits_diff_grad = np.zeros_like(logits) + logits_diff_grad[c_minimize] = 1 + logits_diff_grad[c_maximize] = -1 + is_adv_loss_grad = a.backward(logits_diff_grad, x) + assert is_adv_loss >= 0 + if is_adv_loss == 0: + is_adv_loss_grad = 0 + + squared_l2_distance_grad = (2 / s ** 2) * (x - reconstructed_original) + linf_distance_grad = np.sign(x - reconstructed_original) * ( + np.abs(x - reconstructed_original) - tau > 0) + total_loss_grad = linf_distance_grad + const * is_adv_loss_grad + return total_loss, total_loss_grad, is_adv_loss, linf_distance + + @staticmethod + def best_other_class(logits, exclude): + """Returns the index of the largest logit, ignoring the class that + is passed as `exclude`.""" + other_logits = logits - onehot_like(logits, exclude, value=np.inf) + return np.argmax(other_logits) + + +class AdamOptimizer: + """Basic Adam optimizer implementation that can minimize w.r.t. + a single variable. + + Parameters + ---------- + shape : tuple + shape of the variable w.r.t. which the loss should be minimized + + """ + + def __init__(self, shape): + self.m = np.zeros(shape) + self.v = np.zeros(shape) + self.t = 0 + + def __call__(self, gradient, learning_rate, + beta1=0.9, beta2=0.999, epsilon=10e-8): + """Updates internal parameters of the optimizer and returns + the change that should be applied to the variable. + + Parameters + ---------- + gradient : `np.ndarray` + the gradient of the loss w.r.t. to the variable + learning_rate: float + the learning rate in the current iteration + beta1: float + decay rate for calculating the exponentially + decaying average of past gradients + beta2: float + decay rate for calculating the exponentially + decaying average of past squared gradients + epsilon: float + small value to avoid division by zero + + """ + + self.t += 1 + + self.m = beta1 * self.m + (1 - beta1) * gradient + self.v = beta2 * self.v + (1 - beta2) * gradient ** 2 + + bias_correction_1 = 1 - beta1 ** self.t + bias_correction_2 = 1 - beta2 ** self.t + + m_hat = self.m / bias_correction_1 + v_hat = self.v / bias_correction_2 + + return -learning_rate * m_hat / (np.sqrt(v_hat) + epsilon) diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/attack_evaluation.py b/Robustness_Eval/GAIRAT/GAIR_RST/attack_evaluation.py new file mode 100644 index 0000000..109f12f --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/attack_evaluation.py @@ -0,0 +1,246 @@ +""" +Evaluate robustness against specific attack. +Loosely based on code from https://github.com/yaodongyu/TRADES +""" + +import os +import json +import numpy as np +import argparse +import logging +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from torch.autograd import Variable +from datasets import SemiSupervisedDataset, DATASETS +from torchvision import transforms +from attack_pgd import pgd +from attack_cw import cw +import torch.backends.cudnn as cudnn +from utils import get_model + + +def eval_adv_test(model, device, test_loader, attack, attack_params, + results_dir, num_eval_batches): + """ + evaluate model by white-box attack + """ + model.eval() + if attack == 'pgd': + restarts_matrices = [] + for restart in range(attack_params['num_restarts']): + is_correct_adv_rows = [] + count = 0 + batch_num = 0 + natural_num_correct = 0 + for data, target in test_loader: + batch_num = batch_num + 1 + if num_eval_batches and batch_num > num_eval_batches: + break + data, target = data.to(device), target.to(device) + count += len(target) + X, y = Variable(data, requires_grad=True), Variable(target) + # is_correct_adv has batch_size*num_iterations dimensions + is_correct_natural, is_correct_adv = pgd( + model, X, y, + epsilon=attack_params['epsilon'], + num_steps=attack_params['num_steps'], + step_size=attack_params['step_size'], + random_start=attack_params['random_start']) + natural_num_correct += is_correct_natural.sum() + is_correct_adv_rows.append(is_correct_adv) + + is_correct_adv_matrix = np.concatenate(is_correct_adv_rows, axis=0) + restarts_matrices.append(is_correct_adv_matrix) + + is_correct_adv_over_restarts = np.stack(restarts_matrices, axis=-1) + num_correct_adv = is_correct_adv_over_restarts.prod( + axis=-1).prod(axis=-1).sum() + + logging.info("Accuracy after %d restarts: %.4g%%" % + (restart + 1, 100 * num_correct_adv / count)) + stats = {'attack': 'pgd', + 'count': count, + 'attack_params': attack_params, + 'natural_accuracy': natural_num_correct / count, + 'is_correct_adv_array': is_correct_adv_over_restarts, + 'robust_accuracy': num_correct_adv / count, + 'restart_num': restart + } + + np.save(os.path.join(results_dir, 'pgd_results.npy'), stats) + + elif attack == 'cw': + all_linf_distances = [] + count = 0 + for data, target in test_loader: + logging.info('Batch: %g', count) + count = count + 1 + if num_eval_batches and count > num_eval_batches: + break + data, target = data.to(device), target.to(device) + X, y = Variable(data, requires_grad=True), Variable(target) + batch_linf_distances = cw(model, X, y, + binary_search_steps=attack_params[ + 'binary_search_steps'], + max_iterations=attack_params[ + 'max_iterations'], + learning_rate=attack_params[ + 'learning_rate'], + initial_const=attack_params[ + 'initial_const'], + tau_decrease_factor=attack_params[ + 'tau_decrease_factor']) + all_linf_distances.append(batch_linf_distances) + stats = {'attack': 'cw', + 'attack_params': attack_params, + 'linf_distances': np.array(all_linf_distances), + } + np.save(os.path.join(results_dir, 'cw_results.npy'), stats) + + else: + raise ValueError('Unknown attack %s' % attack) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='PyTorch CIFAR Attack Evaluation') + parser.add_argument('--dataset', type=str, default='cifar10', + choices=DATASETS, + help='The dataset') + parser.add_argument('--model_path', + help='Model for attack evaluation') + parser.add_argument('--model', '-m', default='wrn-32-10', type=str, + help='Name of the model') + parser.add_argument('--output_suffix', default='', type=str, + help='String to add to log filename') + parser.add_argument('--batch_size', type=int, default=128, metavar='N', + help='Input batch size for testing (default: 200)') + parser.add_argument('--no_cuda', action='store_true', default=False, + help='Disables CUDA training') + parser.add_argument('--epsilon', default=0.031, type=float, + help='Attack perturbation magnitude') + parser.add_argument('--attack', default='pgd', type=str, + help='Attack type (CW requires FoolBox)', + choices=('pgd', 'cw')) + parser.add_argument('--num_steps', default=40, type=int, + help='Number of PGD steps') + parser.add_argument('--step_size', default=0.01, type=float, + help='PGD step size') + parser.add_argument('--num_restarts', default=5, type=int, + help='Number of restarts for PGD attack') + parser.add_argument('--no_random_start', dest='random_start', + action='store_false', + help='Disable random PGD initialization') + parser.add_argument('--binary_search_steps', default=5, type=int, + help='Number of binary search steps for CW attack') + parser.add_argument('--max_iterations', default=1000, type=int, + help='Max number of Adam iterations in each CW' + ' optimization') + parser.add_argument('--learning_rate', default=5E-3, type=float, + help='Learning rate for CW attack') + parser.add_argument('--initial_const', default=1E-2, type=float, + help='Initial constant for CW attack') + parser.add_argument('--tau_decrease_factor', default=0.9, type=float, + help='Tau decrease factor for CW attack') + parser.add_argument('--random_seed', default=0, type=int, + help='Random seed for permutation of test instances') + parser.add_argument('--num_eval_batches', default=None, type=int, + help='Number of batches to run evalaution on') + parser.add_argument('--shuffle_testset', action='store_true', default=False, + help='Shuffles the test set') + + args = parser.parse_args() + torch.manual_seed(args.random_seed) + + output_dir, checkpoint_name = os.path.split(args.model_path) + + results_dir = os.path.join('./', args.output_suffix) + if not os.path.isdir(results_dir): + os.mkdir(results_dir) + #epoch = int(re.search('epoch(\d+)', checkpoint_name).group(1)) + epoch = 0 + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(message)s", + handlers=[ + logging.FileHandler(os.path.join(results_dir, + 'attack_epoch%d%s.log' % + (epoch, args.output_suffix))), + logging.StreamHandler() + ]) + logger = logging.getLogger() + + + + logging.info('Attack evaluation') + logging.info('Args: %s' % args) + + # settings + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + dl_kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} + + # set up data loader + transform_test = transforms.Compose([transforms.ToTensor(), ]) + testset = SemiSupervisedDataset(base_dataset=args.dataset, + train=False, root='./data/cifar-10', + download=True, + transform=transform_test) + + if args.shuffle_testset: + np.random.seed(123) + logging.info("Permuting testset") + permutation = np.random.permutation(len(testset)) + testset.data = testset.data[permutation, :] + testset.targets = [testset.targets[i] for i in permutation] + + test_loader = torch.utils.data.DataLoader(testset, + batch_size=args.batch_size, + shuffle=False, **dl_kwargs) + + checkpoint = torch.load(args.model_path) + state_dict = checkpoint.get('state_dict', checkpoint) + num_classes = checkpoint.get('num_classes', 10) + normalize_input = checkpoint.get('normalize_input', False) + model = get_model(args.model, num_classes=num_classes, + normalize_input=normalize_input) + if use_cuda: + model = torch.nn.DataParallel(model).cuda() + cudnn.benchmark = True + if not all([k.startswith('module') for k in state_dict]): + state_dict = {'module.' + k: v for k, v in state_dict.items()} + else: + def strip_data_parallel(s): + if s.startswith('module'): + return s[len('module.'):] + else: + return s + state_dict = {strip_data_parallel(k): v for k, v in state_dict.items()} + model.load_state_dict(state_dict) + + attack_params = { + 'epsilon': args.epsilon, + 'seed': args.random_seed + } + if args.attack == 'pgd': + attack_params.update({ + 'num_restarts': args.num_restarts, + 'step_size': args.step_size, + 'num_steps': args.num_steps, + 'random_start': args.random_start, + }) + elif args.attack == 'cw': + attack_params.update({ + 'binary_search_steps': args.binary_search_steps, + 'max_iterations': args.max_iterations, + 'learning_rate': args.learning_rate, + 'initial_const': args.initial_const, + 'tau_decrease_factor': args.tau_decrease_factor + }) + + logging.info('Running %s' % attack_params) + eval_adv_test(model, device, test_loader, attack=args.attack, + attack_params=attack_params, results_dir=results_dir, + num_eval_batches=args.num_eval_batches) diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/attack_pgd.py b/Robustness_Eval/GAIRAT/GAIR_RST/attack_pgd.py new file mode 100644 index 0000000..5180c4e --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/attack_pgd.py @@ -0,0 +1,53 @@ +### PGD implementation + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from torch.autograd import Variable +import torch.optim as optim +import logging + + +def pgd(model, + X, + y, + epsilon=8 / 255, + num_steps=20, + step_size=0.01, + random_start=True): + out = model(X) + is_correct_natural = (out.max(1)[1] == y).float().cpu().numpy() + perturbation = torch.zeros_like(X, requires_grad=True) + + if random_start: + perturbation = torch.rand_like(X, requires_grad=True) + perturbation.data = perturbation.data * 2 * epsilon - epsilon + + is_correct_adv = [] + opt = optim.SGD([perturbation], lr=1e-3) # This is just to clear the grad + + for _ in range(num_steps): + opt.zero_grad() + + with torch.enable_grad(): + loss = nn.CrossEntropyLoss()(model(X + perturbation), y) + + loss.backward() + + perturbation.data = ( + perturbation + step_size * perturbation.grad.detach().sign()).clamp( + -epsilon, epsilon) + perturbation.data = torch.min(torch.max(perturbation.detach(), -X), + 1 - X) # clip X+delta to [0,1] + X_pgd = Variable(torch.clamp(X.data + perturbation.data, 0, 1.0), + requires_grad=False) + is_correct_adv.append(np.reshape( + (model(X_pgd).max(1)[1] == y).float().cpu().numpy(), + [-1, 1])) + + is_correct_adv = np.concatenate(is_correct_adv, axis=1) + return is_correct_natural, is_correct_adv + + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/__init__.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/__init__.py new file mode 100644 index 0000000..00ac5fa --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/__init__.py @@ -0,0 +1 @@ +from .autoattack import AutoAttack diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/autoattack.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/autoattack.py new file mode 100644 index 0000000..9daab82 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/autoattack.py @@ -0,0 +1,252 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable +import numpy as np +import argparse +import time +from other_utils import Logger + +class AutoAttack(): + def __init__(self, model, norm='Linf', eps=.3, seed=None, verbose=True, + attacks_to_run=[], version='standard', is_tf_model=False, + device='cuda', log_path=None): + self.model = model + self.norm = norm + assert norm in ['Linf', 'L2'] + self.epsilon = eps + self.seed = seed + self.verbose = verbose + self.attacks_to_run = attacks_to_run + self.version = version + self.is_tf_model = is_tf_model + self.device = device + self.logger = Logger(log_path) + + if not self.is_tf_model: + from autopgd_pt import APGDAttack + self.apgd = APGDAttack(self.model, n_restarts=5, n_iter=100, verbose=False, + eps=self.epsilon, norm=self.norm, eot_iter=1, rho=.75, seed=self.seed, device=self.device) + + from fab_pt import FABAttack + self.fab = FABAttack(self.model, n_restarts=5, n_iter=100, eps=self.epsilon, seed=self.seed, + norm=self.norm, verbose=False, device=self.device) + + from square import SquareAttack + self.square = SquareAttack(self.model, p_init=.8, n_queries=5000, eps=self.epsilon, norm=self.norm, + n_restarts=1, seed=self.seed, verbose=False, device=self.device, resc_schedule=False) + + from autopgd_pt import APGDAttack_targeted + self.apgd_targeted = APGDAttack_targeted(self.model, n_restarts=1, n_iter=100, verbose=False, + eps=self.epsilon, norm=self.norm, eot_iter=1, rho=.75, seed=self.seed, device=self.device) + + else: + from autopgd_tf import APGDAttack + self.apgd = APGDAttack(self.model, n_restarts=5, n_iter=100, verbose=False, + eps=self.epsilon, norm=self.norm, eot_iter=1, rho=.75, seed=self.seed, device=self.device) + + from fab_tf import FABAttack + self.fab = FABAttack(self.model, n_restarts=5, n_iter=100, eps=self.epsilon, seed=self.seed, + norm=self.norm, verbose=False, device=self.device) + + from square import SquareAttack + self.square = SquareAttack(self.model.predict, p_init=.8, n_queries=5000, eps=self.epsilon, norm=self.norm, + n_restarts=1, seed=self.seed, verbose=False, device=self.device, resc_schedule=False) + + from autopgd_tf import APGDAttack_targeted + self.apgd_targeted = APGDAttack_targeted(self.model, n_restarts=1, n_iter=100, verbose=False, + eps=self.epsilon, norm=self.norm, eot_iter=1, rho=.75, seed=self.seed, device=self.device) + + if version in ['standard', 'plus', 'rand']: + self.set_version(version) + + def get_logits(self, x): + if not self.is_tf_model: + return self.model(x) + else: + return self.model.predict(x) + + def get_seed(self): + return time.time() if self.seed is None else self.seed + + def run_standard_evaluation(self, x_orig, y_orig, bs=250): + if self.verbose: + print('using {} version including {}'.format(self.version, + ', '.join(self.attacks_to_run))) + + with torch.no_grad(): + # calculate accuracy + n_batches = int(np.ceil(x_orig.shape[0] / bs)) + robust_flags = torch.zeros(x_orig.shape[0], dtype=torch.bool, device=x_orig.device) + for batch_idx in range(n_batches): + start_idx = batch_idx * bs + end_idx = min( (batch_idx + 1) * bs, x_orig.shape[0]) + + x = x_orig[start_idx:end_idx, :].clone().to(self.device) + y = y_orig[start_idx:end_idx].clone().to(self.device) + output = self.get_logits(x) + correct_batch = y.eq(output.max(dim=1)[1]) + robust_flags[start_idx:end_idx] = correct_batch.detach().to(robust_flags.device) + + robust_accuracy = torch.sum(robust_flags).item() / x_orig.shape[0] + + if self.verbose: + self.logger.log('initial accuracy: {:.2%}'.format(robust_accuracy)) + + x_adv = x_orig.clone().detach() + startt = time.time() + for attack in self.attacks_to_run: + # item() is super important as pytorch int division uses floor rounding + num_robust = torch.sum(robust_flags).item() + + if num_robust == 0: + break + + n_batches = int(np.ceil(num_robust / bs)) + + robust_lin_idcs = torch.nonzero(robust_flags, as_tuple=False) + if num_robust > 1: + robust_lin_idcs.squeeze_() + + for batch_idx in range(n_batches): + start_idx = batch_idx * bs + end_idx = min((batch_idx + 1) * bs, num_robust) + + batch_datapoint_idcs = robust_lin_idcs[start_idx:end_idx] + if len(batch_datapoint_idcs.shape) > 1: + batch_datapoint_idcs.squeeze_(-1) + x = x_orig[batch_datapoint_idcs, :].clone().to(self.device) + y = y_orig[batch_datapoint_idcs].clone().to(self.device) + + # make sure that x is a 4d tensor even if there is only a single datapoint left + if len(x.shape) == 3: + x.unsqueeze_(dim=0) + + # run attack + if attack == 'apgd-ce': + # apgd on cross-entropy loss + self.apgd.loss = 'ce' + self.apgd.seed = self.get_seed() + _, adv_curr = self.apgd.perturb(x, y, cheap=True) + + elif attack == 'apgd-dlr': + # apgd on dlr loss + self.apgd.loss = 'dlr' + self.apgd.seed = self.get_seed() + _, adv_curr = self.apgd.perturb(x, y, cheap=True) + + elif attack == 'fab': + # fab + self.fab.targeted = False + self.fab.seed = self.get_seed() + adv_curr = self.fab.perturb(x, y) + + elif attack == 'square': + # square + self.square.seed = self.get_seed() + adv_curr = self.square.perturb(x, y) + + elif attack == 'apgd-t': + # targeted apgd + self.apgd_targeted.seed = self.get_seed() + _, adv_curr = self.apgd_targeted.perturb(x, y, cheap=True) + + elif attack == 'fab-t': + # fab targeted + self.fab.targeted = True + self.fab.n_restarts = 1 + self.fab.seed = self.get_seed() + adv_curr = self.fab.perturb(x, y) + + else: + raise ValueError('Attack not supported') + + output = self.get_logits(adv_curr) + false_batch = ~y.eq(output.max(dim=1)[1]).to(robust_flags.device) + non_robust_lin_idcs = batch_datapoint_idcs[false_batch] + robust_flags[non_robust_lin_idcs] = False + + x_adv[non_robust_lin_idcs] = adv_curr[false_batch].detach().to(x_adv.device) + + if self.verbose: + num_non_robust_batch = torch.sum(false_batch) + self.logger.log('{} - {}/{} - {} out of {} successfully perturbed'.format( + attack, batch_idx + 1, n_batches, num_non_robust_batch, x.shape[0])) + + robust_accuracy = torch.sum(robust_flags).item() / x_orig.shape[0] + if self.verbose: + self.logger.log('robust accuracy after {}: {:.2%} (total time {:.1f} s)'.format( + attack.upper(), robust_accuracy, time.time() - startt)) + + # final check + if self.verbose: + if self.norm == 'Linf': + res = (x_adv - x_orig).abs().view(x_orig.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_adv - x_orig) ** 2).view(x_orig.shape[0], -1).sum(-1).sqrt() + self.logger.log('max {} perturbation: {:.5f}, nan in tensor: {}, max: {:.5f}, min: {:.5f}'.format( + self.norm, res.max(), (x_adv != x_adv).sum(), x_adv.max(), x_adv.min())) + self.logger.log('robust accuracy: {:.2%}'.format(robust_accuracy)) + + return x_adv + + def clean_accuracy(self, x_orig, y_orig, bs=250): + n_batches = x_orig.shape[0] // bs + acc = 0. + for counter in range(n_batches): + x = x_orig[counter * bs:min((counter + 1) * bs, x_orig.shape[0])].clone().to(self.device) + y = y_orig[counter * bs:min((counter + 1) * bs, x_orig.shape[0])].clone().to(self.device) + output = self.get_logits(x) + acc += (output.max(1)[1] == y).float().sum() + + if self.verbose: + print('clean accuracy: {:.2%}'.format(acc / x_orig.shape[0])) + + return acc.item() / x_orig.shape[0] + + def run_standard_evaluation_individual(self, x_orig, y_orig, bs=250): + if self.verbose: + print('using {} version including {}'.format(self.version, + ', '.join(self.attacks_to_run))) + + l_attacks = self.attacks_to_run + adv = {} + verbose_indiv = self.verbose + self.verbose = False + + for c in l_attacks: + startt = time.time() + self.attacks_to_run = [c] + adv[c] = self.run_standard_evaluation(x_orig, y_orig, bs=bs) + if verbose_indiv: + acc_indiv = self.clean_accuracy(adv[c], y_orig, bs=bs) + space = '\t \t' if c == 'fab' else '\t' + self.logger.log('robust accuracy by {} {} {:.2%} \t (time attack: {:.1f} s)'.format( + c.upper(), space, acc_indiv, time.time() - startt)) + + return adv + + def set_version(self, version='standard'): + if version == 'standard': + self.attacks_to_run = ['apgd-ce', 'apgd-t', 'fab-t', 'square'] + self.apgd.n_restarts = 1 + self.fab.n_restarts = 1 + self.apgd_targeted.n_restarts = 1 + self.fab.n_target_classes = 9 + self.apgd_targeted.n_target_classes = 9 + self.square.n_queries = 5000 + + elif version == 'plus': + self.attacks_to_run = ['apgd-ce', 'apgd-dlr', 'fab', 'square', 'apgd-t', 'fab-t'] + self.apgd.n_restarts = 5 + self.fab.n_restarts = 5 + self.apgd_targeted.n_restarts = 1 + self.fab.n_target_classes = 9 + self.apgd_targeted.n_target_classes = 9 + self.square.n_queries = 5000 + + elif version == 'rand': + self.attacks_to_run = ['apgd-ce', 'apgd-dlr'] + self.apgd.n_restarts = 1 + self.apgd.eot_iter = 20 + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/autopgd_pt.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/autopgd_pt.py new file mode 100644 index 0000000..8187954 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/autopgd_pt.py @@ -0,0 +1,431 @@ +import numpy as np +import time +import torch +#import scipy.io + +#import numpy.linalg as nl + +# +import os +import sys + +import torch.nn as nn +import torch.nn.functional as F + + +class APGDAttack(): + def __init__(self, model, n_iter=100, norm='Linf', n_restarts=1, eps=None, + seed=0, loss='ce', eot_iter=1, rho=.75, verbose=False, + device='cuda'): + self.model = model + self.n_iter = n_iter + self.eps = eps + self.norm = norm + self.n_restarts = n_restarts + self.seed = seed + self.loss = loss + self.eot_iter = eot_iter + self.thr_decr = rho + self.verbose = verbose + self.device = device + + def check_oscillation(self, x, j, k, y5, k3=0.75): + t = np.zeros(x.shape[1]) + for counter5 in range(k): + t += x[j - counter5] > x[j - counter5 - 1] + + return t <= k*k3*np.ones(t.shape) + + def check_shape(self, x): + return x if len(x.shape) > 0 else np.expand_dims(x, 0) + + def dlr_loss(self, x, y): + x_sorted, ind_sorted = x.sort(dim=1) + ind = (ind_sorted[:, -1] == y).float() + + return -(x[np.arange(x.shape[0]), y] - x_sorted[:, -2] * ind - x_sorted[:, -1] * (1. - ind)) / (x_sorted[:, -1] - x_sorted[:, -3] + 1e-12) + + def attack_single_run(self, x_in, y_in): + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + self.n_iter_2, self.n_iter_min, self.size_decr = max(int(0.22 * self.n_iter), 1), max(int(0.06 * self.n_iter), 1), max(int(0.03 * self.n_iter), 1) + if self.verbose: + print('parameters: ', self.n_iter, self.n_iter_2, self.n_iter_min, self.size_decr) + + if self.norm == 'Linf': + t = 2 * torch.rand(x.shape).to(self.device).detach() - 1 + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / (t.reshape([t.shape[0], -1]).abs().max(dim=1, keepdim=True)[0].reshape([-1, 1, 1, 1])) + elif self.norm == 'L2': + t = torch.randn(x.shape).to(self.device).detach() + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / ((t ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv = x_adv.clamp(0., 1.) + x_best = x_adv.clone() + x_best_adv = x_adv.clone() + loss_steps = torch.zeros([self.n_iter, x.shape[0]]) + loss_best_steps = torch.zeros([self.n_iter + 1, x.shape[0]]) + acc_steps = torch.zeros_like(loss_best_steps) + + if self.loss == 'ce': + criterion_indiv = nn.CrossEntropyLoss(reduce=False, reduction='none') + elif self.loss == 'dlr': + criterion_indiv = self.dlr_loss + else: + raise ValueError('unknowkn loss') + + x_adv.requires_grad_() + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + with torch.enable_grad(): + logits = self.model(x_adv) # 1 forward pass (eot_iter = 1) + loss_indiv = criterion_indiv(logits, y) + loss = loss_indiv.sum() + + grad += torch.autograd.grad(loss, [x_adv])[0].detach() # 1 backward pass (eot_iter = 1) + + grad /= float(self.eot_iter) + grad_best = grad.clone() + + acc = logits.detach().max(1)[1] == y + acc_steps[0] = acc + 0 + loss_best = loss_indiv.detach().clone() + + step_size = self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * torch.Tensor([2.0]).to(self.device).detach().reshape([1, 1, 1, 1]) + x_adv_old = x_adv.clone() + counter = 0 + k = self.n_iter_2 + 0 + u = np.arange(x.shape[0]) + counter3 = 0 + + loss_best_last_check = loss_best.clone() + reduced_last_check = np.zeros(loss_best.shape) == np.zeros(loss_best.shape) + n_reduced = 0 + + for i in range(self.n_iter): + ### gradient step + with torch.no_grad(): + x_adv = x_adv.detach() + grad2 = x_adv - x_adv_old + x_adv_old = x_adv.clone() + + a = 0.75 if i > 0 else 1.0 + + if self.norm == 'Linf': + x_adv_1 = x_adv + step_size * torch.sign(grad) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv_1, x - self.eps), x + self.eps), 0.0, 1.0) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a), x - self.eps), x + self.eps), 0.0, 1.0) + + elif self.norm == 'L2': + x_adv_1 = x_adv + step_size * grad / ((grad ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt()), 0.0, 1.0) + x_adv_1 = x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12), 0.0, 1.0) + + x_adv = x_adv_1 + 0. + + ### get gradient + x_adv.requires_grad_() + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + with torch.enable_grad(): + logits = self.model(x_adv) # 1 forward pass (eot_iter = 1) + loss_indiv = criterion_indiv(logits, y) + loss = loss_indiv.sum() + + grad += torch.autograd.grad(loss, [x_adv])[0].detach() # 1 backward pass (eot_iter = 1) + + + grad /= float(self.eot_iter) + + pred = logits.detach().max(1)[1] == y + acc = torch.min(acc, pred) + acc_steps[i + 1] = acc + 0 + x_best_adv[(pred == 0).nonzero().squeeze()] = x_adv[(pred == 0).nonzero().squeeze()] + 0. + if self.verbose: + print('iteration: {} - Best loss: {:.6f}'.format(i, loss_best.sum())) + + ### check step size + with torch.no_grad(): + y1 = loss_indiv.detach().clone() + loss_steps[i] = y1.cpu() + 0 + ind = (y1 > loss_best).nonzero().squeeze() + x_best[ind] = x_adv[ind].clone() + grad_best[ind] = grad[ind].clone() + loss_best[ind] = y1[ind] + 0 + loss_best_steps[i + 1] = loss_best + 0 + + counter3 += 1 + + if counter3 == k: + fl_oscillation = self.check_oscillation(loss_steps.detach().cpu().numpy(), i, k, loss_best.detach().cpu().numpy(), k3=self.thr_decr) + fl_reduce_no_impr = (~reduced_last_check) * (loss_best_last_check.cpu().numpy() >= loss_best.cpu().numpy()) + fl_oscillation = ~(~fl_oscillation * ~fl_reduce_no_impr) + reduced_last_check = np.copy(fl_oscillation) + loss_best_last_check = loss_best.clone() + + if np.sum(fl_oscillation) > 0: + step_size[u[fl_oscillation]] /= 2.0 + n_reduced = fl_oscillation.astype(float).sum() + + fl_oscillation = np.where(fl_oscillation) + + x_adv[fl_oscillation] = x_best[fl_oscillation].clone() + grad[fl_oscillation] = grad_best[fl_oscillation].clone() + + counter3 = 0 + k = np.maximum(k - self.size_decr, self.n_iter_min) + + return x_best, acc, loss_best, x_best_adv + + def perturb(self, x_in, y_in, best_loss=False, cheap=True): + assert self.norm in ['Linf', 'L2'] + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + adv = x.clone() + acc = self.model(x).max(1)[1] == y + loss = -1e10 * torch.ones_like(acc).float() + if self.verbose: + print('-------------------------- running {}-attack with epsilon {:.4f} --------------------------'.format(self.norm, self.eps)) + print('initial accuracy: {:.2%}'.format(acc.float().mean())) + startt = time.time() + + if not best_loss: + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not cheap: + raise ValueError('not implemented yet') + + else: + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + best_curr, acc_curr, loss_curr, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + ind_curr = (acc_curr == 0).nonzero().squeeze() + # + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - robust accuracy: {:.2%} - cum. time: {:.1f} s'.format( + counter, acc.float().mean(), time.time() - startt)) + + return acc, adv + + else: + adv_best = x.detach().clone() + loss_best = torch.ones([x.shape[0]]).to(self.device) * (-float('inf')) + for counter in range(self.n_restarts): + best_curr, _, loss_curr, _ = self.attack_single_run(x, y) + ind_curr = (loss_curr > loss_best).nonzero().squeeze() + adv_best[ind_curr] = best_curr[ind_curr] + 0. + loss_best[ind_curr] = loss_curr[ind_curr] + 0. + + if self.verbose: + print('restart {} - loss: {:.5f}'.format(counter, loss_best.sum())) + + return loss_best, adv_best + +class APGDAttack_targeted(): + def __init__(self, model, n_iter=100, norm='Linf', n_restarts=1, eps=None, + seed=0, eot_iter=1, rho=.75, verbose=False, device='cuda', + n_target_classes=9): + self.model = model + self.n_iter = n_iter + self.eps = eps + self.norm = norm + self.n_restarts = n_restarts + self.seed = seed + self.eot_iter = eot_iter + self.thr_decr = rho + self.verbose = verbose + self.target_class = None + self.device = device + self.n_target_classes = n_target_classes + + def check_oscillation(self, x, j, k, y5, k3=0.5): + t = np.zeros(x.shape[1]) + for counter5 in range(k): + t += x[j - counter5] > x[j - counter5 - 1] + + return t <= k*k3*np.ones(t.shape) + + def check_shape(self, x): + return x if len(x.shape) > 0 else np.expand_dims(x, 0) + + def dlr_loss_targeted(self, x, y, y_target): + x_sorted, ind_sorted = x.sort(dim=1) + + return -(x[np.arange(x.shape[0]), y] - x[np.arange(x.shape[0]), y_target]) / (x_sorted[:, -1] - .5 * x_sorted[:, -3] - .5 * x_sorted[:, -4] + 1e-12) + + def attack_single_run(self, x_in, y_in): + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + self.n_iter_2, self.n_iter_min, self.size_decr = max(int(0.22 * self.n_iter), 1), max(int(0.06 * self.n_iter), 1), max(int(0.03 * self.n_iter), 1) + if self.verbose: + print('parameters: ', self.n_iter, self.n_iter_2, self.n_iter_min, self.size_decr) + + if self.norm == 'Linf': + t = 2 * torch.rand(x.shape).to(self.device).detach() - 1 + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / (t.reshape([t.shape[0], -1]).abs().max(dim=1, keepdim=True)[0].reshape([-1, 1, 1, 1])) + elif self.norm == 'L2': + t = torch.randn(x.shape).to(self.device).detach() + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / ((t ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv = x_adv.clamp(0., 1.) + x_best = x_adv.clone() + x_best_adv = x_adv.clone() + loss_steps = torch.zeros([self.n_iter, x.shape[0]]) + loss_best_steps = torch.zeros([self.n_iter + 1, x.shape[0]]) + acc_steps = torch.zeros_like(loss_best_steps) + + output = self.model(x) + y_target = output.sort(dim=1)[1][:, -self.target_class] + + x_adv.requires_grad_() + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + with torch.enable_grad(): + logits = self.model(x_adv) # 1 forward pass (eot_iter = 1) + loss_indiv = self.dlr_loss_targeted(logits, y, y_target) + loss = loss_indiv.sum() + + grad += torch.autograd.grad(loss, [x_adv])[0].detach() # 1 backward pass (eot_iter = 1) + + grad /= float(self.eot_iter) + grad_best = grad.clone() + + acc = logits.detach().max(1)[1] == y + acc_steps[0] = acc + 0 + loss_best = loss_indiv.detach().clone() + + step_size = self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * torch.Tensor([2.0]).to(self.device).detach().reshape([1, 1, 1, 1]) + x_adv_old = x_adv.clone() + counter = 0 + k = self.n_iter_2 + 0 + u = np.arange(x.shape[0]) + counter3 = 0 + + loss_best_last_check = loss_best.clone() + reduced_last_check = np.zeros(loss_best.shape) == np.zeros(loss_best.shape) + n_reduced = 0 + + for i in range(self.n_iter): + ### gradient step + with torch.no_grad(): + x_adv = x_adv.detach() + grad2 = x_adv - x_adv_old + x_adv_old = x_adv.clone() + + a = 0.75 if i > 0 else 1.0 + + if self.norm == 'Linf': + x_adv_1 = x_adv + step_size * torch.sign(grad) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv_1, x - self.eps), x + self.eps), 0.0, 1.0) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv + (x_adv_1 - x_adv)*a + grad2*(1 - a), x - self.eps), x + self.eps), 0.0, 1.0) + + elif self.norm == 'L2': + x_adv_1 = x_adv + step_size[0] * grad / ((grad ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt()), 0.0, 1.0) + x_adv_1 = x_adv + (x_adv_1 - x_adv)*a + grad2*(1 - a) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12), 0.0, 1.0) + + x_adv = x_adv_1 + 0. + + ### get gradient + x_adv.requires_grad_() + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + with torch.enable_grad(): + logits = self.model(x_adv) # 1 forward pass (eot_iter = 1) + loss_indiv = self.dlr_loss_targeted(logits, y, y_target) + loss = loss_indiv.sum() + + grad += torch.autograd.grad(loss, [x_adv])[0].detach() # 1 backward pass (eot_iter = 1) + + grad /= float(self.eot_iter) + + pred = logits.detach().max(1)[1] == y + acc = torch.min(acc, pred) + acc_steps[i + 1] = acc + 0 + x_best_adv[(pred == 0).nonzero().squeeze()] = x_adv[(pred == 0).nonzero().squeeze()] + 0. + if self.verbose: + print('iteration: {} - Best loss: {:.6f}'.format(i, loss_best.sum())) + + ### check step size + with torch.no_grad(): + y1 = loss_indiv.detach().clone() + loss_steps[i] = y1.cpu() + 0 + ind = (y1 > loss_best).nonzero().squeeze() + x_best[ind] = x_adv[ind].clone() + grad_best[ind] = grad[ind].clone() + loss_best[ind] = y1[ind] + 0 + loss_best_steps[i + 1] = loss_best + 0 + + counter3 += 1 + + if counter3 == k: + fl_oscillation = self.check_oscillation(loss_steps.detach().cpu().numpy(), i, k, loss_best.detach().cpu().numpy(), k3=self.thr_decr) + fl_reduce_no_impr = (~reduced_last_check) * (loss_best_last_check.cpu().numpy() >= loss_best.cpu().numpy()) + fl_oscillation = ~(~fl_oscillation * ~fl_reduce_no_impr) + reduced_last_check = np.copy(fl_oscillation) + loss_best_last_check = loss_best.clone() + + if np.sum(fl_oscillation) > 0: + step_size[u[fl_oscillation]] /= 2.0 + n_reduced = fl_oscillation.astype(float).sum() + + fl_oscillation = np.where(fl_oscillation) + + x_adv[fl_oscillation] = x_best[fl_oscillation].clone() + grad[fl_oscillation] = grad_best[fl_oscillation].clone() + + counter3 = 0 + k = np.maximum(k - self.size_decr, self.n_iter_min) + + return x_best, acc, loss_best, x_best_adv + + def perturb(self, x_in, y_in, best_loss=False, cheap=True): + assert self.norm in ['Linf', 'L2'] + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + adv = x.clone() + acc = self.model(x).max(1)[1] == y + loss = -1e10 * torch.ones_like(acc).float() + if self.verbose: + print('-------------------------- running {}-attack with epsilon {:.4f} --------------------------'.format(self.norm, self.eps)) + print('initial accuracy: {:.2%}'.format(acc.float().mean())) + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not cheap: + raise ValueError('not implemented yet') + + else: + for target_class in range(2, self.n_target_classes + 2): + self.target_class = target_class + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + best_curr, acc_curr, loss_curr, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + ind_curr = (acc_curr == 0).nonzero().squeeze() + # + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, self.target_class, acc.float().mean(), self.eps, time.time() - startt)) + + return acc, adv + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/autopgd_tf.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/autopgd_tf.py new file mode 100644 index 0000000..728c836 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/autopgd_tf.py @@ -0,0 +1,389 @@ +import numpy as np +import time +import torch +#import scipy.io + +#import numpy.linalg as nl + +# +import os +import sys + +import torch.nn as nn +import torch.nn.functional as F + + +class APGDAttack(): + def __init__(self, model, n_iter=100, norm='Linf', n_restarts=1, eps=None, + seed=0, loss='ce', eot_iter=1, rho=.75, verbose=False, + device='cuda'): + self.model = model + self.n_iter = n_iter + self.eps = eps + self.norm = norm + self.n_restarts = n_restarts + self.seed = seed + self.loss = loss + self.eot_iter = eot_iter + self.thr_decr = rho + self.verbose = verbose + self.device = device + + def check_oscillation(self, x, j, k, y5, k3=0.75): + t = np.zeros(x.shape[1]) + for counter5 in range(k): + t += x[j - counter5] > x[j - counter5 - 1] + + return t <= k*k3*np.ones(t.shape) + + def check_shape(self, x): + return x if len(x.shape) > 0 else np.expand_dims(x, 0) + + def attack_single_run(self, x_in, y_in): + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + self.n_iter_2, self.n_iter_min, self.size_decr = max(int(0.22 * self.n_iter), 1), max(int(0.06 * self.n_iter), 1), max(int(0.03 * self.n_iter), 1) + if self.verbose: + print('parameters: ', self.n_iter, self.n_iter_2, self.n_iter_min, self.size_decr) + + if self.norm == 'Linf': + t = 2 * torch.rand(x.shape).to(self.device).detach() - 1 + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / (t.reshape([t.shape[0], -1]).abs().max(dim=1, keepdim=True)[0].reshape([-1, 1, 1, 1])) + elif self.norm == 'L2': + t = torch.randn(x.shape).to(self.device).detach() + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / ((t ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv = x_adv.clamp(0., 1.) + x_best = x_adv.clone() + x_best_adv = x_adv.clone() + loss_steps = torch.zeros([self.n_iter, x.shape[0]]) + loss_best_steps = torch.zeros([self.n_iter + 1, x.shape[0]]) + acc_steps = torch.zeros_like(loss_best_steps) + + if self.loss == 'ce': + criterion_indiv = self.model.get_logits_loss_grad_xent + elif self.loss == 'dlr': + criterion_indiv = self.model.get_logits_loss_grad_dlr + else: + raise ValueError('unknowkn loss') + + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + logits, loss_indiv, grad_curr = criterion_indiv(x_adv, y) + grad += grad_curr + + grad /= float(self.eot_iter) + grad_best = grad.clone() + + acc = logits.detach().max(1)[1] == y + acc_steps[0] = acc + 0 + loss_best = loss_indiv.detach().clone() + + step_size = self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * torch.Tensor([2.0]).to(self.device).detach().reshape([1, 1, 1, 1]) + x_adv_old = x_adv.clone() + counter = 0 + k = self.n_iter_2 + 0 + u = np.arange(x.shape[0]) + counter3 = 0 + + loss_best_last_check = loss_best.clone() + reduced_last_check = np.zeros(loss_best.shape) == np.zeros(loss_best.shape) + n_reduced = 0 + + for i in range(self.n_iter): + ### gradient step + with torch.no_grad(): + x_adv = x_adv.detach() + grad2 = x_adv - x_adv_old + x_adv_old = x_adv.clone() + + a = 0.75 if i > 0 else 1.0 + + if self.norm == 'Linf': + x_adv_1 = x_adv + step_size * torch.sign(grad) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv_1, x - self.eps), x + self.eps), 0.0, 1.0) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a), x - self.eps), x + self.eps), 0.0, 1.0) + + elif self.norm == 'L2': + x_adv_1 = x_adv + step_size * grad / ((grad ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt()), 0.0, 1.0) + x_adv_1 = x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12), 0.0, 1.0) + + x_adv = x_adv_1 + 0. + + ### get gradient + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + logits, loss_indiv, grad_curr = criterion_indiv(x_adv, y) + grad += grad_curr + + grad /= float(self.eot_iter) + + pred = logits.detach().max(1)[1] == y + acc = torch.min(acc, pred) + acc_steps[i + 1] = acc + 0 + x_best_adv[(pred == 0).nonzero().squeeze()] = x_adv[(pred == 0).nonzero().squeeze()] + 0. + if self.verbose: + print('iteration: {} - Best loss: {:.6f}'.format(i, loss_best.sum())) + + ### check step size + with torch.no_grad(): + y1 = loss_indiv.detach().clone() + loss_steps[i] = y1.cpu() + 0 + ind = (y1 > loss_best).nonzero().squeeze() + x_best[ind] = x_adv[ind].clone() + grad_best[ind] = grad[ind].clone() + loss_best[ind] = y1[ind] + 0 + loss_best_steps[i + 1] = loss_best + 0 + + counter3 += 1 + + if counter3 == k: + fl_oscillation = self.check_oscillation(loss_steps.detach().cpu().numpy(), i, k, loss_best.detach().cpu().numpy(), k3=self.thr_decr) + fl_reduce_no_impr = (~reduced_last_check) * (loss_best_last_check.cpu().numpy() >= loss_best.cpu().numpy()) + fl_oscillation = ~(~fl_oscillation * ~fl_reduce_no_impr) + reduced_last_check = np.copy(fl_oscillation) + loss_best_last_check = loss_best.clone() + + if np.sum(fl_oscillation) > 0: + step_size[u[fl_oscillation]] /= 2.0 + n_reduced = fl_oscillation.astype(float).sum() + + fl_oscillation = np.where(fl_oscillation) + + x_adv[fl_oscillation] = x_best[fl_oscillation].clone() + grad[fl_oscillation] = grad_best[fl_oscillation].clone() + + counter3 = 0 + k = np.maximum(k - self.size_decr, self.n_iter_min) + + return x_best, acc, loss_best, x_best_adv + + def perturb(self, x_in, y_in, cheap=True): + assert self.norm in ['Linf', 'L2'] + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + adv = x.clone() + acc = self.model.predict(x).max(1)[1] == y + loss = -1e10 * torch.ones_like(acc).float() + if self.verbose: + print('-------------------------- running {}-attack with epsilon {:.4f} --------------------------'.format(self.norm, self.eps)) + print('initial accuracy: {:.2%}'.format(acc.float().mean())) + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not cheap: + raise ValueError('not implemented yet') + + else: + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + best_curr, acc_curr, loss_curr, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + ind_curr = (acc_curr == 0).nonzero().squeeze() + # + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - robust accuracy: {:.2%} - cum. time: {:.1f} s'.format( + counter, acc.float().mean(), time.time() - startt)) + + return acc, adv + +class APGDAttack_targeted(): + def __init__(self, model, n_iter=100, norm='Linf', n_restarts=1, eps=None, + seed=0, eot_iter=1, rho=.75, verbose=False, device='cuda', + n_target_classes=9): + self.model = model + self.n_iter = n_iter + self.eps = eps + self.norm = norm + self.n_restarts = n_restarts + self.seed = seed + self.eot_iter = eot_iter + self.thr_decr = rho + self.verbose = verbose + self.target_class = None + self.device = device + self.n_target_classes = n_target_classes + + def check_oscillation(self, x, j, k, y5, k3=0.5): + t = np.zeros(x.shape[1]) + for counter5 in range(k): + t += x[j - counter5] > x[j - counter5 - 1] + + return t <= k*k3*np.ones(t.shape) + + def check_shape(self, x): + return x if len(x.shape) > 0 else np.expand_dims(x, 0) + + def custom_loss_targeted(self, x, y, y_target): + x_sorted, ind_sorted = x.sort(dim=1) + + return -(x[np.arange(x.shape[0]), y] - x[np.arange(x.shape[0]), y_target]) / (x_sorted[:, -1] - .5 * x_sorted[:, -3] - .5 * x_sorted[:, -4] + 1e-12) + + def attack_single_run(self, x_in, y_in): + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + self.n_iter_2, self.n_iter_min, self.size_decr = max(int(0.22 * self.n_iter), 1), max(int(0.06 * self.n_iter), 1), max(int(0.03 * self.n_iter), 1) + if self.verbose: + print('parameters: ', self.n_iter, self.n_iter_2, self.n_iter_min, self.size_decr) + + if self.norm == 'Linf': + t = 2 * torch.rand(x.shape).to(self.device).detach() - 1 + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / (t.reshape([t.shape[0], -1]).abs().max(dim=1, keepdim=True)[0].reshape([-1, 1, 1, 1])) + elif self.norm == 'L2': + t = torch.randn(x.shape).to(self.device).detach() + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / ((t ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv = x_adv.clamp(0., 1.) + x_best = x_adv.clone() + x_best_adv = x_adv.clone() + loss_steps = torch.zeros([self.n_iter, x.shape[0]]) + loss_best_steps = torch.zeros([self.n_iter + 1, x.shape[0]]) + acc_steps = torch.zeros_like(loss_best_steps) + + output = self.model.predict(x) + y_target = output.sort(dim=1)[1][:, -self.target_class] + + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + logits, loss_indiv, grad_curr = self.model.get_logits_loss_grad_target(x_adv, y, y_target) + grad += grad_curr + + grad /= float(self.eot_iter) + grad_best = grad.clone() + + acc = logits.detach().max(1)[1] == y + acc_steps[0] = acc + 0 + loss_best = loss_indiv.detach().clone() + + step_size = self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * torch.Tensor([2.0]).to(self.device).detach().reshape([1, 1, 1, 1]) + x_adv_old = x_adv.clone() + counter = 0 + k = self.n_iter_2 + 0 + u = np.arange(x.shape[0]) + counter3 = 0 + + loss_best_last_check = loss_best.clone() + reduced_last_check = np.zeros(loss_best.shape) == np.zeros(loss_best.shape) + n_reduced = 0 + + for i in range(self.n_iter): + ### gradient step + with torch.no_grad(): + x_adv = x_adv.detach() + grad2 = x_adv - x_adv_old + x_adv_old = x_adv.clone() + + a = 0.75 if i > 0 else 1.0 + + if self.norm == 'Linf': + x_adv_1 = x_adv + step_size * torch.sign(grad) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv_1, x - self.eps), x + self.eps), 0.0, 1.0) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv + (x_adv_1 - x_adv)*a + grad2*(1 - a), x - self.eps), x + self.eps), 0.0, 1.0) + + elif self.norm == 'L2': + x_adv_1 = x_adv + step_size[0] * grad / ((grad ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt()), 0.0, 1.0) + x_adv_1 = x_adv + (x_adv_1 - x_adv)*a + grad2*(1 - a) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12), 0.0, 1.0) + + x_adv = x_adv_1 + 0. + + ### get gradient + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + logits, loss_indiv, grad_curr = self.model.get_logits_loss_grad_target(x_adv, y, y_target) + grad += grad_curr + + grad /= float(self.eot_iter) + + pred = logits.detach().max(1)[1] == y + acc = torch.min(acc, pred) + acc_steps[i + 1] = acc + 0 + x_best_adv[(pred == 0).nonzero().squeeze()] = x_adv[(pred == 0).nonzero().squeeze()] + 0. + if self.verbose: + print('iteration: {} - Best loss: {:.6f}'.format(i, loss_best.sum())) + + ### check step size + with torch.no_grad(): + y1 = loss_indiv.detach().clone() + loss_steps[i] = y1.cpu() + 0 + ind = (y1 > loss_best).nonzero().squeeze() + x_best[ind] = x_adv[ind].clone() + grad_best[ind] = grad[ind].clone() + loss_best[ind] = y1[ind] + 0 + loss_best_steps[i + 1] = loss_best + 0 + + counter3 += 1 + + if counter3 == k: + fl_oscillation = self.check_oscillation(loss_steps.detach().cpu().numpy(), i, k, loss_best.detach().cpu().numpy(), k3=self.thr_decr) + fl_reduce_no_impr = (~reduced_last_check) * (loss_best_last_check.cpu().numpy() >= loss_best.cpu().numpy()) + fl_oscillation = ~(~fl_oscillation * ~fl_reduce_no_impr) + reduced_last_check = np.copy(fl_oscillation) + loss_best_last_check = loss_best.clone() + + if np.sum(fl_oscillation) > 0: + step_size[u[fl_oscillation]] /= 2.0 + n_reduced = fl_oscillation.astype(float).sum() + + fl_oscillation = np.where(fl_oscillation) + + x_adv[fl_oscillation] = x_best[fl_oscillation].clone() + grad[fl_oscillation] = grad_best[fl_oscillation].clone() + + counter3 = 0 + k = np.maximum(k - self.size_decr, self.n_iter_min) + + return x_best, acc, loss_best, x_best_adv + + def perturb(self, x_in, y_in, best_loss=False, cheap=True): + assert self.norm in ['Linf', 'L2'] + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + adv = x.clone() + acc = self.model.predict(x).max(1)[1] == y + loss = -1e10 * torch.ones_like(acc).float() + if self.verbose: + print('-------------------------- running {}-attack with epsilon {:.4f} --------------------------'.format(self.norm, self.eps)) + print('initial accuracy: {:.2%}'.format(acc.float().mean())) + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not cheap: + raise ValueError('not implemented yet') + + else: + for target_class in range(2, self.n_target_classes + 2): + self.target_class = target_class + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + best_curr, acc_curr, loss_curr, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + ind_curr = (acc_curr == 0).nonzero().squeeze() + # + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, self.target_class, acc.float().mean(), self.eps, time.time() - startt)) + + return acc, adv + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/eval.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/eval.py new file mode 100644 index 0000000..d1802b1 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/eval.py @@ -0,0 +1,88 @@ +import os +import argparse +import torch +import torch.nn as nn +import torchvision.datasets as datasets +import torch.utils.data as data +import torchvision.transforms as transforms +from wideresnet import * + +import sys +sys.path.insert(0, '..') + +from resnet import * + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='~/dataset/cifar-10') + parser.add_argument('--norm', type=str, default='Linf') + parser.add_argument('--epsilon', type=float, default=8./255.) + parser.add_argument('--model', type=str, default='./model_test.pt') + parser.add_argument('--n_ex', type=int, default=10000) + parser.add_argument('--individual', action='store_true') + parser.add_argument('--save_dir', type=str, default='./results') + parser.add_argument('--batch_size', type=int, default=500) + parser.add_argument('--log_path', type=str, default='./log_file.txt') + parser.add_argument('--version', type=str, default='standard') + + args = parser.parse_args() + + # load model + + model = WideResNet().cuda() + model = nn.DataParallel(model) + ckpt = torch.load(args.model) + model.load_state_dict(ckpt['state_dict']) + + model.cuda() + model.eval() + ''' + model = ResNet18() + ckpt = torch.load(args.model) + model.load_state_dict(ckpt) + model.cuda() + model.eval() + ''' + # load data + transform_list = [transforms.ToTensor()] + transform_chain = transforms.Compose(transform_list) + item = datasets.CIFAR10(root=args.data_dir, train=False, transform=transform_chain, download=True) + test_loader = data.DataLoader(item, batch_size=1000, shuffle=False, num_workers=0) + + # create save dir + if not os.path.exists(args.save_dir): + os.makedirs(args.save_dir) + + # load attack + from autoattack import AutoAttack + adversary = AutoAttack(model, norm=args.norm, eps=args.epsilon, log_path=args.log_path, + version=args.version) + + l = [x for (x, y) in test_loader] + x_test = torch.cat(l, 0) + l = [y for (x, y) in test_loader] + y_test = torch.cat(l, 0) + + # example of custom version + if args.version == 'custom': + adversary.attacks_to_run = ['apgd-ce', 'fab'] + adversary.apgd.n_restarts = 2 + adversary.fab.n_restarts = 2 + + # run attack and save images + with torch.no_grad(): + if not args.individual: + adv_complete = adversary.run_standard_evaluation(x_test[:args.n_ex], y_test[:args.n_ex], + bs=args.batch_size) + + torch.save({'adv_complete': adv_complete}, '{}/{}_{}_1_{}_eps_{:.5f}.pth'.format( + args.save_dir, 'aa', args.version, adv_complete.shape[0], args.epsilon)) + + else: + # individual version, each attack is run on all test points + adv_complete = adversary.run_standard_evaluation_individual(x_test[:args.n_ex], + y_test[:args.n_ex], bs=args.batch_size) + + torch.save(adv_complete, '{}/{}_{}_individual_1_{}_eps_{:.5f}_plus_{}_cheap.pth'.format( + args.save_dir, 'aa', args.version, args.n_ex, args.epsilon)) + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/eval_tf1.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/eval_tf1.py new file mode 100644 index 0000000..c2f5472 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/eval_tf1.py @@ -0,0 +1,150 @@ +#%% +from argparse import ArgumentParser + +import numpy as np +import tensorflow as tf + +import torch +import torch.nn as nn +import torchvision.datasets as datasets +import torch.utils.data as data +import torchvision.transforms as transforms + +import sys +#sys.path.insert(0,'..') + +from autoattack import AutoAttack, utils_tf +# + +#%% +class mnist_loader: + def __init__(self): + + self.n_class = 10 + self.dim_x = 28 + self.dim_y = 28 + self.dim_z = 1 + self.img_min = 0.0 + self.img_max = 1.0 + self.epsilon = 0.3 + + def download(self): + (trainX, trainY), (testX, testY) = tf.keras.datasets.mnist.load_data() + + trainX = trainX.astype(np.float32) + testX = testX.astype(np.float32) + + # ont-hot + trainY = tf.keras.utils.to_categorical(trainY, self.n_class) + testY = tf.keras.utils.to_categorical(testY , self.n_class) + + # get validation sets + training_size = 55000 + validX = trainX[training_size:,:] + validY = trainY[training_size:,:] + + trainX = trainX[:training_size,:] + trainY = trainY[:training_size,:] + + # expand dimesion + trainX = np.expand_dims(trainX, axis=3) + validX = np.expand_dims(validX, axis=3) + testX = np.expand_dims(testX , axis=3) + + return trainX, trainY, validX, validY, testX, testY + + def get_raw_data(self): + return self.download() + + def get_normalized_data(self): + trainX, trainY, validX, validY, testX, testY = self.get_raw_data() + trainX = trainX / 255.0 * (self.img_max - self.img_min) + self.img_min + validX = validX / 255.0 * (self.img_max - self.img_min) + self.img_min + testX = testX / 255.0 * (self.img_max - self.img_min) + self.img_min + trainY = trainY + validY = validY + testY = testY + return trainX, trainY, validX, validY, testX, testY + +#%% +def mnist_model(): + # declare variables + model_layers = [ tf.keras.layers.Input(shape=(28,28,1), name="model/input"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c1"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c2"), + tf.keras.layers.MaxPooling2D(pool_size=(2, 2), name="clf/p1"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c3"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c4"), + tf.keras.layers.MaxPooling2D(pool_size=(2, 2), name="clf/p2"), + tf.keras.layers.Flatten(name="clf/f1"), + tf.keras.layers.Dense(256, activation="relu", kernel_initializer='he_normal', name="clf/d1"), + tf.keras.layers.Dense(10 , activation=None , kernel_initializer='he_normal', name="clf/d2"), + tf.keras.layers.Activation('softmax', name="clf_output") + ] + + # clf_model + clf_model = tf.keras.Sequential() + for ii in model_layers: + clf_model.add(ii) + + clf_model.compile(loss='categorical_crossentropy', optimizer='Nadam', metrics=['accuracy']) + clf_model.summary() + + return clf_model + +#%% +def arg_parser(parser): + + parser.add_argument("--path" , dest ="path", type=str, default='./', help="path of tf.keras model's wieghts") + args, unknown = parser.parse_known_args() + if unknown: + msg = " ".join(unknown) + print('[Warning] Unrecognized arguments: {:s}'.format(msg) ) + + return args + +#%% +if __name__ == '__main__': + + # get arguments + parser = ArgumentParser() + args = arg_parser(parser) + + # MODEL PATH + MODEL_PATH = args.path + + # init tf/keras + tf.compat.v1.keras.backend.clear_session() + gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) + sess = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) + tf.compat.v1.keras.backend.set_session(sess) + tf.compat.v1.keras.backend.set_learning_phase(0) + + # load data + batch_size = 1000 + epsilon = mnist_loader().epsilon + _, _, _, _, testX, testY = mnist_loader().get_normalized_data() + + # convert to pytorch format + testY = np.argmax(testY, axis=1) + torch_testX = torch.from_numpy( np.transpose(testX, (0, 3, 1, 2)) ).float().cuda() + torch_testY = torch.from_numpy( testY ).float() + + # load model from saved weights + print('[INFO] MODEL_PATH: {:s}'.format(MODEL_PATH) ) + tf_model = mnist_model() + tf_model.load_weights(MODEL_PATH) + + # remove 'softmax layer' and put it into adapter + atk_model = tf.keras.models.Model(inputs=tf_model.input, outputs=tf_model.get_layer(index=-2).output) + atk_model.summary() + y_input = tf.placeholder(tf.int64, shape = [None]) + x_input = atk_model.input + logits = atk_model.output + model_adapted = utils_tf.ModelAdapter(logits, x_input, y_input, sess) + + # run attack + adversary = AutoAttack(model_adapted, norm='Linf', eps=epsilon, plus=False, is_tf_model=True) + x_adv = adversary.run_standard_evaluation(torch_testX, torch_testY, bs=batch_size) + np_x_adv = np.moveaxis(x_adv.cpu().numpy(), 1, 3) + np.save("./output/mnist_adv.npy", np_x_adv) diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/eval_tf2.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/eval_tf2.py new file mode 100644 index 0000000..189e2ca --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/eval_tf2.py @@ -0,0 +1,145 @@ +#%% +from argparse import ArgumentParser + +import numpy as np +import tensorflow as tf + +import torch +import torch.nn as nn +import torchvision.datasets as datasets +import torch.utils.data as data +import torchvision.transforms as transforms + +import sys +sys.path.insert(0, '..') + +from autoattack import AutoAttack, utils_tf2 + + +#%% +class mnist_loader: + def __init__(self): + + self.n_class = 10 + self.dim_x = 28 + self.dim_y = 28 + self.dim_z = 1 + self.img_min = 0.0 + self.img_max = 1.0 + self.epsilon = 0.3 + + def download(self): + (trainX, trainY), (testX, testY) = tf.keras.datasets.mnist.load_data() + + trainX = trainX.astype(np.float32) + testX = testX.astype(np.float32) + + # ont-hot + trainY = tf.keras.utils.to_categorical(trainY, self.n_class) + testY = tf.keras.utils.to_categorical(testY , self.n_class) + + # get validation sets + training_size = 55000 + validX = trainX[training_size:,:] + validY = trainY[training_size:,:] + + trainX = trainX[:training_size,:] + trainY = trainY[:training_size,:] + + # expand dimesion + trainX = np.expand_dims(trainX, axis=3) + validX = np.expand_dims(validX, axis=3) + testX = np.expand_dims(testX , axis=3) + + return trainX, trainY, validX, validY, testX, testY + + def get_raw_data(self): + return self.download() + + def get_normalized_data(self): + trainX, trainY, validX, validY, testX, testY = self.get_raw_data() + trainX = trainX / 255.0 * (self.img_max - self.img_min) + self.img_min + validX = validX / 255.0 * (self.img_max - self.img_min) + self.img_min + testX = testX / 255.0 * (self.img_max - self.img_min) + self.img_min + trainY = trainY + validY = validY + testY = testY + return trainX, trainY, validX, validY, testX, testY + +#%% +def mnist_model(): + # declare variables + model_layers = [ tf.keras.layers.Input(shape=(28,28,1), name="model/input"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c1"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c2"), + tf.keras.layers.MaxPooling2D(pool_size=(2, 2), name="clf/p1"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c3"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c4"), + tf.keras.layers.MaxPooling2D(pool_size=(2, 2), name="clf/p2"), + tf.keras.layers.Flatten(name="clf/f1"), + tf.keras.layers.Dense(256, activation="relu", kernel_initializer='he_normal', name="clf/d1"), + tf.keras.layers.Dense(10 , activation=None , kernel_initializer='he_normal', name="clf/d2"), + tf.keras.layers.Activation('softmax', name="clf_output") + ] + + # clf_model + clf_model = tf.keras.Sequential() + for ii in model_layers: + clf_model.add(ii) + + clf_model.compile(loss='categorical_crossentropy', optimizer='Nadam', metrics=['accuracy']) + clf_model.summary() + + return clf_model + +#%% +def arg_parser(parser): + + parser.add_argument("--path" , dest ="path", type=str, default='./', help="path of tf.keras model's wieghts") + args, unknown = parser.parse_known_args() + if unknown: + msg = " ".join(unknown) + print('[Warning] Unrecognized arguments: {:s}'.format(msg) ) + + return args + +#%% +if __name__ == '__main__': + + # get arguments + parser = ArgumentParser() + args = arg_parser(parser) + + # MODEL PATH + MODEL_PATH = args.path + + # init tf/keras + gpus = tf.config.list_physical_devices('GPU') + for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) + + # load data + batch_size = 1000 + epsilon = mnist_loader().epsilon + _, _, _, _, testX, testY = mnist_loader().get_normalized_data() + + # convert to pytorch format + testY = np.argmax(testY, axis=1) + torch_testX = torch.from_numpy( np.transpose(testX, (0, 3, 1, 2)) ).float().cuda() + torch_testY = torch.from_numpy( testY ).float() + + # load model from saved weights + print('[INFO] MODEL_PATH: {:s}'.format(MODEL_PATH) ) + tf_model = mnist_model() + tf_model.load_weights(MODEL_PATH) + + # remove 'softmax layer' and put it into adapter + atk_model = tf.keras.models.Model(inputs=tf_model.input, outputs=tf_model.get_layer(index=-2).output) + atk_model.summary() + model_adapted = utils_tf2.ModelAdapter(atk_model) + + # run attack + adversary = AutoAttack(model_adapted, norm='Linf', eps=epsilon, plus=False, is_tf_model=True) + x_adv = adversary.run_standard_evaluation(torch_testX, torch_testY, bs=batch_size) + np_x_adv = np.moveaxis(x_adv.cpu().numpy(), 1, 3) + np.save("./output/mnist_adv.npy", np_x_adv) diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/resnet.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/resnet.py new file mode 100644 index 0000000..cef5876 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/resnet.py @@ -0,0 +1,115 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion * planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512 * block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def ResNet18(): + return ResNet(BasicBlock, [2, 2, 2, 2]) + + +def ResNet34(): + return ResNet(BasicBlock, [3, 4, 6, 3]) + + +def ResNet50(): + return ResNet(Bottleneck, [3, 4, 6, 3]) + + +def ResNet101(): + return ResNet(Bottleneck, [3, 4, 23, 3]) + + +def ResNet152(): + return ResNet(Bottleneck, [3, 8, 36, 3]) + + +def test(): + net = ResNet18() + y = net(torch.randn(1, 3, 32, 32)) + print(y.size()) diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/run_eval.sh b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/run_eval.sh new file mode 100644 index 0000000..34a4d4b --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/run_eval.sh @@ -0,0 +1 @@ +CUDA_VISIBLE_DEVICES='0' python3 eval.py --model='../../GAIR_RST/GAIR_RST_results/checkpoint-epoch200.pt' --log_path='./GAIR_RST_AA_results.txt' --save_dir='./GAIR_RST_AA_results' & diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/wideresnet.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/wideresnet.py new file mode 100644 index 0000000..5a2eb19 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/wideresnet.py @@ -0,0 +1,98 @@ +"""Based on code from https://github.com/yaodongyu/TRADES""" + +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class WideResNet(nn.Module): + def __init__(self, depth=28, num_classes=10, widen_factor=10, dropRate=0.0): + super(WideResNet, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 4) % 6 == 0) + n = (depth - 4) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x, return_prelogit=False): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + if return_prelogit: + return self.fc(out), out + else: + return self.fc(out) + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/wrn_madry.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/wrn_madry.py new file mode 100644 index 0000000..08473ee --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/examples/wrn_madry.py @@ -0,0 +1,101 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class Wide_ResNet_Madry(nn.Module): + def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0): + super(Wide_ResNet_Madry, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 4) % 6 == 0) + n = (depth - 4) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + # self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + return self.fc(out) +def test(): + net = Wide_ResNet_Madry() + y = net(Variable(torch.randn(1, 3, 32, 32))) + #print(y.size()) + print(net) +# test() + +def wideresnet(): + return Wide_ResNet_Madry(depth=28, num_classes=10, widen_factor=10, dropRate=0.0) \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/fab_pt.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/fab_pt.py new file mode 100644 index 0000000..90e1ccf --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/fab_pt.py @@ -0,0 +1,748 @@ +# Copyright (c) 2019-present, Francesco Croce +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import torch +from torch.autograd.gradcheck import zero_gradients +import time + + +DEFAULT_EPS_DICT_BY_NORM = {'Linf': .3, 'L2': 1., 'L1': 5.0} + + +class FABAttack(): + """ + Fast Adaptive Boundary Attack (Linf, L2, L1) + https://arxiv.org/abs/1907.02044 + + :param predict: forward pass function + :param norm: Lp-norm to minimize ('Linf', 'L2', 'L1' supported) + :param n_restarts: number of random restarts + :param n_iter: number of iterations + :param eps: epsilon for the random restarts + :param alpha_max: alpha_max + :param eta: overshooting + :param beta: backward step + """ + + def __init__( + self, + predict, + norm='Linf', + n_restarts=1, + n_iter=100, + eps=None, + alpha_max=0.1, + eta=1.05, + beta=0.9, + loss_fn=None, + verbose=False, + seed=0, + targeted=False, + device=None, + n_target_classes=9): + """ FAB-attack implementation in pytorch """ + + self.predict = predict + self.norm = norm + self.n_restarts = n_restarts + self.n_iter = n_iter + self.eps = eps if eps is not None else DEFAULT_EPS_DICT_BY_NORM[norm] + self.alpha_max = alpha_max + self.eta = eta + self.beta = beta + self.targeted = False + self.verbose = verbose + self.seed = seed + self.target_class = None + self.device = device + self.n_target_classes = n_target_classes + + def _get_predicted_label(self, x): + with torch.no_grad(): + outputs = self.predict(x) + _, y = torch.max(outputs, dim=1) + return y + + def check_shape(self, x): + return x if len(x.shape) > 0 else x.unsqueeze(0) + + def get_diff_logits_grads_batch(self, imgs, la): + im = imgs.clone().requires_grad_() + with torch.enable_grad(): + y = self.predict(im) + + g2 = torch.zeros([y.shape[-1], *imgs.size()]).to(self.device) + grad_mask = torch.zeros_like(y) + for counter in range(y.shape[-1]): + zero_gradients(im) + grad_mask[:, counter] = 1.0 + y.backward(grad_mask, retain_graph=True) + grad_mask[:, counter] = 0.0 + g2[counter] = im.grad.data + + g2 = torch.transpose(g2, 0, 1).detach() + #y2 = self.predict(imgs).detach() + y2 = y.detach() + df = y2 - y2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + dg = g2 - g2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + df[torch.arange(imgs.shape[0]), la] = 1e10 + + return df, dg + + def get_diff_logits_grads_batch_targeted(self, imgs, la, la_target): + u = torch.arange(imgs.shape[0]) + im = imgs.clone().requires_grad_() + with torch.enable_grad(): + y = self.predict(im) + diffy = -(y[u, la] - y[u, la_target]) + sumdiffy = diffy.sum() + + zero_gradients(im) + sumdiffy.backward() + graddiffy = im.grad.data + df = diffy.detach().unsqueeze(1) + dg = graddiffy.unsqueeze(1) + + return df, dg + + def projection_linf(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + ind2 = ((w * t).sum(1) - b < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + b[ind2] *= -1 + + c5 = (w < 0).float() + a = torch.ones(t.shape).to(self.device) + d = (a * c5 - t) * (w != 0).float() + a -= a * (1 - c5) + + p = torch.ones(t.shape).to(self.device) * c5 - t * (2 * c5 - 1) + indp = torch.argsort(p, dim=1) + + b = b - (w * t).sum(1) + b0 = (w * d).sum(1) + b1 = b0.clone() + + counter = 0 + indp2 = indp.unsqueeze(-1).flip(dims=(1, 2)).squeeze() + u = torch.arange(0, w.shape[0]) + ws = w[u.unsqueeze(1), indp2] + bs2 = - ws * d[u.unsqueeze(1), indp2] + + s = torch.cumsum(ws.abs(), dim=1) + sb = torch.cumsum(bs2, dim=1) + b0.unsqueeze(1) + + c = b - b1 > 0 + b2 = sb[u, -1] - s[u, -1] * p[u, indp[u, 0]] + c_l = (b - b2 > 0).nonzero().squeeze() + c2 = ((b - b1 > 0) * (b - b2 <= 0)).nonzero().squeeze() + c_l = self.check_shape(c_l) + c2 = self.check_shape(c2) + + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(w.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + indcurr = indp[c2, -counter2 - 1] + b2 = sb[c2, counter2] - s[c2, counter2] * p[c2, indcurr] + c = b[c2] - b2 > 0 + ind3 = c.nonzero().squeeze() + ind32 = (~c).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb = lb.long() + counter2 = 0 + + if c_l.nelement != 0: + lmbd_opt = (torch.max((b[c_l] - sb[c_l, -1]) / (-s[c_l, -1]), + torch.zeros(sb[c_l, -1].shape) + .to(self.device))).unsqueeze(-1) + d[c_l] = (2 * a[c_l] - 1) * lmbd_opt + + lmbd_opt = (torch.max((b[c2] - sb[c2, lb]) / (-s[c2, lb]), + torch.zeros(sb[c2, lb].shape) + .to(self.device))).unsqueeze(-1) + d[c2] = torch.min(lmbd_opt, d[c2]) * c5[c2]\ + + torch.max(-lmbd_opt, d[c2]) * (1 - c5[c2]) + + return d * (w != 0).float() + + def projection_l2(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + c = (w * t).sum(1) - b + ind2 = (c < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + c[ind2] *= -1 + + u = torch.arange(0, w.shape[0]).unsqueeze(1) + + r = torch.max(t / w, (t - 1) / w) + u2 = torch.ones(r.shape).to(self.device) + r = torch.min(r, 1e12 * u2) + r = torch.max(r, -1e12 * u2) + r[w.abs() < 1e-8] = 1e12 + r[r == -1e12] = -r[r == -1e12] + rs, indr = torch.sort(r, dim=1) + rs2 = torch.cat((rs[:, 1:], + torch.zeros(rs.shape[0], 1).to(self.device)), 1) + rs[rs == 1e12] = 0 + rs2[rs2 == 1e12] = 0 + + w3 = w ** 2 + w3s = w3[u, indr] + w5 = w3s.sum(dim=1, keepdim=True) + ws = w5 - torch.cumsum(w3s, dim=1) + d = -(r * w).clone() + d = d * (w.abs() > 1e-8).float() + s = torch.cat(((-w5.squeeze() * rs[:, 0]).unsqueeze(1), + torch.cumsum((-rs2 + rs) * ws, dim=1) - + w5 * rs[:, 0].unsqueeze(-1)), 1) + + c4 = (s[:, 0] + c < 0) + c3 = ((d * w).sum(dim=1) + c > 0) + c6 = c4.nonzero().squeeze() + c2 = ((1 - c4.float()) * (1 - c3.float())).nonzero().squeeze() + c6 = self.check_shape(c6) + c2 = self.check_shape(c2) + + counter = 0 + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(w.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + c3 = s[c2, counter2] + c[c2] > 0 + ind3 = c3.nonzero().squeeze() + ind32 = (~c3).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb = lb.long() + alpha = torch.zeros([1]) + + if c6.nelement() != 0: + alpha = c[c6] / w5[c6].squeeze(-1) + d[c6] = -alpha.unsqueeze(-1) * w[c6] + + if c2.nelement() != 0: + alpha = (s[c2, lb] + c[c2]) / ws[c2, lb] + rs[c2, lb] + if torch.sum(ws[c2, lb] == 0) > 0: + ind = (ws[c2, lb] == 0).nonzero().squeeze().long() + ind = self.check_shape(ind) + alpha[ind] = 0 + c5 = (alpha.unsqueeze(-1) > r[c2]).float() + d[c2] = d[c2] * c5 - alpha.unsqueeze(-1) * w[c2] * (1 - c5) + + return d * (w.abs() > 1e-8).float() + + def projection_l1(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + c = (w * t).sum(1) - b + ind2 = (c < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + c[ind2] *= -1 + + r = torch.max(1 / w, -1 / w) + r = torch.min(r, 1e12 * torch.ones(r.shape).to(self.device)) + rs, indr = torch.sort(r, dim=1) + _, indr_rev = torch.sort(indr) + + u = torch.arange(0, w.shape[0]).unsqueeze(1) + u2 = torch.arange(0, w.shape[1]).repeat(w.shape[0], 1) + c6 = (w < 0).float() + d = (-t + c6) * (w != 0).float() + d2 = torch.min(-w * t, w * (1 - t)) + ds = d2[u, indr] + ds2 = torch.cat((c.unsqueeze(-1), ds), 1) + s = torch.cumsum(ds2, dim=1) + + c4 = s[:, -1] < 0 + c2 = c4.nonzero().squeeze(-1) + c2 = self.check_shape(c2) + + counter = 0 + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (s.shape[1]) + nitermax = torch.ceil(torch.log2(torch.tensor(s.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + c3 = s[c2, counter2] > 0 + ind3 = c3.nonzero().squeeze() + ind32 = (~c3).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb2 = lb.long() + + if c2.nelement() != 0: + alpha = -s[c2, lb2] / w[c2, indr[c2, lb2]] + c5 = u2[c2].float() < lb.unsqueeze(-1).float() + u3 = c5[u[:c5.shape[0]], indr_rev[c2]] + d[c2] = d[c2] * u3.float().to(self.device) + d[c2, indr[c2, lb2]] = alpha + + return d * (w.abs() > 1e-8).float() + + def attack_single_run(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + #self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + #assert next(self.predict.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print('Clean accuracy: {:.2%}'.format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == 'Linf': + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.reshape([t.shape[0], -1]).abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1]*self.ndims])) * .5 + elif self.norm == 'L2': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / ((t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1]*self.ndims)) * .5 + elif self.norm == 'L1': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.abs().view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1]*self.ndims)) / 2 + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.n_iter: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch(x1, la2) + if self.norm == 'Linf': + dist1 = df.abs() / (1e-12 + + dg.abs() + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1)) + elif self.norm == 'L2': + dist1 = df.abs() / (1e-12 + (dg ** 2) + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1).sqrt()) + elif self.norm == 'L1': + dist1 = df.abs() / (1e-12 + dg.abs().reshape( + [df.shape[0], df.shape[1], -1]).max(dim=2)[0]) + else: + raise ValueError('norm not supported') + ind = dist1.min(dim=1)[1] + dg2 = dg[u1, ind] + b = (- df[u1, ind] + (dg2 * x1).view(x1.shape[0], -1) + .sum(dim=-1)) + w = dg2.reshape([bs, -1]) + + if self.norm == 'Linf': + d3 = self.projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L2': + d3 = self.projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L1': + d3 = self.projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == 'Linf': + a0 = d3.abs().max(dim=1, keepdim=True)[0]\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L2': + a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt()\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L1': + a0 = d3.abs().sum(dim=1, keepdim=True)\ + .view(-1, *[1]*self.ndims) + a0 = torch.max(a0, 1e-8 * torch.ones( + a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min(torch.max(a1 / (a1 + a2), + torch.zeros(a1.shape) + .to(self.device))[0], + self.alpha_max * torch.ones(a1.shape) + .to(self.device)) + x1 = ((x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == 'Linf': + t = (x1[ind_adv] - im2[ind_adv]).reshape( + [ind_adv.shape[0], -1]).abs().max(dim=1)[0] + elif self.norm == 'L2': + t = ((x1[ind_adv] - im2[ind_adv]) ** 2)\ + .view(ind_adv.shape[0], -1).sum(dim=-1).sqrt() + elif self.norm == 'L1': + t = (x1[ind_adv] - im2[ind_adv])\ + .abs().view(ind_adv.shape[0], -1).sum(dim=-1) + adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]).\ + float().reshape([-1, *[1]*self.ndims]) + adv[ind_adv]\ + * (t >= res2[ind_adv]).float().reshape( + [-1, *[1]*self.ndims]) + res2[ind_adv] = t * (t < res2[ind_adv]).float()\ + + res2[ind_adv] * (t >= res2[ind_adv]).float() + x1[ind_adv] = im2[ind_adv] + ( + x1[ind_adv] - im2[ind_adv]) * self.beta + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print('success rate: {:.0f}/{:.0f}' + .format(ind_succ.float().sum(), corr_classified) + + ' (on correctly classified points) in {:.1f} s' + .format(time.time() - startt)) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def attack_single_run_targeted(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + if self.device is None: + self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + #assert next(self.predict.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print('Clean accuracy: {:.2%}'.format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + output = self.predict(x) + la_target = output.sort(dim=-1)[1][:, -self.target_class] + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + la_target2 = la_target[pred].detach().clone() + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == 'Linf': + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.reshape([t.shape[0], -1]).abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1]*self.ndims])) * .5 + elif self.norm == 'L2': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / ((t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1]*self.ndims)) * .5 + elif self.norm == 'L1': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.abs().view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1]*self.ndims)) / 2 + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.n_iter: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch_targeted(x1, la2, la_target2) + if self.norm == 'Linf': + dist1 = df.abs() / (1e-12 + + dg.abs() + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1)) + elif self.norm == 'L2': + dist1 = df.abs() / (1e-12 + (dg ** 2) + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1).sqrt()) + elif self.norm == 'L1': + dist1 = df.abs() / (1e-12 + dg.abs().reshape( + [df.shape[0], df.shape[1], -1]).max(dim=2)[0]) + else: + raise ValueError('norm not supported') + ind = dist1.min(dim=1)[1] + #print(ind) + dg2 = dg[u1, ind] + b = (- df[u1, ind] + (dg2 * x1).view(x1.shape[0], -1) + .sum(dim=-1)) + w = dg2.reshape([bs, -1]) + + if self.norm == 'Linf': + d3 = self.projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L2': + d3 = self.projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L1': + d3 = self.projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == 'Linf': + a0 = d3.abs().max(dim=1, keepdim=True)[0]\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L2': + a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt()\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L1': + a0 = d3.abs().sum(dim=1, keepdim=True)\ + .view(-1, *[1]*self.ndims) + a0 = torch.max(a0, 1e-8 * torch.ones( + a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min(torch.max(a1 / (a1 + a2), + torch.zeros(a1.shape) + .to(self.device))[0], + self.alpha_max * torch.ones(a1.shape) + .to(self.device)) + x1 = ((x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == 'Linf': + t = (x1[ind_adv] - im2[ind_adv]).reshape( + [ind_adv.shape[0], -1]).abs().max(dim=1)[0] + elif self.norm == 'L2': + t = ((x1[ind_adv] - im2[ind_adv]) ** 2)\ + .view(ind_adv.shape[0], -1).sum(dim=-1).sqrt() + elif self.norm == 'L1': + t = (x1[ind_adv] - im2[ind_adv])\ + .abs().view(ind_adv.shape[0], -1).sum(dim=-1) + adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]).\ + float().reshape([-1, *[1]*self.ndims]) + adv[ind_adv]\ + * (t >= res2[ind_adv]).float().reshape( + [-1, *[1]*self.ndims]) + res2[ind_adv] = t * (t < res2[ind_adv]).float()\ + + res2[ind_adv] * (t >= res2[ind_adv]).float() + x1[ind_adv] = im2[ind_adv] + ( + x1[ind_adv] - im2[ind_adv]) * self.beta + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print('success rate: {:.0f}/{:.0f}' + .format(ind_succ.float().sum(), corr_classified) + + ' (on correctly classified points) in {:.1f} s' + .format(time.time() - startt)) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def perturb(self, x, y): + adv = x.clone() + with torch.no_grad(): + acc = self.predict(x).max(1)[1] == y + + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not self.targeted: + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + adv_curr = self.attack_single_run(x_to_fool, y_to_fool, use_rand_start=(counter > 0)) + + acc_curr = self.predict(adv_curr).max(1)[1] == y_to_fool + if self.norm == 'Linf': + res = (x_to_fool - adv_curr).abs().view(x_to_fool.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_to_fool - adv_curr) ** 2).view(x_to_fool.shape[0], -1).sum(dim=-1).sqrt() + acc_curr = torch.max(acc_curr, res > self.eps) + + ind_curr = (acc_curr == 0).nonzero().squeeze() + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + + if self.verbose: + print('restart {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, acc.float().mean(), self.eps, time.time() - startt)) + + else: + for target_class in range(2, self.n_target_classes + 2): + self.target_class = target_class + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + adv_curr = self.attack_single_run_targeted(x_to_fool, y_to_fool, use_rand_start=(counter > 0)) + + acc_curr = self.predict(adv_curr).max(1)[1] == y_to_fool + if self.norm == 'Linf': + res = (x_to_fool - adv_curr).abs().view(x_to_fool.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_to_fool - adv_curr) ** 2).view(x_to_fool.shape[0], -1).sum(dim=-1).sqrt() + acc_curr = torch.max(acc_curr, res > self.eps) + + ind_curr = (acc_curr == 0).nonzero().squeeze() + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + + if self.verbose: + print('restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, self.target_class, acc.float().mean(), self.eps, time.time() - startt)) + + return adv + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/fab_tf.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/fab_tf.py new file mode 100644 index 0000000..8498f21 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/fab_tf.py @@ -0,0 +1,745 @@ +# Copyright (c) 2019-present, Francesco Croce +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import torch +from torch.autograd.gradcheck import zero_gradients +import time + +import tensorflow as tf + +#from advertorch.utils import replicate_input + +#from .base import Attack +#from .base import LabelMixin + +DEFAULT_EPS_DICT_BY_NORM = {'Linf': .3, 'L2': 1., 'L1': 5.0} + + +class FABAttack(): + """ + Fast Adaptive Boundary Attack (Linf, L2, L1) + https://arxiv.org/abs/1907.02044 + + :param predict: forward pass function + :param norm: Lp-norm to minimize ('Linf', 'L2', 'L1' supported) + :param n_restarts: number of random restarts + :param n_iter: number of iterations + :param eps: epsilon for the random restarts + :param alpha_max: alpha_max + :param eta: overshooting + :param beta: backward step + """ + + def __init__( + self, + model, + norm='Linf', + n_restarts=1, + n_iter=100, + eps=None, + alpha_max=0.1, + eta=1.05, + beta=0.9, + loss_fn=None, + verbose=False, + seed=0, + targeted=False, + device=None, + n_target_classes=9): + """ FAB-attack implementation in pytorch """ + + self.model = model + self.norm = norm + self.n_restarts = n_restarts + self.n_iter = n_iter + self.eps = eps if eps is not None else DEFAULT_EPS_DICT_BY_NORM[norm] + self.alpha_max = alpha_max + self.eta = eta + self.beta = beta + self.targeted = False + self.verbose = verbose + self.seed = seed + self.target_class = None + self.device = device + self.n_target_classes = n_target_classes + + def _get_predicted_label(self, x): + with torch.no_grad(): + outputs = self.model.predict(x) + _, y = torch.max(outputs, dim=1) + return y + + def check_shape(self, x): + return x if len(x.shape) > 0 else x.unsqueeze(0) + + def get_diff_logits_grads_batch(self, imgs, la): + ### TODO: get both gradients and values with a single pass + g2 = self.model.grad_logits(imgs) + y2 = self.model.predict(imgs) + df = y2 - y2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + dg = g2 - g2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + df[torch.arange(imgs.shape[0]), la] = 1e10 + + return df, dg + + def get_diff_logits_grads_batch_targeted(self, imgs, la, la_target): + df, dg = self.model.get_grad_diff_logits_target(imgs, la, la_target) + + return df, dg + + + def projection_linf(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + ind2 = ((w * t).sum(1) - b < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + b[ind2] *= -1 + + c5 = (w < 0).float() + a = torch.ones(t.shape).to(self.device) + d = (a * c5 - t) * (w != 0).float() + a -= a * (1 - c5) + + p = torch.ones(t.shape).to(self.device) * c5 - t * (2 * c5 - 1) + indp = torch.argsort(p, dim=1) + + b = b - (w * t).sum(1) + b0 = (w * d).sum(1) + b1 = b0.clone() + + counter = 0 + indp2 = indp.unsqueeze(-1).flip(dims=(1, 2)).squeeze() + u = torch.arange(0, w.shape[0]) + ws = w[u.unsqueeze(1), indp2] + bs2 = - ws * d[u.unsqueeze(1), indp2] + + s = torch.cumsum(ws.abs(), dim=1) + sb = torch.cumsum(bs2, dim=1) + b0.unsqueeze(1) + + c = b - b1 > 0 + b2 = sb[u, -1] - s[u, -1] * p[u, indp[u, 0]] + c_l = (b - b2 > 0).nonzero().squeeze() + c2 = ((b - b1 > 0) * (b - b2 <= 0)).nonzero().squeeze() + c_l = self.check_shape(c_l) + c2 = self.check_shape(c2) + + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(w.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + indcurr = indp[c2, -counter2 - 1] + b2 = sb[c2, counter2] - s[c2, counter2] * p[c2, indcurr] + c = b[c2] - b2 > 0 + ind3 = c.nonzero().squeeze() + ind32 = (~c).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb = lb.long() + counter2 = 0 + + if c_l.nelement != 0: + lmbd_opt = (torch.max((b[c_l] - sb[c_l, -1]) / (-s[c_l, -1]), + torch.zeros(sb[c_l, -1].shape) + .to(self.device))).unsqueeze(-1) + d[c_l] = (2 * a[c_l] - 1) * lmbd_opt + + lmbd_opt = (torch.max((b[c2] - sb[c2, lb]) / (-s[c2, lb]), + torch.zeros(sb[c2, lb].shape) + .to(self.device))).unsqueeze(-1) + d[c2] = torch.min(lmbd_opt, d[c2]) * c5[c2]\ + + torch.max(-lmbd_opt, d[c2]) * (1 - c5[c2]) + + return d * (w != 0).float() + + def projection_l2(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + c = (w * t).sum(1) - b + ind2 = (c < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + c[ind2] *= -1 + + u = torch.arange(0, w.shape[0]).unsqueeze(1) + + r = torch.max(t / w, (t - 1) / w) + u2 = torch.ones(r.shape).to(self.device) + r = torch.min(r, 1e12 * u2) + r = torch.max(r, -1e12 * u2) + r[w.abs() < 1e-8] = 1e12 + r[r == -1e12] = -r[r == -1e12] + rs, indr = torch.sort(r, dim=1) + rs2 = torch.cat((rs[:, 1:], + torch.zeros(rs.shape[0], 1).to(self.device)), 1) + rs[rs == 1e12] = 0 + rs2[rs2 == 1e12] = 0 + + w3 = w ** 2 + w3s = w3[u, indr] + w5 = w3s.sum(dim=1, keepdim=True) + ws = w5 - torch.cumsum(w3s, dim=1) + d = -(r * w).clone() + d = d * (w.abs() > 1e-8).float() + s = torch.cat(((-w5.squeeze() * rs[:, 0]).unsqueeze(1), + torch.cumsum((-rs2 + rs) * ws, dim=1) - + w5 * rs[:, 0].unsqueeze(-1)), 1) + + c4 = (s[:, 0] + c < 0) + c3 = ((d * w).sum(dim=1) + c > 0) + c6 = c4.nonzero().squeeze() + c2 = ((1 - c4.float()) * (1 - c3.float())).nonzero().squeeze() + c6 = self.check_shape(c6) + c2 = self.check_shape(c2) + + counter = 0 + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(w.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + c3 = s[c2, counter2] + c[c2] > 0 + ind3 = c3.nonzero().squeeze() + ind32 = (~c3).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb = lb.long() + alpha = torch.zeros([1]) + + if c6.nelement() != 0: + alpha = c[c6] / w5[c6].squeeze(-1) + d[c6] = -alpha.unsqueeze(-1) * w[c6] + + if c2.nelement() != 0: + alpha = (s[c2, lb] + c[c2]) / ws[c2, lb] + rs[c2, lb] + if torch.sum(ws[c2, lb] == 0) > 0: + ind = (ws[c2, lb] == 0).nonzero().squeeze().long() + ind = self.check_shape(ind) + alpha[ind] = 0 + c5 = (alpha.unsqueeze(-1) > r[c2]).float() + d[c2] = d[c2] * c5 - alpha.unsqueeze(-1) * w[c2] * (1 - c5) + + return d * (w.abs() > 1e-8).float() + + def projection_l1(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + c = (w * t).sum(1) - b + ind2 = (c < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + c[ind2] *= -1 + + r = torch.max(1 / w, -1 / w) + r = torch.min(r, 1e12 * torch.ones(r.shape).to(self.device)) + rs, indr = torch.sort(r, dim=1) + _, indr_rev = torch.sort(indr) + + u = torch.arange(0, w.shape[0]).unsqueeze(1) + u2 = torch.arange(0, w.shape[1]).repeat(w.shape[0], 1) + c6 = (w < 0).float() + d = (-t + c6) * (w != 0).float() + d2 = torch.min(-w * t, w * (1 - t)) + ds = d2[u, indr] + ds2 = torch.cat((c.unsqueeze(-1), ds), 1) + s = torch.cumsum(ds2, dim=1) + + c4 = s[:, -1] < 0 + c2 = c4.nonzero().squeeze(-1) + c2 = self.check_shape(c2) + + counter = 0 + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (s.shape[1]) + nitermax = torch.ceil(torch.log2(torch.tensor(s.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + c3 = s[c2, counter2] > 0 + ind3 = c3.nonzero().squeeze() + ind32 = (~c3).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb2 = lb.long() + + if c2.nelement() != 0: + alpha = -s[c2, lb2] / w[c2, indr[c2, lb2]] + c5 = u2[c2].float() < lb.unsqueeze(-1).float() + u3 = c5[u[:c5.shape[0]], indr_rev[c2]] + d[c2] = d[c2] * u3.float().to(self.device) + d[c2, indr[c2, lb2]] = alpha + + return d * (w.abs() > 1e-8).float() + + def attack_single_run(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + if self.device is None: + self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + #assert next(self.predict.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print('Clean accuracy: {:.2%}'.format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == 'Linf': + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.reshape([t.shape[0], -1]).abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1]*self.ndims])) * .5 + elif self.norm == 'L2': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / ((t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1]*self.ndims)) * .5 + elif self.norm == 'L1': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.abs().view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1]*self.ndims)) / 2 + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.n_iter: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch(x1, la2) + if self.norm == 'Linf': + dist1 = df.abs() / (1e-12 + + dg.abs() + .reshape(dg.shape[0], dg.shape[1], -1) # view(...) + .sum(dim=-1)) + elif self.norm == 'L2': + dist1 = df.abs() / (1e-12 + (dg ** 2) + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1).sqrt()) + elif self.norm == 'L1': + dist1 = df.abs() / (1e-12 + dg.abs().reshape( + [df.shape[0], df.shape[1], -1]).max(dim=2)[0]) + else: + raise ValueError('norm not supported') + ind = dist1.min(dim=1)[1] + dg2 = dg[u1, ind] + b = (- df[u1, ind] + (dg2 * x1).reshape(x1.shape[0], -1) # view(...) + .sum(dim=-1)) + w = dg2.reshape([bs, -1]) + + if self.norm == 'Linf': + d3 = self.projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L2': + d3 = self.projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L1': + d3 = self.projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == 'Linf': + a0 = d3.abs().max(dim=1, keepdim=True)[0]\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L2': + a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt()\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L1': + a0 = d3.abs().sum(dim=1, keepdim=True)\ + .view(-1, *[1]*self.ndims) + a0 = torch.max(a0, 1e-8 * torch.ones( + a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min(torch.max(a1 / (a1 + a2), + torch.zeros(a1.shape) + .to(self.device))[0], + self.alpha_max * torch.ones(a1.shape) + .to(self.device)) + x1 = ((x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == 'Linf': + t = (x1[ind_adv] - im2[ind_adv]).reshape( + [ind_adv.shape[0], -1]).abs().max(dim=1)[0] + elif self.norm == 'L2': + t = ((x1[ind_adv] - im2[ind_adv]) ** 2)\ + .view(ind_adv.shape[0], -1).sum(dim=-1).sqrt() + elif self.norm == 'L1': + t = (x1[ind_adv] - im2[ind_adv])\ + .abs().view(ind_adv.shape[0], -1).sum(dim=-1) + adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]).\ + float().reshape([-1, *[1]*self.ndims]) + adv[ind_adv]\ + * (t >= res2[ind_adv]).float().reshape( + [-1, *[1]*self.ndims]) + res2[ind_adv] = t * (t < res2[ind_adv]).float()\ + + res2[ind_adv] * (t >= res2[ind_adv]).float() + x1[ind_adv] = im2[ind_adv] + ( + x1[ind_adv] - im2[ind_adv]) * self.beta + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print('success rate: {:.0f}/{:.0f}' + .format(ind_succ.float().sum(), corr_classified) + + ' (on correctly classified points) in {:.1f} s' + .format(time.time() - startt)) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def attack_single_run_targeted(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + #assert next(self.predict.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print('Clean accuracy: {:.2%}'.format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + output = self.model.predict(x) + la_target = output.sort(dim=-1)[1][:, -self.target_class] + + # set target class + #self.model.set_target_class(y, la_target) + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + la_target2 = la_target[pred].detach().clone() + + # set target class for correcty classified points + #self.model.set_target_class(la2, la_target2) + + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == 'Linf': + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.reshape([t.shape[0], -1]).abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1]*self.ndims])) * .5 + elif self.norm == 'L2': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / ((t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1]*self.ndims)) * .5 + elif self.norm == 'L1': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.abs().view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1]*self.ndims)) / 2 + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.n_iter: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch_targeted(x1, la2, la_target2) + if len(df.shape) == 1: + df.unsqueeze_(1) + dg.unsqueeze_(1) + if self.norm == 'Linf': + dist1 = df.abs() / (1e-12 + + dg.abs() + .reshape(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1)) + elif self.norm == 'L2': + dist1 = df.abs() / (1e-12 + (dg ** 2) + .reshape(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1).sqrt()) + elif self.norm == 'L1': + dist1 = df.abs() / (1e-12 + dg.abs().reshape( + [df.shape[0], df.shape[1], -1]).max(dim=2)[0]) + else: + raise ValueError('norm not supported') + ind = dist1.min(dim=1)[1] + #print(ind) + dg2 = dg[u1, ind] + b = (- df[u1, ind] + (dg2 * x1).reshape(x1.shape[0], -1) + .sum(dim=-1)) + w = dg2.reshape([bs, -1]) + + if self.norm == 'Linf': + d3 = self.projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L2': + d3 = self.projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L1': + d3 = self.projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == 'Linf': + a0 = d3.abs().max(dim=1, keepdim=True)[0]\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L2': + a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt()\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L1': + a0 = d3.abs().sum(dim=1, keepdim=True)\ + .view(-1, *[1]*self.ndims) + a0 = torch.max(a0, 1e-8 * torch.ones( + a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min(torch.max(a1 / (a1 + a2), + torch.zeros(a1.shape) + .to(self.device))[0], + self.alpha_max * torch.ones(a1.shape) + .to(self.device)) + x1 = ((x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == 'Linf': + t = (x1[ind_adv] - im2[ind_adv]).reshape( + [ind_adv.shape[0], -1]).abs().max(dim=1)[0] + elif self.norm == 'L2': + t = ((x1[ind_adv] - im2[ind_adv]) ** 2)\ + .reshape(ind_adv.shape[0], -1).sum(dim=-1).sqrt() + elif self.norm == 'L1': + t = (x1[ind_adv] - im2[ind_adv])\ + .abs().reshape(ind_adv.shape[0], -1).sum(dim=-1) + adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]).\ + float().reshape([-1, *[1]*self.ndims]) + adv[ind_adv]\ + * (t >= res2[ind_adv]).float().reshape( + [-1, *[1]*self.ndims]) + res2[ind_adv] = t * (t < res2[ind_adv]).float()\ + + res2[ind_adv] * (t >= res2[ind_adv]).float() + x1[ind_adv] = im2[ind_adv] + ( + x1[ind_adv] - im2[ind_adv]) * self.beta + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print('success rate: {:.0f}/{:.0f}' + .format(ind_succ.float().sum(), corr_classified) + + ' (on correctly classified points) in {:.1f} s' + .format(time.time() - startt)) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def perturb(self, x, y): + if self.device is None: + self.device = x.device + adv = x.clone() + with torch.no_grad(): + acc = self.model.predict(x).max(1)[1] == y + + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not self.targeted: + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + adv_curr = self.attack_single_run(x_to_fool, y_to_fool, use_rand_start=(counter > 0)) + + acc_curr = self.model.predict(adv_curr).max(1)[1] == y_to_fool + if self.norm == 'Linf': + res = (x_to_fool - adv_curr).abs().reshape(x_to_fool.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_to_fool - adv_curr) ** 2).reshape(x_to_fool.shape[0], -1).sum(dim=-1).sqrt() + acc_curr = torch.max(acc_curr, res > self.eps) + + ind_curr = (acc_curr == 0).nonzero().squeeze() + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + + if self.verbose: + print('restart {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, acc.float().mean(), self.eps, time.time() - startt)) + + else: + for target_class in range(2, self.n_target_classes + 2): + self.target_class = target_class + # + + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + adv_curr = self.attack_single_run_targeted(x_to_fool, y_to_fool, use_rand_start=(counter > 0)) + + acc_curr = self.model.predict(adv_curr).max(1)[1] == y_to_fool + if self.norm == 'Linf': + res = (x_to_fool - adv_curr).abs().reshape(x_to_fool.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_to_fool - adv_curr) ** 2).reshape(x_to_fool.shape[0], -1).sum(dim=-1).sqrt() + acc_curr = torch.max(acc_curr, res > self.eps) + + ind_curr = (acc_curr == 0).nonzero().squeeze() + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + + if self.verbose: + print('restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, self.target_class, acc.float().mean(), self.eps, time.time() - startt)) + + return adv + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/other_utils.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/other_utils.py new file mode 100644 index 0000000..98bdb81 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/other_utils.py @@ -0,0 +1,13 @@ +import os + +class Logger(): + def __init__(self, log_path): + self.log_path = log_path + + def log(self, str_to_log): + print(str_to_log) + if not self.log_path is None: + with open(self.log_path, 'a') as f: + f.write(str_to_log + '\n') + f.flush() + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/square.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/square.py new file mode 100644 index 0000000..1f32bba --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/square.py @@ -0,0 +1,441 @@ +# Copyright (c) 2020-present, Francesco Croce +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import torch +import time +import math +import torch.nn.functional as F + + +class SquareAttack(): + """ + Square Attack + https://arxiv.org/abs/1912.00049 + + :param predict: forward pass function + :param norm: Lp-norm of the attack ('Linf', 'L2' supported) + :param n_restarts: number of random restarts + :param n_queries: max number of queries (each restart) + :param eps: bound on the norm of perturbations + :param seed: random seed for the starting point + :param p_init: parameter to control size of squares + :param loss: loss function optimized ('margin', 'ce' supported) + :param resc_schedule adapt schedule of p to n_queries + """ + + def __init__( + self, + predict, + norm='Linf', + n_queries=5000, + eps=None, + p_init=.8, + n_restarts=1, + seed=0, + verbose=False, + targeted=False, + loss='margin', + resc_schedule=True, + device=None): + """ + Square Attack implementation in PyTorch + """ + + self.predict = predict + self.norm = norm + self.n_queries = n_queries + self.eps = eps + self.p_init = p_init + self.n_restarts = n_restarts + self.seed = seed + self.verbose = verbose + self.targeted = targeted + self.loss = loss + self.rescale_schedule = resc_schedule + self.device = device + + def margin_and_loss(self, x, y): + """ + :param y: correct labels if untargeted else target labels + """ + + logits = self.predict(x) + xent = F.cross_entropy(logits, y, reduction='none') + u = torch.arange(x.shape[0]) + y_corr = logits[u, y].clone() + logits[u, y] = -float('inf') + y_others = logits.max(dim=-1)[0] + + if not self.targeted: + if self.loss == 'ce': + return y_corr - y_others, -1. * xent + elif self.loss == 'margin': + return y_corr - y_others, y_corr - y_others + else: + return y_others - y_corr, xent + + def init_hyperparam(self, x): + assert self.norm in ['Linf', 'L2'] + assert not self.eps is None + assert self.loss in ['ce', 'margin'] + + if self.device is None: + self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + if self.seed is None: + self.seed = time.time() + + def random_target_classes(self, y_pred, n_classes): + y = torch.zeros_like(y_pred) + for counter in range(y_pred.shape[0]): + l = list(range(n_classes)) + l.remove(y_pred[counter]) + t = self.random_int(0, len(l)) + y[counter] = l[t] + + return y.long().to(self.device) + + def check_shape(self, x): + return x if len(x.shape) == (self.ndims + 1) else x.unsqueeze(0) + + def random_choice(self, shape): + t = 2 * torch.rand(shape).to(self.device) - 1 + return torch.sign(t) + + def random_int(self, low=0, high=1, shape=[1]): + t = low + (high - low) * torch.rand(shape).to(self.device) + return t.long() + + def normalize(self, x): + if self.norm == 'Linf': + t = x.abs().view(x.shape[0], -1).max(1)[0] + return x / (t.view(-1, *([1] * self.ndims)) + 1e-12) + + elif self.norm == 'L2': + t = (x ** 2).view(x.shape[0], -1).sum(-1).sqrt() + return x / (t.view(-1, *([1] * self.ndims)) + 1e-12) + + def lp_norm(self, x): + if self.norm == 'L2': + t = (x ** 2).view(x.shape[0], -1).sum(-1).sqrt() + return t.view(-1, *([1] * self.ndims)) + + def eta_rectangles(self, x, y): + delta = torch.zeros([x, y]).to(self.device) + x_c, y_c = x // 2 + 1, y // 2 + 1 + + counter2 = [x_c - 1, y_c - 1] + for counter in range(0, max(x_c, y_c)): + delta[max(counter2[0], 0):min(counter2[0] + (2*counter + 1), x), + max(0, counter2[1]):min(counter2[1] + (2*counter + 1), y) + ] += 1.0/(torch.Tensor([counter + 1]).view(1, 1).to( + self.device) ** 2) + counter2[0] -= 1 + counter2[1] -= 1 + + delta /= (delta ** 2).sum(dim=(0,1), keepdim=True).sqrt() + + return delta + + def eta(self, s): + delta = torch.zeros([s, s]).to(self.device) + delta[:s // 2] = self.eta_rectangles(s // 2, s) + delta[s // 2:] = -1. * self.eta_rectangles(s - s // 2, s) + delta /= (delta ** 2).sum(dim=(0, 1), keepdim=True).sqrt() + if torch.rand([1]) > 0.5: + delta = delta.permute([1, 0]) + + return delta + + def p_selection(self, it): + """ schedule to decrease the parameter p """ + + if self.rescale_schedule: + it = int(it / self.n_queries * 10000) + + if 10 < it <= 50: + p = self.p_init / 2 + elif 50 < it <= 200: + p = self.p_init / 4 + elif 200 < it <= 500: + p = self.p_init / 8 + elif 500 < it <= 1000: + p = self.p_init / 16 + elif 1000 < it <= 2000: + p = self.p_init / 32 + elif 2000 < it <= 4000: + p = self.p_init / 64 + elif 4000 < it <= 6000: + p = self.p_init / 128 + elif 6000 < it <= 8000: + p = self.p_init / 256 + elif 8000 < it: + p = self.p_init / 512 + else: + p = self.p_init + + return p + + def attack_single_run(self, x, y): + with torch.no_grad(): + adv = x.clone() + c, h, w = x.shape[1:] + n_features = c * h * w + n_ex_total = x.shape[0] + + if self.norm == 'Linf': + x_best = torch.clamp(x + self.eps * self.random_choice( + [x.shape[0], c, 1, w]), 0., 1.) + margin_min, loss_min = self.margin_and_loss(x_best, y) + n_queries = torch.ones(x.shape[0]).to(self.device) + s_init = int(math.sqrt(self.p_init * n_features / c)) + + for i_iter in range(self.n_queries): + idx_to_fool = (margin_min > 0.0).nonzero().squeeze() + + x_curr = self.check_shape(x[idx_to_fool]) + x_best_curr = self.check_shape(x_best[idx_to_fool]) + y_curr = y[idx_to_fool] + if len(y_curr.shape) == 0: + y_curr = y_curr.unsqueeze(0) + margin_min_curr = margin_min[idx_to_fool] + loss_min_curr = loss_min[idx_to_fool] + + p = self.p_selection(i_iter) + s = max(int(round(math.sqrt(p * n_features / c))), 1) + vh = self.random_int(0, h - s) + vw = self.random_int(0, w - s) + new_deltas = torch.zeros([c, h, w]).to(self.device) + new_deltas[:, vh:vh + s, vw:vw + s + ] = 2. * self.eps * self.random_choice([c, 1, 1]) + + x_new = x_best_curr + new_deltas + x_new = torch.min(torch.max(x_new, x_curr - self.eps), + x_curr + self.eps) + x_new = torch.clamp(x_new, 0., 1.) + x_new = self.check_shape(x_new) + + margin, loss = self.margin_and_loss(x_new, y_curr) + + # update loss if new loss is better + idx_improved = (loss < loss_min_curr).float() + + loss_min[idx_to_fool] = idx_improved * loss + ( + 1. - idx_improved) * loss_min_curr + + # update margin and x_best if new loss is better + # or misclassification + idx_miscl = (margin <= 0.).float() + idx_improved = torch.max(idx_improved, idx_miscl) + + margin_min[idx_to_fool] = idx_improved * margin + ( + 1. - idx_improved) * margin_min_curr + idx_improved = idx_improved.reshape([-1, + *[1]*len(x.shape[:-1])]) + x_best[idx_to_fool] = idx_improved * x_new + ( + 1. - idx_improved) * x_best_curr + n_queries[idx_to_fool] += 1. + + ind_succ = (margin_min <= 0.).nonzero().squeeze() + if self.verbose and ind_succ.numel() != 0: + print('{}'.format(i_iter + 1), + '- success rate={}/{} ({:.2%})'.format( + ind_succ.numel(), n_ex_total, + float(ind_succ.numel()) / n_ex_total), + '- avg # queries={:.1f}'.format( + n_queries[ind_succ].mean().item()), + '- med # queries={:.1f}'.format( + n_queries[ind_succ].median().item()), + '- loss={:.3f}'.format(loss_min.mean())) + + if ind_succ.numel() == n_ex_total: + break + + elif self.norm == 'L2': + delta_init = torch.zeros_like(x) + s = h // 5 + sp_init = (h - s * 5) // 2 + vh = sp_init + 0 + for _ in range(h // s): + vw = sp_init + 0 + for _ in range(w // s): + delta_init[:, :, vh:vh + s, vw:vw + s] += self.eta( + s).view(1, 1, s, s) * self.random_choice( + [x.shape[0], c, 1, 1]) + vw += s + vh += s + + x_best = torch.clamp(x + self.normalize(delta_init + ) * self.eps, 0., 1.) + margin_min, loss_min = self.margin_and_loss(x_best, y) + n_queries = torch.ones(x.shape[0]).to(self.device) + s_init = int(math.sqrt(self.p_init * n_features / c)) + + for i_iter in range(self.n_queries): + idx_to_fool = (margin_min > 0.0).nonzero().squeeze() + + x_curr = self.check_shape(x[idx_to_fool]) + x_best_curr = self.check_shape(x_best[idx_to_fool]) + y_curr = y[idx_to_fool] + if len(y_curr.shape) == 0: + y_curr = y_curr.unsqueeze(0) + margin_min_curr = margin_min[idx_to_fool] + loss_min_curr = loss_min[idx_to_fool] + + delta_curr = x_best_curr - x_curr + p = self.p_selection(i_iter) + s = max(int(round(math.sqrt(p * n_features / c))), 3) + if s % 2 == 0: + s += 1 + + vh = self.random_int(0, h - s) + vw = self.random_int(0, w - s) + new_deltas_mask = torch.zeros_like(x_curr) + new_deltas_mask[:, :, vh:vh + s, vw:vw + s] = 1.0 + norms_window_1 = (delta_curr[:, :, vh:vh + s, vw:vw + s + ] ** 2).sum(dim=(-2, -1), keepdim=True).sqrt() + + vh2 = self.random_int(0, h - s) + vw2 = self.random_int(0, w - s) + new_deltas_mask_2 = torch.zeros_like(x_curr) + new_deltas_mask_2[:, :, vh2:vh2 + s, vw2:vw2 + s] = 1. + + norms_image = self.lp_norm(x_best_curr - x_curr) + mask_image = torch.max(new_deltas_mask, new_deltas_mask_2) + norms_windows = self.lp_norm(delta_curr * mask_image) + + new_deltas = torch.ones([x_curr.shape[0], c, s, s] + ).to(self.device) + new_deltas *= (self.eta(s).view(1, 1, s, s) * + self.random_choice([x_curr.shape[0], c, 1, 1])) + old_deltas = delta_curr[:, :, vh:vh + s, vw:vw + s] / ( + 1e-12 + norms_window_1) + new_deltas += old_deltas + new_deltas = new_deltas / (1e-12 + (new_deltas ** 2).sum( + dim=(-2, -1), keepdim=True).sqrt()) * (torch.max( + (self.eps * torch.ones_like(new_deltas)) ** 2 - + norms_image ** 2, torch.zeros_like(new_deltas)) / + c + norms_windows ** 2).sqrt() + delta_curr[:, :, vh2:vh2 + s, vw2:vw2 + s] = 0. + delta_curr[:, :, vh:vh + s, vw:vw + s] = new_deltas + 0 + + x_new = torch.clamp(x_curr + self.normalize(delta_curr + ) * self.eps, 0. ,1.) + x_new = self.check_shape(x_new) + norms_image = self.lp_norm(x_new - x_curr) + + margin, loss = self.margin_and_loss(x_new, y_curr) + + # update loss if new loss is better + idx_improved = (loss < loss_min_curr).float() + + loss_min[idx_to_fool] = idx_improved * loss + ( + 1. - idx_improved) * loss_min_curr + + # update margin and x_best if new loss is better + # or misclassification + idx_miscl = (margin <= 0.).float() + idx_improved = torch.max(idx_improved, idx_miscl) + + margin_min[idx_to_fool] = idx_improved * margin + ( + 1. - idx_improved) * margin_min_curr + idx_improved = idx_improved.reshape([-1, + *[1]*len(x.shape[:-1])]) + x_best[idx_to_fool] = idx_improved * x_new + ( + 1. - idx_improved) * x_best_curr + n_queries[idx_to_fool] += 1. + + ind_succ = (margin_min <= 0.).nonzero().squeeze() + if self.verbose and ind_succ.numel() != 0: + print('{}'.format(i_iter + 1), + '- success rate={}/{} ({:.2%})'.format( + ind_succ.numel(), n_ex_total, float( + ind_succ.numel()) / n_ex_total), + '- avg # queries={:.1f}'.format( + n_queries[ind_succ].mean().item()), + '- med # queries={:.1f}'.format( + n_queries[ind_succ].median().item()), + '- loss={:.3f}'.format(loss_min.mean())) + + assert (x_new != x_new).sum() == 0 + assert (x_best != x_best).sum() == 0 + + if ind_succ.numel() == n_ex_total: + break + + return n_queries, x_best + + def perturb(self, x, y=None): + """ + :param x: clean images + :param y: untargeted attack -> clean labels, + if None we use the predicted labels + targeted attack -> target labels, if None random classes, + different from the predicted ones, are sampled + """ + + self.init_hyperparam(x) + + adv = x.clone() + if y is None: + if not self.targeted: + with torch.no_grad(): + output = self.predict(x) + y_pred = output.max(1)[1] + y = y_pred.detach().clone().long().to(self.device) + else: + with torch.no_grad(): + output = self.predict(x) + n_classes = output.shape[-1] + y_pred = output.max(1)[1] + y = self.random_target_classes(y_pred, n_classes) + else: + y = y.detach().clone().long().to(self.device) + + if not self.targeted: + acc = self.predict(x).max(1)[1] == y + else: + acc = self.predict(x).max(1)[1] != y + + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: + ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool = x[ind_to_fool].clone() + y_to_fool = y[ind_to_fool].clone() + + _, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + + output_curr = self.predict(adv_curr) + if not self.targeted: + acc_curr = output_curr.max(1)[1] == y_to_fool + else: + acc_curr = output_curr.max(1)[1] != y_to_fool + ind_curr = (acc_curr == 0).nonzero().squeeze() + + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - robust accuracy: {:.2%}'.format( + counter, acc.float().mean()), + '- cum. time: {:.1f} s'.format( + time.time() - startt)) + + return adv + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/utils_tf.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/utils_tf.py new file mode 100644 index 0000000..f359068 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/utils_tf.py @@ -0,0 +1,104 @@ +import tensorflow as tf +import numpy as np +import torch + +class ModelAdapter(): + def __init__(self, logits, x, y, sess, num_classes=10): + self.logits = logits + self.sess = sess + self.x_input = x + self.y_input = y + self.num_classes = num_classes + + # gradients of logits + if num_classes <= 10: + self.grads = [None] * num_classes + for cl in range(num_classes): + self.grads[cl] = tf.gradients(self.logits[:, cl], self.x_input)[0] + + # cross-entropy loss + self.xent = tf.nn.sparse_softmax_cross_entropy_with_logits( + logits=self.logits, labels=self.y_input) + self.grad_xent = tf.gradients(self.xent, self.x_input)[0] + + # dlr loss + self.dlr = dlr_loss(self.logits, self.y_input, num_classes=self.num_classes) + self.grad_dlr = tf.gradients(self.dlr, self.x_input)[0] + + # targeted dlr loss + self.y_target = tf.placeholder(tf.int64, shape=[None]) + self.dlr_target = dlr_loss_targeted(self.logits, self.y_input, self.y_target, num_classes=self.num_classes) + self.grad_target = tf.gradients(self.dlr_target, self.x_input)[0] + + self.la = tf.placeholder(tf.int64, shape=[None]) + self.la_target = tf.placeholder(tf.int64, shape=[None]) + la_mask = tf.one_hot(self.la, self.num_classes) + la_target_mask = tf.one_hot(self.la_target, self.num_classes) + la_logit = tf.reduce_sum(la_mask * self.logits, axis=1) + la_target_logit = tf.reduce_sum(la_target_mask * self.logits, axis=1) + self.diff_logits = la_target_logit - la_logit + self.grad_diff_logits = tf.gradients(self.diff_logits, self.x_input)[0] + + def predict(self, x): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + y = self.sess.run(self.logits, {self.x_input: x2}) + + return torch.from_numpy(y).cuda() + + def grad_logits(self, x): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + g2 = self.sess.run(self.grads, {self.x_input: x2}) + g2 = np.moveaxis(np.array(g2), 0, 1) + g2 = np.transpose(g2, (0, 1, 4, 2, 3)) + + return torch.from_numpy(g2).cuda() + + def get_grad_diff_logits_target(self, x, y=None, y_target=None): + la = y.cpu().numpy() + la_target = y_target.cpu().numpy() + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + dl, g2 = self.sess.run([self.diff_logits, self.grad_diff_logits], {self.x_input: x2, self.la: la, self.la_target: la_target}) + g2 = np.transpose(np.array(g2), (0, 3, 1, 2)) + + return torch.from_numpy(dl).cuda(), torch.from_numpy(g2).cuda() + + def get_logits_loss_grad_xent(self, x, y): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + y2 = y.clone().cpu().numpy() + logits_val, loss_indiv_val, grad_val = self.sess.run([self.logits, self.xent, self.grad_xent], {self.x_input: x2, self.y_input: y2}) + grad_val = np.moveaxis(grad_val, 3, 1) + + return torch.from_numpy(logits_val).cuda(), torch.from_numpy(loss_indiv_val).cuda(), torch.from_numpy(grad_val).cuda() + + def get_logits_loss_grad_dlr(self, x, y): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + y2 = y.clone().cpu().numpy() + logits_val, loss_indiv_val, grad_val = self.sess.run([self.logits, self.dlr, self.grad_dlr], {self.x_input: x2, self.y_input: y2}) + grad_val = np.moveaxis(grad_val, 3, 1) + + return torch.from_numpy(logits_val).cuda(), torch.from_numpy(loss_indiv_val).cuda(), torch.from_numpy(grad_val).cuda() + + def get_logits_loss_grad_target(self, x, y, y_target): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + y2 = y.clone().cpu().numpy() + y_targ = y_target.clone().cpu().numpy() + logits_val, loss_indiv_val, grad_val = self.sess.run([self.logits, self.dlr_target, self.grad_target], {self.x_input: x2, self.y_input: y2, self.y_target: y_targ}) + grad_val = np.moveaxis(grad_val, 3, 1) + + return torch.from_numpy(logits_val).cuda(), torch.from_numpy(loss_indiv_val).cuda(), torch.from_numpy(grad_val).cuda() + +def dlr_loss(x, y, num_classes=10): + x_sort = tf.contrib.framework.sort(x, axis=1) + y_onehot = tf.one_hot(y, num_classes) + ### TODO: adapt to the case when the point is already misclassified + loss = -(x_sort[:, -1] - x_sort[:, -2]) / (x_sort[:, -1] - x_sort[:, -3] + 1e-12) + + return loss + +def dlr_loss_targeted(x, y, y_target, num_classes=10): + x_sort = tf.contrib.framework.sort(x, axis=1) + y_onehot = tf.one_hot(y, num_classes) + y_target_onehot = tf.one_hot(y_target, num_classes) + loss = -(tf.reduce_sum(x * y_onehot, axis=1) - tf.reduce_sum(x * y_target_onehot, axis=1)) / (x_sort[:, -1] - .5 * x_sort[:, -3] - .5 * x_sort[:, -4] + 1e-12) + + return loss diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/utils_tf2.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/utils_tf2.py new file mode 100644 index 0000000..74473f8 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoattack/utils_tf2.py @@ -0,0 +1,210 @@ +import tensorflow as tf +import numpy as np +import torch + +class ModelAdapter(): + def __init__(self, model, num_classes=10): + """ + Please note that model should be tf.keras model without activation function 'softmax' + """ + self.num_classes = num_classes + self.tf_model = model + self.__check_channel_ordering() + + def __check_channel_ordering(self): + + for L in self.tf_model.layers: + if isinstance(L, tf.keras.layers.Conv2D): + print("[INFO] set data_format = '{:s}'".format(L.data_format)) + self.data_format = L.data_format + return + + print("[INFO] Can not find Conv2D layer") + input_shape = self.tf_model.input_shape + + if input_shape[3] == 3: + print("[INFO] Because detecting input_shape[3] == 3, set data_format = 'channels_last'") + self.data_format = 'channels_last' + + elif input_shape[3] == 1: + print("[INFO] Because detecting input_shape[3] == 1, set data_format = 'channels_last'") + self.data_format = 'channels_last' + + else: + print("[INFO] set data_format = 'channels_first'") + self.data_format = 'channels_first' + + def __get_logits(self, x_input): + logits = self.tf_model(x_input, training=False) + return logits + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_jacobian(self, x_input): + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x_input) + logits = self.__get_logits(x_input) + + jacobian = g.batch_jacobian(logits, x_input) + + if self.data_format == 'channels_last': + jacobian = tf.transpose(jacobian, perm=[0,1,4,2,3]) + + return jacobian + + def __get_xent(self, logits, y_input): + xent = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y_input) + return xent + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_grad_xent(self, x_input, y_input): + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x_input) + logits = self.__get_logits(x_input) + xent = self.__get_xent(logits, y_input) + + grad_xent = g.gradient(xent, x_input) + + return logits, xent, grad_xent + + def __get_dlr(self, logits, y_input): + val_dlr = dlr_loss(logits, y_input, num_classes=self.num_classes) + return val_dlr + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_grad_dlr(self, x_input, y_input): + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x_input) + logits = self.__get_logits(x_input) + val_dlr = self.__get_dlr(logits, y_input) + + grad_dlr = g.gradient(val_dlr, x_input) + + return logits, val_dlr, grad_dlr + + def __get_dlr_target(self, logits, y_input, y_target): + dlr_target = dlr_loss_targeted(logits, y_input, y_target, num_classes=self.num_classes) + return dlr_target + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_grad_dlr_target(self, x_input, y_input, y_target): + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x_input) + logits = self.__get_logits(x_input) + dlr_target = self.__get_dlr_target(logits, y_input, y_target) + + grad_target = g.gradient(dlr_target, x_input) + + return logits, dlr_target, grad_target + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_grad_diff_logits_target(self, x, la, la_target): + la_mask = tf.one_hot(la, self.num_classes) + la_target_mask = tf.one_hot(la_target, self.num_classes) + + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x) + logits = self.__get_logits(x) + difflogits = tf.reduce_sum((la_target_mask - la_mask) * logits, axis=1) + + g2 = g.gradient(difflogits, x) + + return difflogits, g2 + + def predict(self, x): + + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + + y = self.__get_logits(x2).numpy() + + return torch.from_numpy(y).cuda() + + def grad_logits(self, x): + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + g2 = self.__get_jacobian(x2) + + return torch.from_numpy(g2.numpy()).cuda() + + def set_target_class(self, y, y_target): + pass + + def get_grad_diff_logits_target(self, x, y, y_target): + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + la = y.cpu().numpy() + la_target = y_target.cpu().numpy() + + difflogits, g2 = self.__get_grad_diff_logits_target(x2, la, la_target) + + if self.data_format == 'channels_last': + g2 = tf.transpose(g2, perm=[0, 3, 1, 2]) + + return torch.from_numpy(difflogits.numpy()).cuda(), torch.from_numpy(g2.numpy()).cuda() + + def get_logits_loss_grad_xent(self, x, y): + + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + y2 = tf.convert_to_tensor(y.clone().cpu().numpy(), dtype=tf.int32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + + logits_val, loss_indiv_val, grad_val = self.__get_grad_xent(x2, y2) + + if self.data_format == 'channels_last': + grad_val = tf.transpose(grad_val, perm=[0,3,1,2]) + + return torch.from_numpy(logits_val.numpy()).cuda(), torch.from_numpy(loss_indiv_val.numpy()).cuda(), torch.from_numpy(grad_val.numpy()).cuda() + + def get_logits_loss_grad_dlr(self, x, y): + + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + y2 = tf.convert_to_tensor(y.clone().cpu().numpy(), dtype=tf.int32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + + logits_val, loss_indiv_val, grad_val = self.__get_grad_dlr(x2, y2) + + if self.data_format == 'channels_last': + grad_val = tf.transpose(grad_val, perm=[0,3,1,2]) + + return torch.from_numpy(logits_val.numpy()).cuda(), torch.from_numpy(loss_indiv_val.numpy()).cuda(), torch.from_numpy(grad_val.numpy()).cuda() + + def get_logits_loss_grad_target(self, x, y, y_target): + + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + y2 = tf.convert_to_tensor(y.clone().cpu().numpy(), dtype=tf.int32) + y_targ = tf.convert_to_tensor(y_target.clone().cpu().numpy(), dtype=tf.int32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + + logits_val, loss_indiv_val, grad_val = self.__get_grad_dlr_target(x2, y2, y_targ) + + if self.data_format == 'channels_last': + grad_val = tf.transpose(grad_val, perm=[0,3,1,2]) + + return torch.from_numpy(logits_val.numpy()).cuda(), torch.from_numpy(loss_indiv_val.numpy()).cuda(), torch.from_numpy(grad_val.numpy()).cuda() + +def dlr_loss(x, y, num_classes=10): + x_sort = tf.sort(x, axis=1) + y_onehot = tf.one_hot(y, num_classes) + ### TODO: adapt to the case when the point is already misclassified + loss = -(x_sort[:, -1] - x_sort[:, -2]) / (x_sort[:, -1] - x_sort[:, -3] + 1e-12) + + return loss + +def dlr_loss_targeted(x, y, y_target, num_classes=10): + x_sort = tf.sort(x, axis=1) + y_onehot = tf.one_hot(y, num_classes) + y_target_onehot = tf.one_hot(y_target, num_classes) + loss = -(tf.reduce_sum(x * y_onehot, axis=1) - tf.reduce_sum(x * y_target_onehot, axis=1)) / (x_sort[:, -1] - .5 * x_sort[:, -3] - .5 * x_sort[:, -4] + 1e-12) + + return loss diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/autoaugment.py b/Robustness_Eval/GAIRAT/GAIR_RST/autoaugment.py new file mode 100644 index 0000000..ff5f01a --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/autoaugment.py @@ -0,0 +1,243 @@ +""" +AutoAugment implementation from https://github.com/DeepVoltaire/AutoAugment +""" + +from PIL import Image, ImageEnhance, ImageOps +import numpy as np +import random + +# TODO: add credit + + +class ImageNetPolicy(object): + """ Randomly choose one of the best 24 Sub-policies on ImageNet. + + Example: + >>> policy = ImageNetPolicy() + >>> transformed = policy(image) + + Example as a PyTorch Transform: + >>> transform=transforms.Compose([ + >>> transforms.Resize(256), + >>> ImageNetPolicy(), + >>> transforms.ToTensor()]) + """ + def __init__(self, fillcolor=(128, 128, 128)): + self.policies = [ + SubPolicy(0.4, "posterize", 8, 0.6, "rotate", 9, fillcolor), + SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), + SubPolicy(0.8, "equalize", 8, 0.6, "equalize", 3, fillcolor), + SubPolicy(0.6, "posterize", 7, 0.6, "posterize", 6, fillcolor), + SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), + + SubPolicy(0.4, "equalize", 4, 0.8, "rotate", 8, fillcolor), + SubPolicy(0.6, "solarize", 3, 0.6, "equalize", 7, fillcolor), + SubPolicy(0.8, "posterize", 5, 1.0, "equalize", 2, fillcolor), + SubPolicy(0.2, "rotate", 3, 0.6, "solarize", 8, fillcolor), + SubPolicy(0.6, "equalize", 8, 0.4, "posterize", 6, fillcolor), + + SubPolicy(0.8, "rotate", 8, 0.4, "color", 0, fillcolor), + SubPolicy(0.4, "rotate", 9, 0.6, "equalize", 2, fillcolor), + SubPolicy(0.0, "equalize", 7, 0.8, "equalize", 8, fillcolor), + SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), + SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor), + + SubPolicy(0.8, "rotate", 8, 1.0, "color", 2, fillcolor), + SubPolicy(0.8, "color", 8, 0.8, "solarize", 7, fillcolor), + SubPolicy(0.4, "sharpness", 7, 0.6, "invert", 8, fillcolor), + SubPolicy(0.6, "shearX", 5, 1.0, "equalize", 9, fillcolor), + SubPolicy(0.4, "color", 0, 0.6, "equalize", 3, fillcolor), + + SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), + SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), + SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), + SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor) + ] + + + def __call__(self, img): + policy_idx = random.randint(0, len(self.policies) - 1) + return self.policies[policy_idx](img) + + def __repr__(self): + return "AutoAugment ImageNet Policy" + + +class CIFAR10Policy(object): + """ Randomly choose one of the best 25 Sub-policies on CIFAR10. + + Example: + >>> policy = CIFAR10Policy() + >>> transformed = policy(image) + + Example as a PyTorch Transform: + >>> transform=transforms.Compose([ + >>> transforms.Resize(256), + >>> CIFAR10Policy(), + >>> transforms.ToTensor()]) + """ + def __init__(self, fillcolor=(128, 128, 128)): + self.policies = [ + SubPolicy(0.1, "invert", 7, 0.2, "contrast", 6, fillcolor), + SubPolicy(0.7, "rotate", 2, 0.3, "translateX", 9, fillcolor), + SubPolicy(0.8, "sharpness", 1, 0.9, "sharpness", 3, fillcolor), + SubPolicy(0.5, "shearY", 8, 0.7, "translateY", 9, fillcolor), + SubPolicy(0.5, "autocontrast", 8, 0.9, "equalize", 2, fillcolor), + + SubPolicy(0.2, "shearY", 7, 0.3, "posterize", 7, fillcolor), + SubPolicy(0.4, "color", 3, 0.6, "brightness", 7, fillcolor), + SubPolicy(0.3, "sharpness", 9, 0.7, "brightness", 9, fillcolor), + SubPolicy(0.6, "equalize", 5, 0.5, "equalize", 1, fillcolor), + SubPolicy(0.6, "contrast", 7, 0.6, "sharpness", 5, fillcolor), + + SubPolicy(0.7, "color", 7, 0.5, "translateX", 8, fillcolor), + SubPolicy(0.3, "equalize", 7, 0.4, "autocontrast", 8, fillcolor), + SubPolicy(0.4, "translateY", 3, 0.2, "sharpness", 6, fillcolor), + SubPolicy(0.9, "brightness", 6, 0.2, "color", 8, fillcolor), + SubPolicy(0.5, "solarize", 2, 0.0, "invert", 3, fillcolor), + + SubPolicy(0.2, "equalize", 0, 0.6, "autocontrast", 0, fillcolor), + SubPolicy(0.2, "equalize", 8, 0.8, "equalize", 4, fillcolor), + SubPolicy(0.9, "color", 9, 0.6, "equalize", 6, fillcolor), + SubPolicy(0.8, "autocontrast", 4, 0.2, "solarize", 8, fillcolor), + SubPolicy(0.1, "brightness", 3, 0.7, "color", 0, fillcolor), + + SubPolicy(0.4, "solarize", 5, 0.9, "autocontrast", 3, fillcolor), + SubPolicy(0.9, "translateY", 9, 0.7, "translateY", 9, fillcolor), + SubPolicy(0.9, "autocontrast", 2, 0.8, "solarize", 3, fillcolor), + SubPolicy(0.8, "equalize", 8, 0.1, "invert", 3, fillcolor), + SubPolicy(0.7, "translateY", 9, 0.9, "autocontrast", 1, fillcolor) + ] + + + def __call__(self, img): + policy_idx = random.randint(0, len(self.policies) - 1) + return self.policies[policy_idx](img) + + def __repr__(self): + return "AutoAugment CIFAR10 Policy" + + +class SVHNPolicy(object): + """ Randomly choose one of the best 25 Sub-policies on SVHN. + + Example: + >>> policy = SVHNPolicy() + >>> transformed = policy(image) + + Example as a PyTorch Transform: + >>> transform=transforms.Compose([ + >>> transforms.Resize(256), + >>> SVHNPolicy(), + >>> transforms.ToTensor()]) + """ + def __init__(self, fillcolor=(128, 128, 128)): + self.policies = [ + SubPolicy(0.9, "shearX", 4, 0.2, "invert", 3, fillcolor), + SubPolicy(0.9, "shearY", 8, 0.7, "invert", 5, fillcolor), + SubPolicy(0.6, "equalize", 5, 0.6, "solarize", 6, fillcolor), + SubPolicy(0.9, "invert", 3, 0.6, "equalize", 3, fillcolor), + SubPolicy(0.6, "equalize", 1, 0.9, "rotate", 3, fillcolor), + + SubPolicy(0.9, "shearX", 4, 0.8, "autocontrast", 3, fillcolor), + SubPolicy(0.9, "shearY", 8, 0.4, "invert", 5, fillcolor), + SubPolicy(0.9, "shearY", 5, 0.2, "solarize", 6, fillcolor), + SubPolicy(0.9, "invert", 6, 0.8, "autocontrast", 1, fillcolor), + SubPolicy(0.6, "equalize", 3, 0.9, "rotate", 3, fillcolor), + + SubPolicy(0.9, "shearX", 4, 0.3, "solarize", 3, fillcolor), + SubPolicy(0.8, "shearY", 8, 0.7, "invert", 4, fillcolor), + SubPolicy(0.9, "equalize", 5, 0.6, "translateY", 6, fillcolor), + SubPolicy(0.9, "invert", 4, 0.6, "equalize", 7, fillcolor), + SubPolicy(0.3, "contrast", 3, 0.8, "rotate", 4, fillcolor), + + SubPolicy(0.8, "invert", 5, 0.0, "translateY", 2, fillcolor), + SubPolicy(0.7, "shearY", 6, 0.4, "solarize", 8, fillcolor), + SubPolicy(0.6, "invert", 4, 0.8, "rotate", 4, fillcolor), + SubPolicy(0.3, "shearY", 7, 0.9, "translateX", 3, fillcolor), + SubPolicy(0.1, "shearX", 6, 0.6, "invert", 5, fillcolor), + + SubPolicy(0.7, "solarize", 2, 0.6, "translateY", 7, fillcolor), + SubPolicy(0.8, "shearY", 4, 0.8, "invert", 8, fillcolor), + SubPolicy(0.7, "shearX", 9, 0.8, "translateY", 3, fillcolor), + SubPolicy(0.8, "shearY", 5, 0.7, "autocontrast", 3, fillcolor), + SubPolicy(0.7, "shearX", 2, 0.1, "invert", 5, fillcolor) + ] + + + def __call__(self, img): + policy_idx = random.randint(0, len(self.policies) - 1) + return self.policies[policy_idx](img) + + def __repr__(self): + return "AutoAugment SVHN Policy" + + +class SubPolicy(object): + def __init__(self, p1, operation1, magnitude_idx1, p2, operation2, magnitude_idx2, fillcolor=(128, 128, 128)): + ranges = { + "shearX": np.linspace(0, 0.3, 10), + "shearY": np.linspace(0, 0.3, 10), + "translateX": np.linspace(0, 150 / 331, 10), + "translateY": np.linspace(0, 150 / 331, 10), + "rotate": np.linspace(0, 30, 10), + "color": np.linspace(0.0, 0.9, 10), + "posterize": np.round(np.linspace(8, 4, 10), 0).astype(np.int), + "solarize": np.linspace(256, 0, 10), + "contrast": np.linspace(0.0, 0.9, 10), + "sharpness": np.linspace(0.0, 0.9, 10), + "brightness": np.linspace(0.0, 0.9, 10), + "autocontrast": [0] * 10, + "equalize": [0] * 10, + "invert": [0] * 10 + } + + # from https://stackoverflow.com/questions/5252170/specify-image-filling-color-when-rotating-in-python-with-pil-and-setting-expand + def rotate_with_fill(img, magnitude): + rot = img.convert("RGBA").rotate(magnitude) + return Image.composite(rot, Image.new("RGBA", rot.size, (128,) * 4), rot).convert(img.mode) + + func = { + "shearX": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0), + Image.BICUBIC, fillcolor=fillcolor), + "shearY": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0), + Image.BICUBIC, fillcolor=fillcolor), + "translateX": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, 0, magnitude * img.size[0] * random.choice([-1, 1]), 0, 1, 0), + fillcolor=fillcolor), + "translateY": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * img.size[1] * random.choice([-1, 1])), + fillcolor=fillcolor), + "rotate": lambda img, magnitude: rotate_with_fill(img, magnitude), + # "rotate": lambda img, magnitude: img.rotate(magnitude * random.choice([-1, 1])), + "color": lambda img, magnitude: ImageEnhance.Color(img).enhance(1 + magnitude * random.choice([-1, 1])), + "posterize": lambda img, magnitude: ImageOps.posterize(img, magnitude), + "solarize": lambda img, magnitude: ImageOps.solarize(img, magnitude), + "contrast": lambda img, magnitude: ImageEnhance.Contrast(img).enhance( + 1 + magnitude * random.choice([-1, 1])), + "sharpness": lambda img, magnitude: ImageEnhance.Sharpness(img).enhance( + 1 + magnitude * random.choice([-1, 1])), + "brightness": lambda img, magnitude: ImageEnhance.Brightness(img).enhance( + 1 + magnitude * random.choice([-1, 1])), + "autocontrast": lambda img, magnitude: ImageOps.autocontrast(img), + "equalize": lambda img, magnitude: ImageOps.equalize(img), + "invert": lambda img, magnitude: ImageOps.invert(img) + } + + # self.name = "{}_{:.2f}_and_{}_{:.2f}".format( + # operation1, ranges[operation1][magnitude_idx1], + # operation2, ranges[operation2][magnitude_idx2]) + self.p1 = p1 + self.operation1 = func[operation1] + self.magnitude1 = ranges[operation1][magnitude_idx1] + self.p2 = p2 + self.operation2 = func[operation2] + self.magnitude2 = ranges[operation2][magnitude_idx2] + + + def __call__(self, img): + if random.random() < self.p1: img = self.operation1(img, self.magnitude1) + if random.random() < self.p2: img = self.operation2(img, self.magnitude2) + return img \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/cutout.py b/Robustness_Eval/GAIRAT/GAIR_RST/cutout.py new file mode 100644 index 0000000..2606903 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/cutout.py @@ -0,0 +1,50 @@ +""" +Cutout augmentation implementation. Code taken from +https://github.com/uoguelph-mlrg/Cutout/blob/master/util/cutout.py +""" + +import torch +import numpy as np + +# TODO: add credit + + +class Cutout(object): + """Randomly mask out one or more patches from an image. + + Args: + n_holes (int): Number of patches to cut out of each image. + length (int): The length (in pixels) of each square patch. + """ + def __init__(self, n_holes, length): + self.n_holes = n_holes + self.length = length + + def __call__(self, img): + """ + Args: + img (Tensor): Tensor image of size (C, H, W). + Returns: + Tensor: Image with n_holes of dimension length x length cut out of it. + """ + h = img.size(1) + w = img.size(2) + + mask = np.ones((h, w), np.float32) + + for n in range(self.n_holes): + y = np.random.randint(h) + x = np.random.randint(w) + + y1 = np.clip(y - self.length // 2, 0, h) + y2 = np.clip(y + self.length // 2, 0, h) + x1 = np.clip(x - self.length // 2, 0, w) + x2 = np.clip(x + self.length // 2, 0, w) + + mask[y1: y2, x1: x2] = 0. + + mask = torch.from_numpy(mask) + mask = mask.expand_as(img) + img = img * mask + + return img diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/datasets.py b/Robustness_Eval/GAIRAT/GAIR_RST/datasets.py new file mode 100644 index 0000000..7ca7f7a --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/datasets.py @@ -0,0 +1,212 @@ +""" +Datasets with unlabeled (or pseudo-labeled) data +""" + +from torchvision.datasets import CIFAR10, SVHN +from torch.utils.data import Sampler, Dataset +import torch +import numpy as np + +import os +import pickle + +import logging + +DATASETS = ['cifar10', 'svhn'] + + +class SemiSupervisedDataset(Dataset): + def __init__(self, + base_dataset='cifar10', + take_amount=None, + take_amount_seed=13, + add_svhn_extra=False, + aux_data_filename=None, + add_aux_labels=False, + aux_take_amount=None, + train=False, + **kwargs): + """A dataset with auxiliary pseudo-labeled data""" + + if base_dataset == 'cifar10': + self.dataset = CIFAR10(train=train, **kwargs) + elif base_dataset == 'svhn': + if train: + self.dataset = SVHN(split='train', **kwargs) + else: + self.dataset = SVHN(split='test', **kwargs) + # because torchvision is annoying + self.dataset.targets = self.dataset.labels + self.targets = list(self.targets) + + if train and add_svhn_extra: + svhn_extra = SVHN(split='extra', **kwargs) + self.data = np.concatenate([self.data, svhn_extra.data]) + self.targets.extend(svhn_extra.labels) + else: + raise ValueError('Dataset %s not supported' % base_dataset) + self.base_dataset = base_dataset + self.train = train + + if self.train: + if take_amount is not None: + rng_state = np.random.get_state() + np.random.seed(take_amount_seed) + take_inds = np.random.choice(len(self.sup_indices), + take_amount, replace=False) + np.random.set_state(rng_state) + + logger = logging.getLogger() + logger.info('Randomly taking only %d/%d examples from training' + ' set, seed=%d, indices=%s', + take_amount, len(self.sup_indices), + take_amount_seed, take_inds) + self.targets = self.targets[take_inds] + self.data = self.data[take_inds] + + self.sup_indices = list(range(len(self.targets))) + self.unsup_indices = [] + + if aux_data_filename is not None: + aux_path = os.path.join(kwargs['root'], aux_data_filename) + aux_path = os.path.expanduser(aux_path) + print("Loading data from %s" % aux_path) + with open(aux_path, 'rb') as f: + aux = pickle.load(f) + aux_data = aux['data'] + aux_targets = aux['extrapolated_targets'] + orig_len = len(self.data) + + if aux_take_amount is not None: + rng_state = np.random.get_state() + np.random.seed(take_amount_seed) + take_inds = np.random.choice(len(aux_data), + aux_take_amount, replace=False) + np.random.set_state(rng_state) + + logger = logging.getLogger() + logger.info( + 'Randomly taking only %d/%d examples from aux data' + ' set, seed=%d, indices=%s', + aux_take_amount, len(aux_data), + take_amount_seed, take_inds) + aux_data = aux_data[take_inds] + aux_targets = aux_targets[take_inds] + + self.data = np.concatenate((self.data, aux_data), axis=0) + + if not add_aux_labels: + self.targets.extend([-1] * len(aux_data)) + else: + self.targets.extend(aux_targets) + # note that we use unsup indices to track the labeled datapoints + # whose labels are "fake" + self.unsup_indices.extend( + range(orig_len, orig_len+len(aux_data))) + + logger = logging.getLogger() + logger.info("Training set") + logger.info("Number of training samples: %d", len(self.targets)) + logger.info("Number of supervised samples: %d", + len(self.sup_indices)) + logger.info("Number of unsup samples: %d", len(self.unsup_indices)) + logger.info("Label (and pseudo-label) histogram: %s", + tuple( + zip(*np.unique(self.targets, return_counts=True)))) + logger.info("Shape of training data: %s", np.shape(self.data)) + + # Test set + else: + self.sup_indices = list(range(len(self.targets))) + self.unsup_indices = [] + + logger = logging.getLogger() + logger.info("Test set") + logger.info("Number of samples: %d", len(self.targets)) + logger.info("Label histogram: %s", + tuple( + zip(*np.unique(self.targets, return_counts=True)))) + logger.info("Shape of data: %s", np.shape(self.data)) + + @property + def data(self): + return self.dataset.data + + @data.setter + def data(self, value): + self.dataset.data = value + + @property + def targets(self): + return self.dataset.targets + + @targets.setter + def targets(self, value): + self.dataset.targets = value + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, item): + self.dataset.labels = self.targets # because torchvision is annoying + return self.dataset[item] + + def __repr__(self): + fmt_str = 'Semisupervised Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Training: {}\n'.format(self.train) + fmt_str += ' Root Location: {}\n'.format(self.dataset.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.dataset.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.dataset.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + +class SemiSupervisedSampler(Sampler): + """Balanced sampling from the labeled and unlabeled data""" + def __init__(self, sup_inds, unsup_inds, batch_size, unsup_fraction=0.5, + num_batches=None): + if unsup_fraction is None or unsup_fraction < 0: + self.sup_inds = sup_inds + unsup_inds + unsup_fraction = 0.0 + else: + self.sup_inds = sup_inds + self.unsup_inds = unsup_inds + + self.batch_size = batch_size + unsup_batch_size = int(batch_size * unsup_fraction) + self.sup_batch_size = batch_size - unsup_batch_size + + if num_batches is not None: + self.num_batches = num_batches + else: + self.num_batches = int( + np.ceil(len(self.sup_inds) / self.sup_batch_size)) + + super().__init__(None) + + def __iter__(self): + batch_counter = 0 + while batch_counter < self.num_batches: + sup_inds_shuffled = [self.sup_inds[i] + for i in torch.randperm(len(self.sup_inds))] + for sup_k in range(0, len(self.sup_inds), self.sup_batch_size): + if batch_counter == self.num_batches: + break + batch = sup_inds_shuffled[sup_k:(sup_k + self.sup_batch_size)] + if self.sup_batch_size < self.batch_size: + batch.extend([self.unsup_inds[i] for i in + torch.randint(high=len(self.unsup_inds), + size=( + self.batch_size - len( + batch),), + dtype=torch.int64)]) + # this shuffle operation is very important, without it + # batch-norm / DataParallel hell ensues + np.random.shuffle(batch) + yield batch + batch_counter += 1 + + def __len__(self): + return self.num_batches diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/losses.py b/Robustness_Eval/GAIRAT/GAIR_RST/losses.py new file mode 100644 index 0000000..f1cbb8e --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/losses.py @@ -0,0 +1,262 @@ +""" +Robust training losses. Based on code from +https://github.com/yaodongyu/TRADES +""" + +import contextlib +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +import numpy as np + +import pdb + + +def entropy_loss(unlabeled_logits): + unlabeled_probs = F.softmax(unlabeled_logits, dim=1) + return -(unlabeled_probs * F.log_softmax(unlabeled_logits, dim=1)).sum( + dim=1).mean(dim=0) + +def cwloss(output, target,confidence=50, num_classes=10): + # Compute the probability of the label class versus the maximum other + # The same implementation as in repo CAT https://github.com/sunblaze-ucb/curriculum-adversarial-training-CAT + target = target.data + target_onehot = torch.zeros(target.size() + (num_classes,)) + target_onehot = target_onehot.cuda() + target_onehot.scatter_(1, target.unsqueeze(1), 1.) + target_var = Variable(target_onehot, requires_grad=False) + real = (target_var * output).sum(1) + other = ((1. - target_var) * output - target_var * 10000.).max(1)[0] + loss = -torch.clamp(real - other + confidence, min=0.) # equiv to max(..., 0.) + loss = torch.sum(loss) + return loss + +def pgd_generate(model, data, target, epsilon, step_size, num_steps,loss_fn,category,rand_init): + model.eval() + pgd_steps = torch.zeros(len(data)) + if category == "trades": + x_adv = data.detach() + 0.001 * torch.randn(data.shape).cuda().detach() if rand_init else data.detach() + nat_output = model(data) + if category == "Madry": + x_adv = data.detach() + torch.from_numpy(np.random.uniform(-epsilon, epsilon, data.shape)).float().cuda() if rand_init else data.detach() + x_adv = torch.clamp(x_adv, 0.0, 1.0) + for k in range(num_steps): + x_adv.requires_grad_() + output = model(x_adv) + #nat_output = model(data) + predict = output.max(1, keepdim=True)[1] + for p in range(len(x_adv)): + if predict[p] == target[p]: + pgd_steps[p] += 1 + model.zero_grad() + with torch.enable_grad(): + if loss_fn == "cent": + loss_adv = nn.CrossEntropyLoss(reduction="mean")(output, target) + if loss_fn == "cw": + loss_adv = cwloss(output,target) + if loss_fn == "kl": + criterion_kl = nn.KLDivLoss(size_average=False).cuda() + loss_adv = criterion_kl(F.log_softmax(output, dim=1),F.softmax(nat_output, dim=1)) + loss_adv.backward() + eta = step_size * x_adv.grad.sign() + x_adv = x_adv.detach() + eta + x_adv = torch.min(torch.max(x_adv, data - epsilon), data + epsilon) + x_adv = torch.clamp(x_adv, 0.0, 1.0) + x_adv = Variable(x_adv, requires_grad=False) + return x_adv, pgd_steps + + +def trades_loss(model, + x_natural, + y, + optimizer, + step_size=0.003, + epsilon=0.031, + perturb_steps=10, + beta=1.0, + adversarial=True, + distance='inf', + entropy_weight=0): + """The TRADES KL-robustness regularization term proposed by + Zhang et al., with added support for stability training and entropy + regularization""" + if beta == 0: + logits = model(x_natural) + loss = F.cross_entropy(logits, y) + inf = torch.Tensor([np.inf]) + zero = torch.Tensor([0.]) + return loss, loss, inf, zero + + # define KL-loss + criterion_kl = nn.KLDivLoss(reduction='sum') + model.eval() # moving to eval mode to freeze batchnorm stats + batch_size = len(x_natural) + # generate adversarial example + x_adv = x_natural.detach() + 0. # the + 0. is for copying the tensor + if adversarial: + if distance == 'l_inf': + x_adv += 0.001 * torch.randn(x_natural.shape).cuda().detach() + + for _ in range(perturb_steps): + x_adv.requires_grad_() + with torch.enable_grad(): + loss_kl = criterion_kl(F.log_softmax(model(x_adv), dim=1), + F.softmax(model(x_natural), dim=1)) + grad = torch.autograd.grad(loss_kl, [x_adv])[0] + x_adv = x_adv.detach() + step_size * torch.sign(grad.detach()) + x_adv = torch.min(torch.max(x_adv, x_natural - epsilon), + x_natural + epsilon) + x_adv = torch.clamp(x_adv, 0.0, 1.0) + else: + raise ValueError('No support for distance %s in adversarial ' + 'training' % distance) + else: + if distance == 'l_2': + x_adv = x_adv + epsilon * torch.randn_like(x_adv) + else: + raise ValueError('No support for distance %s in stability ' + 'training' % distance) + + model.train() # moving to train mode to update batchnorm stats + + # zero gradient + optimizer.zero_grad() + + x_adv = Variable(torch.clamp(x_adv, 0.0, 1.0), requires_grad=False) + logits_adv = F.log_softmax(model(x_adv), dim=1) + logits = model(x_natural) + + loss_natural = F.cross_entropy(logits, y, ignore_index=-1) + p_natural = F.softmax(logits, dim=1) + loss_robust = criterion_kl( + logits_adv, p_natural) / batch_size + + loss = loss_natural + beta * loss_robust + + is_unlabeled = (y == -1) + if torch.sum(is_unlabeled) > 0: + logits_unlabeled = logits[is_unlabeled] + loss_entropy_unlabeled = entropy_loss(logits_unlabeled) + loss = loss + entropy_weight * loss_entropy_unlabeled + else: + loss_entropy_unlabeled = torch.tensor(0) + + return loss, loss_natural, loss_robust, loss_entropy_unlabeled + +def GAIR_trades_loss(model, + x_natural, + y, + optimizer, + step_size=0.003, + epsilon=0.031, + perturb_steps=10, + beta=1.0, + adversarial=True, + distance='inf', + entropy_weight=0, + Lambda=-1.0): + """The TRADES KL-robustness regularization term proposed by + Zhang et al., with added support for stability training and entropy + regularization""" + Kappa = torch.zeros(len(x_natural)) + + if beta == 0: + logits = model(x_natural) + loss = F.cross_entropy(logits, y) + inf = torch.Tensor([np.inf]) + zero = torch.Tensor([0.]) + return loss, loss, inf, zero + + # define KL-loss + criterion_kl = nn.KLDivLoss(reduction="sum") + model.eval() # moving to eval mode to freeze batchnorm stats + batch_size = len(x_natural) + # generate adversarial example + x_adv = x_natural.detach() + 0. # the + 0. is for copying the tensor + if adversarial: + if distance == 'l_inf': + x_adv += 0.001 * torch.randn(x_natural.shape).cuda().detach() + + for _ in range(perturb_steps): + x_adv.requires_grad_() + mid_logit = model(x_adv) + mid_predict = mid_logit.max(1, keepdim=True)[1] + # Update Kappa + for p in range(len(x_adv)): + if mid_predict[p] == y[p]: + Kappa[p] += 1 + with torch.enable_grad(): + loss_kl = criterion_kl(F.log_softmax(model(x_adv), dim=1), + F.softmax(model(x_natural), dim=1)) + grad = torch.autograd.grad(loss_kl, [x_adv])[0] + x_adv = x_adv.detach() + step_size * torch.sign(grad.detach()) + x_adv = torch.min(torch.max(x_adv, x_natural - epsilon), + x_natural + epsilon) + x_adv = torch.clamp(x_adv, 0.0, 1.0) + else: + raise ValueError('No support for distance %s in adversarial ' + 'training' % distance) + else: + if distance == 'l_2': + x_adv = x_adv + epsilon * torch.randn_like(x_adv) + else: + raise ValueError('No support for distance %s in stability ' + 'training' % distance) + + if Lambda <= 10.0: + cw_adv, _ = pgd_generate(model,x_natural, y, epsilon, step_size, perturb_steps, loss_fn="cw",category="trades", rand_init=True) + + + Kappa = Kappa.cuda() + model.train() # moving to train mode to update batchnorm stats + + # zero gradient + optimizer.zero_grad() + + x_adv = Variable(torch.clamp(x_adv, 0.0, 1.0), requires_grad=False) + logits_adv = F.log_softmax(model(x_adv), dim=1) + if Lambda <= 10.0: + logits_cw = F.log_softmax(model(cw_adv),dim=1) + logits = model(x_natural) + + loss_natural = F.cross_entropy(logits, y, ignore_index=-1, reduction='none') + + loss_natural = loss_natural.mean() + p_natural = F.softmax(logits, dim=1) + + if Lambda <= 10.0: + criterion_kl = nn.KLDivLoss(reduce=False).cuda() + loss_robust = (criterion_kl(logits_adv, p_natural).sum(dim=1).mul(((Lambda+((perturb_steps/2)-Kappa)).tanh()+1)/2)+criterion_kl(logits_cw, p_natural).sum(dim=1).mul(1-((Lambda+((perturb_steps/2)-Kappa)).tanh()+1)/2)).sum() + loss_robust = loss_robust / batch_size + else: + loss_robust = criterion_kl( + logits_adv, p_natural) / batch_size + + loss = loss_natural + beta * loss_robust + + is_unlabeled = (y == -1) + if torch.sum(is_unlabeled) > 0: + logits_unlabeled = logits[is_unlabeled] + loss_entropy_unlabeled = entropy_loss(logits_unlabeled) + loss = loss + entropy_weight * loss_entropy_unlabeled + else: + loss_entropy_unlabeled = torch.tensor(0) + + return loss, loss_natural, loss_robust, loss_entropy_unlabeled + +def noise_loss(model, + x_natural, + y, + epsilon=0.25, + clamp_x=True): + """Augmenting the input with random noise as in Cohen et al.""" + # logits_natural = model(x_natural) + x_noise = x_natural + epsilon * torch.randn_like(x_natural) + if clamp_x: + x_noise = x_noise.clamp(0.0, 1.0) + logits_noise = model(x_noise) + loss = F.cross_entropy(logits_noise, y, ignore_index=-1) + return loss + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/models/__init__.py b/Robustness_Eval/GAIRAT/GAIR_RST/models/__init__.py new file mode 100644 index 0000000..289333a --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/models/__init__.py @@ -0,0 +1,8 @@ +from . import cifar_resnet, wideresnet, shake_shake, resnet + +__all__ = [ + 'cifar_resnet', + 'wideresnet', + 'shake_shake', + 'resnet', +] \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/models/cifar_resnet.py b/Robustness_Eval/GAIRAT/GAIR_RST/models/cifar_resnet.py new file mode 100644 index 0000000..e3de21d --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/models/cifar_resnet.py @@ -0,0 +1,163 @@ +from __future__ import absolute_import + +''' +This file is from: https://raw.githubusercontent.com/bearpaw/pytorch-classification/master/models/cifar/resnet.py +by Wei Yang +''' +import torch.nn as nn +import math + + +# __all__ = ['resnet'] + +def conv3x3(in_planes, out_planes, stride=1): + "3x3 convolution with padding" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * 4) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, depth, num_classes=1000, block_name='BasicBlock'): + super(ResNet, self).__init__() + # Model type specifies number of layers for CIFAR-10 model + if block_name.lower() == 'basicblock': + assert (depth - 2) % 6 == 0, 'When use basicblock, depth should be 6n+2, e.g. 20, 32, 44, 56, 110, 1202' + n = (depth - 2) // 6 + block = BasicBlock + elif block_name.lower() == 'bottleneck': + assert (depth - 2) % 9 == 0, 'When use bottleneck, depth should be 9n+2, e.g. 20, 29, 47, 56, 110, 1199' + n = (depth - 2) // 9 + block = Bottleneck + else: + raise ValueError('block_name shoule be Basicblock or Bottleneck') + + + self.inplanes = 16 + self.conv1 = nn.Conv2d(3, 16, kernel_size=3, padding=1, + bias=False) + self.bn1 = nn.BatchNorm2d(16) + self.relu = nn.ReLU(inplace=True) + self.layer1 = self._make_layer(block, 16, n) + self.layer2 = self._make_layer(block, 32, n, stride=2) + self.layer3 = self._make_layer(block, 64, n, stride=2) + self.avgpool = nn.AvgPool2d(8) + self.fc = nn.Linear(64 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, + kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) # 32x32 + + x = self.layer1(x) # 32x32 + x = self.layer2(x) # 16x16 + x = self.layer3(x) # 8x8 + + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + + return x + + +# def resnet(**kwargs): +# """ +# Constructs a ResNet model. +# """ +# return ResNet(**kwargs) \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/models/resnet.py b/Robustness_Eval/GAIRAT/GAIR_RST/models/resnet.py new file mode 100644 index 0000000..b143d3c --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/models/resnet.py @@ -0,0 +1,123 @@ +'''ResNet in PyTorch. + +For Pre-activation ResNet, see 'preact_resnet.py'. + +Reference: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 +''' +import torch +import torch.nn as nn +import torch.nn.functional as F + +from torch.autograd import Variable + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion*planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512*block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def ResNet18(): + return ResNet(BasicBlock, [2,2,2,2]) + +def ResNet34(): + return ResNet(BasicBlock, [3,4,6,3]) + +def ResNet50(): + return ResNet(Bottleneck, [3,4,6,3]) + +def ResNet101(): + return ResNet(Bottleneck, [3,4,23,3]) + +def ResNet152(): + return ResNet(Bottleneck, [3,8,36,3]) + + +def test(): + net = ResNet18() + y = net(Variable(torch.randn(1,3,32,32))) + print(y.size()) + print(net) +# test() \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/models/shake_shake.py b/Robustness_Eval/GAIRAT/GAIR_RST/models/shake_shake.py new file mode 100644 index 0000000..df3490a --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/models/shake_shake.py @@ -0,0 +1,200 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .shake_shake_function import get_alpha_beta, shake_function + +""" +Based on code from https://github.com/hysts/pytorch_shake_shake +""" + +def initialize_weights(module): + if isinstance(module, nn.Conv2d): + nn.init.kaiming_normal_(module.weight.data, mode='fan_out') + elif isinstance(module, nn.BatchNorm2d): + module.weight.data.fill_(1) + module.bias.data.zero_() + elif isinstance(module, nn.Linear): + module.bias.data.zero_() + + +class ResidualPath(nn.Module): + def __init__(self, in_channels, out_channels, stride): + super(ResidualPath, self).__init__() + + self.conv1 = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=False, + ) + self.bn1 = nn.BatchNorm2d(out_channels) + self.conv2 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(out_channels) + + def forward(self, x): + x = F.relu(x, inplace=False) + x = F.relu(self.bn1(self.conv1(x)), inplace=False) + x = self.bn2(self.conv2(x)) + return x + + +class DownsamplingShortcut(nn.Module): + def __init__(self, in_channels): + super(DownsamplingShortcut, self).__init__() + self.conv1 = nn.Conv2d( + in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.conv2 = nn.Conv2d( + in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn = nn.BatchNorm2d(in_channels * 2) + + def forward(self, x): + x = F.relu(x, inplace=False) + y1 = F.avg_pool2d(x, kernel_size=1, stride=2, padding=0) + y1 = self.conv1(y1) + + y2 = F.pad(x[:, :, 1:, 1:], (0, 1, 0, 1)) + y2 = F.avg_pool2d(y2, kernel_size=1, stride=2, padding=0) + y2 = self.conv2(y2) + + z = torch.cat([y1, y2], dim=1) + z = self.bn(z) + + return z + + +class BasicBlock(nn.Module): + def __init__(self, in_channels, out_channels, stride, shake_config): + super(BasicBlock, self).__init__() + + self.shake_config = shake_config + + self.residual_path1 = ResidualPath(in_channels, out_channels, stride) + self.residual_path2 = ResidualPath(in_channels, out_channels, stride) + + self.shortcut = nn.Sequential() + if in_channels != out_channels: + self.shortcut.add_module('downsample', + DownsamplingShortcut(in_channels)) + + def forward(self, x): + x1 = self.residual_path1(x) + x2 = self.residual_path2(x) + + if self.training: + shake_config = self.shake_config + else: + shake_config = (False, False, False) + + alpha, beta = get_alpha_beta(x.size(0), shake_config, x.device) + y = shake_function(x1, x2, alpha, beta) + + return self.shortcut(x) + y + + +class ShakeNet(nn.Module): + def __init__(self, config): + super().__init__() + + input_shape = config['input_shape'] + n_classes = config['n_classes'] + + base_channels = config['base_channels'] + depth = config['depth'] + self.shake_config = (config['shake_forward'], config['shake_backward'], + config['shake_image']) + + block = BasicBlock + n_blocks_per_stage = (depth - 2) // 6 + assert n_blocks_per_stage * 6 + 2 == depth + + n_channels = [base_channels, base_channels * 2, base_channels * 4] + + self.conv = nn.Conv2d( + input_shape[1], + n_channels[0], + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.bn = nn.BatchNorm2d(base_channels) + + self.stage1 = self._make_stage( + n_channels[0], n_channels[0], n_blocks_per_stage, block, stride=1) + self.stage2 = self._make_stage( + n_channels[0], n_channels[1], n_blocks_per_stage, block, stride=2) + self.stage3 = self._make_stage( + n_channels[1], n_channels[2], n_blocks_per_stage, block, stride=2) + + # compute conv feature size + with torch.no_grad(): + self.feature_size = self._forward_conv( + torch.zeros(*input_shape)).view(-1).shape[0] + + self.fc = nn.Linear(self.feature_size, n_classes) + + # initialize weights + self.apply(initialize_weights) + + def _make_stage(self, in_channels, out_channels, n_blocks, block, stride): + stage = nn.Sequential() + for index in range(n_blocks): + block_name = 'block{}'.format(index + 1) + if index == 0: + stage.add_module( + block_name, + block( + in_channels, + out_channels, + stride=stride, + shake_config=self.shake_config)) + else: + stage.add_module( + block_name, + block( + out_channels, + out_channels, + stride=1, + shake_config=self.shake_config)) + return stage + + def _forward_conv(self, x): + x = F.relu(self.bn(self.conv(x)), inplace=True) + x = self.stage1(x) + x = self.stage2(x) + x = self.stage3(x) + x = F.adaptive_avg_pool2d(x, output_size=1) + return x + + # def forward(self, x): + # x = self._forward_conv(x) + # x = x.view(x.size(0), -1) + # x = self.fc(x) + # return x + + def forward(self, x, return_prelogit=False): + prelogit = self._forward_conv(x) + prelogit = prelogit.view(x.size(0), -1) + out = self.fc(prelogit) + if return_prelogit: + return out, prelogit + else: + return out diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/models/shake_shake_function.py b/Robustness_Eval/GAIRAT/GAIR_RST/models/shake_shake_function.py new file mode 100644 index 0000000..9b8b5b1 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/models/shake_shake_function.py @@ -0,0 +1,53 @@ +""" +Based on code from https://github.com/hysts/pytorch_shake_shake +""" + + +import torch +from torch.autograd import Function + +class ShakeFunction(Function): + @staticmethod + def forward(ctx, x1, x2, alpha, beta): + ctx.save_for_backward(x1, x2, alpha, beta) + + y = x1 * alpha + x2 * (1 - alpha) + return y + + @staticmethod + def backward(ctx, grad_output): + x1, x2, alpha, beta = ctx.saved_variables + grad_x1 = grad_x2 = grad_alpha = grad_beta = None + + if ctx.needs_input_grad[0]: + grad_x1 = grad_output * beta + if ctx.needs_input_grad[1]: + grad_x2 = grad_output * (1 - beta) + + return grad_x1, grad_x2, grad_alpha, grad_beta + + +shake_function = ShakeFunction.apply + + +def get_alpha_beta(batch_size, shake_config, device): + forward_shake, backward_shake, shake_image = shake_config + + if forward_shake and not shake_image: + alpha = torch.rand(1) + elif forward_shake and shake_image: + alpha = torch.rand(batch_size).view(batch_size, 1, 1, 1) + else: + alpha = torch.FloatTensor([0.5]) + + if backward_shake and not shake_image: + beta = torch.rand(1) + elif backward_shake and shake_image: + beta = torch.rand(batch_size).view(batch_size, 1, 1, 1) + else: + beta = torch.FloatTensor([0.5]) + + alpha = alpha.to(device) + beta = beta.to(device) + + return alpha, beta diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/models/wideresnet.py b/Robustness_Eval/GAIRAT/GAIR_RST/models/wideresnet.py new file mode 100644 index 0000000..47b6e6d --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/models/wideresnet.py @@ -0,0 +1,98 @@ +"""Based on code from https://github.com/yaodongyu/TRADES""" + +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class WideResNet(nn.Module): + def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0.0): + super(WideResNet, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 4) % 6 == 0) + n = (depth - 4) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x, return_prelogit=False): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + if return_prelogit: + return self.fc(out), out + else: + return self.fc(out) + diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/robust_self_training_GAIR.py b/Robustness_Eval/GAIRAT/GAIR_RST/robust_self_training_GAIR.py new file mode 100644 index 0000000..c981fe0 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/robust_self_training_GAIR.py @@ -0,0 +1,520 @@ +""" +Based on code from +https://github.com/yaircarmon/semisup-adv +""" + + +import os +import sys +import argparse + +import torch +import torch.nn.functional as F +import torch.optim as optim +from torchvision import transforms +import torch.backends.cudnn as cudnn +from torch.utils.data import DataLoader + +import pandas as pd +import numpy as np + +from utils import get_model + +from losses import GAIR_trades_loss ,trades_loss, noise_loss +from datasets import SemiSupervisedDataset, SemiSupervisedSampler, DATASETS +from attack_pgd import pgd +#from smoothing import quick_smoothing + +from autoaugment import CIFAR10Policy +from cutout import Cutout + +import logging + + +# ----------------------------- CONFIGURATION ---------------------------------- +parser = argparse.ArgumentParser( + description='PyTorch GAIR-TRADES Adversarial Training') + +# Dataset config +parser.add_argument('--dataset', type=str, default='cifar10', + choices=DATASETS, + help='The dataset to use for training)') +parser.add_argument('--data_dir', default='./data', type=str, + help='Directory where datasets are located') +parser.add_argument('--svhn_extra', action='store_true', default=False, + help='Adds the extra SVHN data') + +# Model config +parser.add_argument('--model', '-m', default='wrn-28-10', type=str, + help='Name of the model (see utils.get_model)') +parser.add_argument('--model_dir', default='./rst-model', + help='Directory of model for saving checkpoint') +parser.add_argument('--overwrite', action='store_true', default=False, + help='Cancels the run if an appropriate checkpoint is found') +parser.add_argument('--normalize_input', action='store_true', default=False, + help='Apply standard CIFAR normalization first thing ' + 'in the network (as part of the model, not in the data' + ' fetching pipline)') + +# Logging and checkpointing +parser.add_argument('--log_interval', type=int, default=40, + help='Number of batches between logging of training status') +parser.add_argument('--save_freq', default=25, type=int, + help='Checkpoint save frequency (in epochs)') + +# Generic training configs +parser.add_argument('--seed', type=int, default=1, + help='Random seed. ' + 'Note: fixing the random seed does not give complete ' + 'reproducibility. See ' + 'https://pytorch.org/docs/stable/notes/randomness.html') + +parser.add_argument('--batch_size', type=int, default=256, metavar='N', + help='Input batch size for training (default: 128)') +parser.add_argument('--test_batch_size', type=int, default=500, metavar='N', + help='Input batch size for testing (default: 128)') +parser.add_argument('--epochs', type=int, default=200, metavar='N', + help='Number of epochs to train. ' + 'Note: we arbitrarily define an epoch as a pass ' + 'through 50K datapoints. This is convenient for ' + 'comparison with standard CIFAR-10 training ' + 'configurations.') + +# Eval config +parser.add_argument('--eval_freq', default=1, type=int, + help='Eval frequency (in epochs)') +parser.add_argument('--train_eval_batches', default=None, type=int, + help='Maximum number for batches in training set eval') +parser.add_argument('--eval_attack_batches', default=1, type=int, + help='Number of eval batches to attack with PGD or certify ' + 'with randomized smoothing') + +# Optimizer config +parser.add_argument('--weight_decay', '--wd', default=5e-4, type=float) +parser.add_argument('--lr', type=float, default=0.1, metavar='LR', + help='Learning rate') +parser.add_argument('--lr_schedule', type=str, default='cosine', + choices=('trades', 'trades_fixed', 'cosine', 'wrn'), + help='Learning rate schedule') +parser.add_argument('--momentum', type=float, default=0.9, metavar='M', + help='SGD momentum') +parser.add_argument('--nesterov', action='store_true', default=True, + help='Use extragrdient steps') + +# Adversarial / stability training config +parser.add_argument('--loss', default='trades', type=str, + choices=('trades', 'noise'), + help='Which loss to use: TRADES-like KL regularization ' + 'or noise augmentation') + +parser.add_argument('--distance', '-d', default='l_2', type=str, + help='Metric for attack model: l_inf uses adversarial ' + 'training and l_2 uses stability training and ' + 'randomized smoothing certification', + choices=['l_inf', 'l_2']) +parser.add_argument('--epsilon', default=0.031, type=float, + help='Adversarial perturbation size (takes the role of' + ' sigma for stability training)') + +parser.add_argument('--pgd_num_steps', default=10, type=int, + help='number of pgd steps in adversarial training') +parser.add_argument('--pgd_step_size', default=0.007, + help='pgd steps size in adversarial training', type=float) +parser.add_argument('--beta', default=6.0, type=float, + help='stability regularization, i.e., 1/lambda in TRADES') + +# Semi-supervised training configuration +parser.add_argument('--aux_data_filename', default=None, type=str, + help='Path to pickle file containing unlabeled data and ' + 'pseudo-labels used for RST') + +parser.add_argument('--unsup_fraction', default=0.5, type=float, + help='Fraction of unlabeled examples in each batch; ' + 'implicitly sets the weight of unlabeled data in the ' + 'loss. If set to -1, batches are sampled from a ' + 'single pool') +parser.add_argument('--aux_take_amount', default=None, type=int, + help='Number of random aux examples to retain. ' + 'None retains all aux data.') + +parser.add_argument('--remove_pseudo_labels', action='store_true', + default=False, + help='Performs training without pseudo-labels (rVAT)') +parser.add_argument('--entropy_weight', type=float, + default=0.0, help='Weight on entropy loss') + +# Additional aggressive data augmentation +parser.add_argument('--autoaugment', action='store_true', default=False, + help='Use autoaugment for data augmentation') +parser.add_argument('--cutout', action='store_true', default=False, + help='Use cutout for data augmentation') + +# GAIR parameters +parser.add_argument('--Lambda', default=0.0, type=float) + +# Resume +parser.add_argument('--resume', default=None ,type=str) + +args = parser.parse_args() + +# ------------------------------ OUTPUT SETUP ---------------------------------- +model_dir = args.model_dir +if not os.path.exists(model_dir): + os.makedirs(model_dir) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(message)s", + handlers=[ + logging.FileHandler(os.path.join(args.model_dir, 'training.log')), + logging.StreamHandler() + ]) +logger = logging.getLogger() + +logging.info('Robust self-training') +logging.info('Args: %s', args) + +if not args.overwrite: + final_checkpoint_path = os.path.join( + model_dir, 'checkpoint-epoch{}.pt'.format(args.epochs)) + if os.path.exists(final_checkpoint_path): + logging.info('Appropriate checkpoint found - quitting!') + sys.exit(0) +# ------------------------------------------------------------------------------ + +# ------------------------------- CUDA SETUP ----------------------------------- +# should provide some improved performance +cudnn.benchmark = True +# useful setting for debugging +# cudnn.benchmark = False +# cudnn.deterministic = True + +use_cuda = torch.cuda.is_available() +torch.manual_seed(args.seed) +device = torch.device('cuda' if use_cuda else 'cpu') +# ------------------------------------------------------------------------------ + +# --------------------------- DATA AUGMENTATION -------------------------------- +if args.dataset == 'cifar10': + transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + ]) +elif args.dataset == 'svhn': + # the WRN paper does no augmentation on SVHN + # obviously flipping is a bad idea, and it makes some sense not to + # crop because there are a lot of distractor digits in the edges of the + # image + transform_train = transforms.ToTensor() + +if args.autoaugment or args.cutout: + assert (args.dataset == 'cifar10') + transform_list = [ + transforms.RandomCrop(32, padding=4, fill=128), + # fill parameter needs torchvision installed from source + transforms.RandomHorizontalFlip()] + if args.autoaugment: + transform_list.append(CIFAR10Policy()) + transform_list.append(transforms.ToTensor()) + if args.cutout: + transform_list.append(Cutout(n_holes=1, length=16)) + + transform_train = transforms.Compose(transform_list) + logger.info('Applying aggressive training augmentation: %s' + % transform_train) + +transform_test = transforms.Compose([ + transforms.ToTensor()]) +# ------------------------------------------------------------------------------ + +# ----------------- DATASET WITH AUX PSEUDO-LABELED DATA ----------------------- +trainset = SemiSupervisedDataset(base_dataset=args.dataset, + add_svhn_extra=args.svhn_extra, + root=args.data_dir, train=True, + download=True, transform=transform_train, + aux_data_filename=args.aux_data_filename, + add_aux_labels=not args.remove_pseudo_labels, + aux_take_amount=args.aux_take_amount) + +# num_batches=50000 enforces the definition of an "epoch" as passing through 50K +# datapoints +# TODO: make sure that this code works also when trainset.unsup_indices=[] +train_batch_sampler = SemiSupervisedSampler( + trainset.sup_indices, trainset.unsup_indices, + args.batch_size, args.unsup_fraction, + num_batches=int(np.ceil(50000 / args.batch_size))) +epoch_size = len(train_batch_sampler) * args.batch_size + +kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} +train_loader = DataLoader(trainset, batch_sampler=train_batch_sampler, **kwargs) + +testset = SemiSupervisedDataset(base_dataset=args.dataset, + root=args.data_dir, train=False, + download=True, + transform=transform_test) +test_loader = DataLoader(testset, batch_size=args.test_batch_size, + shuffle=False, **kwargs) + +trainset_eval = SemiSupervisedDataset( + base_dataset=args.dataset, + add_svhn_extra=args.svhn_extra, + root=args.data_dir, train=True, + download=True, transform=transform_train) + +eval_train_loader = DataLoader(trainset_eval, batch_size=args.test_batch_size, + shuffle=True, **kwargs) + +eval_test_loader = DataLoader(testset, batch_size=args.test_batch_size, + shuffle=False, **kwargs) +# ------------------------------------------------------------------------------ + + +# ----------------------- TRAIN AND EVAL FUNCTIONS ----------------------------- +def train(args, model, device, train_loader, optimizer, epoch, Lam): + model.train() + train_metrics = [] + epsilon = args.epsilon + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + + optimizer.zero_grad() + + # calculate robust loss + if args.loss == 'trades': + # The TRADES KL-robustness regularization term proposed by + # Zhang et al., with some additional features + (loss, natural_loss, robust_loss, + entropy_loss_unlabeled) = GAIR_trades_loss( + model=model, + x_natural=data, + y=target, + optimizer=optimizer, + step_size=args.pgd_step_size, + epsilon=epsilon, + perturb_steps=args.pgd_num_steps, + beta=args.beta, + distance=args.distance, + adversarial=args.distance == 'l_inf', + entropy_weight=args.entropy_weight, + Lambda=Lam) + + elif args.loss == 'noise': + # Augmenting the input with random noise as in Cohen et al. + assert (args.distance == 'l_2') + loss = noise_loss(model=model, x_natural=data, + y=target, clamp_x=True, epsilon=epsilon) + entropy_loss_unlabeled = torch.Tensor([0.]) + natural_loss = robust_loss = loss + + loss.backward() + optimizer.step() + + train_metrics.append(dict( + epoch=epoch, + loss=loss.item(), + natural_loss=natural_loss.item(), + robust_loss=robust_loss.item(), + entropy_loss_unlabeled=entropy_loss_unlabeled.item())) + + # print progress + if batch_idx % args.log_interval == 0: + logging.info( + 'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), epoch_size, + 100. * batch_idx / len(train_loader), loss.item())) + + return train_metrics + + +def eval(args, model, device, eval_set, loader): + loss = 0 + total = 0 + correct = 0 + adv_correct = 0 + adv_correct_clean = 0 + adv_total = 0 + + model.eval() + with torch.no_grad(): + for batch_idx, (data, target) in enumerate(loader): + data, target = data.to(device), target.to(device) + data, target = data[target != -1], target[target != -1] + output = model(data) + loss += F.cross_entropy(output, target, reduction='sum').item() + pred = output.max(1, keepdim=True)[1] + correct += pred.eq(target.view_as(pred)).sum().item() + if batch_idx < args.eval_attack_batches: + if args.distance == 'l_2': + # run coarse certification + #incorrect_clean, incorrect_rob = quick_smoothing( + # model, data, target, + # sigma=args.epsilon, + # eps=args.epsilon, + # num_smooth=100, batch_size=1000) + pass + elif args.distance == 'l_inf': + # run medium-strength gradient attack + is_correct_clean, is_correct_rob = pgd( + model, data, target, + epsilon=args.epsilon, + num_steps=2 * args.pgd_num_steps, + step_size=args.pgd_step_size, + random_start=False) + incorrect_clean = (1-is_correct_clean).sum() + incorrect_rob = (1-np.prod(is_correct_rob, axis=1)).sum() + else: + raise ValueError('No support for distance %s', + args.distance) + adv_correct_clean += (len(data) - int(incorrect_clean)) + adv_correct += (len(data) - int(incorrect_rob)) + adv_total += len(data) + total += len(data) + if ((eval_set == 'train') and + (batch_idx + 1 == args.train_eval_batches)): + break + loss /= total + accuracy = correct / total + if adv_total > 0: + robust_clean_accuracy = adv_correct_clean / adv_total + robust_accuracy = adv_correct / adv_total + else: + robust_accuracy = robust_clean_accuracy = 0. + + eval_data = dict(loss=loss, accuracy=accuracy, + robust_accuracy=robust_accuracy, + robust_clean_accuracy=robust_clean_accuracy) + eval_data = {eval_set + '_' + k: v for k, v in eval_data.items()} + logging.info( + '{}: Clean loss: {:.4f}, ' + 'Clean accuracy: {}/{} ({:.2f}%), ' + '{} clean accuracy: {}/{} ({:.2f}%), ' + 'Robust accuracy {}/{} ({:.2f}%)'.format( + eval_set.upper(), loss, + correct, total, 100.0 * accuracy, + 'Smoothing' if args.distance == 'l_2' else 'PGD', + adv_correct_clean, adv_total, 100.0 * robust_clean_accuracy, + adv_correct, adv_total, 100.0 * robust_accuracy)) + + return eval_data + +def lam(epoch): + Lam = 100.0 + if epoch >= 0.5 * args.epochs: + Lam = args.Lambda + return Lam + + +def adjust_learning_rate(optimizer, epoch): + """decrease the learning rate""" + lr = args.lr + schedule = args.lr_schedule + # schedule from TRADES repo (different from paper due to bug there) + if schedule == 'trades': + if epoch >= 0.75 * args.epochs: + lr = args.lr * 0.1 + # schedule as in TRADES paper + elif schedule == 'trades_fixed': + if epoch >= 0.75 * args.epochs: + lr = args.lr * 0.1 + if epoch >= 0.9 * args.epochs: + lr = args.lr * 0.01 + if epoch >= args.epochs: + lr = args.lr * 0.001 + # cosine schedule + elif schedule == 'cosine': + lr = args.lr * 0.5 * (1 + np.cos((epoch - 1) / args.epochs * np.pi)) + # schedule as in WRN paper + elif schedule == 'wrn': + if epoch >= 0.3 * args.epochs: + lr = args.lr * 0.2 + if epoch >= 0.6 * args.epochs: + lr = args.lr * 0.2 * 0.2 + if epoch >= 0.8 * args.epochs: + lr = args.lr * 0.2 * 0.2 * 0.2 + else: + raise ValueError('Unkown LR schedule %s' % schedule) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr +# ------------------------------------------------------------------------------ + +# ----------------------------- TRAINING LOOP ---------------------------------- +start_epoch = 1 + +num_classes = 10 +model = get_model(args.model, num_classes=num_classes, + normalize_input=args.normalize_input) +if use_cuda: + model = torch.nn.DataParallel(model).cuda() +optimizer = optim.SGD(model.parameters(), lr=args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay, + nesterov=args.nesterov) + +if args.resume: + # resume directly point to checkpoint.pth.tar + print ('==> Resuming from checkpoint ..') + print(args.resume) + assert os.path.isfile(args.resume) + #out_dir = os.path.dirname(resume) + checkpoint = torch.load(args.resume) + start_epoch = checkpoint['save_epoch'] + 1 + num_classes = checkpoint['num_classes'] + #best_acc = checkpoint['test_pgd20_acc'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['opt']) + +def main(): + train_df = pd.DataFrame() + eval_df = pd.DataFrame() + + for epoch in range(start_epoch, args.epochs + 1): + # adjust learning rate for SGD + Lam = lam(epoch) + lr = adjust_learning_rate(optimizer, epoch) + logger.info('Setting learning rate to %g' % lr) + # adversarial training + train_data = train(args, model, device, train_loader, optimizer, epoch, Lam) + train_df = train_df.append(pd.DataFrame(train_data), ignore_index=True) + + # evaluation on natural examples + logging.info(120 * '=') + if epoch % args.eval_freq == 0 or epoch == args.epochs: + eval_data = {'epoch': int(epoch)} + eval_data.update( + eval(args, model, device, 'train', eval_train_loader)) + eval_data.update( + eval(args, model, device, 'test', eval_test_loader)) + eval_df = eval_df.append(pd.Series(eval_data), ignore_index=True) + logging.info(120 * '=') + + # save stats + train_df.to_csv(os.path.join(model_dir, 'stats_train.csv')) + eval_df.to_csv(os.path.join(model_dir, 'stats_eval.csv')) + + # save checkpoint + if epoch % args.save_freq == 0 or epoch == args.epochs: + torch.save(dict(num_classes=num_classes, + state_dict=model.state_dict(), + normalize_input=args.normalize_input, + save_epoch=epoch, + opt=optimizer.state_dict()), + os.path.join(model_dir, + 'checkpoint-epoch{}.pt'.format(epoch))) + torch.save(optimizer.state_dict(), + os.path.join(model_dir, + 'opt-checkpoint_epoch{}.tar'.format(epoch))) + + # save last checkpoint + torch.save(dict(num_classes=num_classes, + state_dict=model.state_dict(), + normalize_input=args.normalize_input, + save_epoch=epoch, + opt=optimizer.state_dict()), + os.path.join(model_dir, + 'lastcheckpoint.pt')) +# ------------------------------------------------------------------------------ + +if __name__ == '__main__': + main() diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/run_evaluation.sh b/Robustness_Eval/GAIRAT/GAIR_RST/run_evaluation.sh new file mode 100644 index 0000000..45d41a1 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/run_evaluation.sh @@ -0,0 +1 @@ +CUDA_VISIBLE_DEVICES='0' python3 attack_evaluation.py --model_path='./GAIR_RST_check/checkpoint-epoch200.pt' --output_suffix='GAIR_RST_PGD_plus_results' & diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/run_training.sh b/Robustness_Eval/GAIRAT/GAIR_RST/run_training.sh new file mode 100644 index 0000000..44bc201 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/run_training.sh @@ -0,0 +1 @@ +CUDA_VISIBLE_DEVICES='0,1,2,3' python3 robust_self_training_GAIR.py --aux_data_filename='ti_500K_pseudo_labeled.pickle' --distance='l_inf' --epsilon=0.031 --Lambda=3.0 --model_dir='./GAIR_RST_check' & diff --git a/Robustness_Eval/GAIRAT/GAIR_RST/utils.py b/Robustness_Eval/GAIRAT/GAIR_RST/utils.py new file mode 100644 index 0000000..e460335 --- /dev/null +++ b/Robustness_Eval/GAIRAT/GAIR_RST/utils.py @@ -0,0 +1,177 @@ +""" +Utilities. Partially based on code from +https://github.com/modestyachts/CIFAR-10.1 +""" +import io +import json +import os +import pickle + +import numpy as np +import pathlib + +from models.wideresnet import WideResNet +from models.shake_shake import ShakeNet +from models.cifar_resnet import ResNet + +import torch +from torch.nn import Sequential, Module + +cifar10_label_names = ['airplane', 'automobile', 'bird', + 'cat', 'deer', 'dog', 'frog', 'horse', + 'ship', 'truck'] + + +def get_model(name, num_classes=10, normalize_input=False): + name_parts = name.split('-') + if name_parts[0] == 'wrn': + depth = int(name_parts[1]) + widen = int(name_parts[2]) + model = WideResNet( + depth=depth, num_classes=num_classes, widen_factor=widen) + + elif name_parts[0] == 'ss': + model = ShakeNet(dict(depth=int(name_parts[1]), + base_channels=int(name_parts[2]), + shake_forward=True, shake_backward=True, + shake_image=True, input_shape=(1, 3, 32, 32), + n_classes=num_classes, + )) + elif name_parts[0] == 'resnet': + model = ResNet(num_classes=num_classes, depth=int(name_parts[1])) + else: + raise ValueError('Could not parse model name %s' % name) + + if normalize_input: + model = Sequential(NormalizeInput(), model) + + return model + + +class NormalizeInput(Module): + def __init__(self, mean=(0.4914, 0.4822, 0.4465), + std=(0.2023, 0.1994, 0.2010)): + super().__init__() + + self.register_buffer('mean', torch.Tensor(mean).reshape(1, -1, 1, 1)) + self.register_buffer('std', torch.Tensor(std).reshape(1, -1, 1, 1)) + + def forward(self, x): + return (x - self.mean) / self.std + +# TODO: decide whether to remove all code below this line +# Should we add some description of how these files +# were obtained? +def load_tinyimage_subset(other_data_path, + version_string='v7'): + image_data_filename = 'tinyimage_subset_data' + if version_string != '': + image_data_filename += '_' + version_string + image_data_filename += '.pickle' + image_data_filepath = os.path.abspath(os.path.join(other_data_path, image_data_filename)) + indices_filename = 'tinyimage_subset_indices' + if version_string != '': + indices_filename += '_' + version_string + indices_filename += '.json' + indices_filepath = os.path.abspath(os.path.join(other_data_path, indices_filename)) + print('Loading indices from file {}'.format(indices_filepath)) + assert pathlib.Path(indices_filepath).is_file() + print('Loading image data from file {}'.format(image_data_filepath)) + assert pathlib.Path(image_data_filepath).is_file() + with open(indices_filepath, 'r') as f: + indices = json.load(f) + with open(image_data_filepath, 'rb') as f: + image_data = pickle.load(f) + num_entries = 0 + for kw, kw_indices in indices.items(): + for entry in kw_indices: + assert entry['tinyimage_index'] in image_data + num_entries += 1 + assert num_entries == len(image_data) + return indices, image_data + + +def load_cifar10_by_keyword(unique_keywords=True, version_string='v7'): + cifar10_keywords = load_cifar10_keywords(unique_keywords=unique_keywords, + lists_for_unique=True, + version_string=version_string) + cifar10_by_keyword = {} + for ii, keyword_entries in enumerate(cifar10_keywords): + for entry in keyword_entries: + cur_keyword = entry['nn_keyword'] + if not cur_keyword in cifar10_by_keyword: + cifar10_by_keyword[cur_keyword] = [] + cifar10_by_keyword[cur_keyword].append(ii) + return cifar10_by_keyword + + +def load_cifar10_keywords(other_data_path, + unique_keywords=True, + lists_for_unique=False, + version_string='v7'): + filename = 'cifar10_keywords' + if unique_keywords: + filename += '_unique' + if version_string != '': + filename += '_' + version_string + filename += '.json' + keywords_filepath = os.path.abspath(os.path.join(other_data_path, filename)) + print('Loading keywords from file {}'.format(keywords_filepath)) + assert pathlib.Path(keywords_filepath).is_file() + with open(keywords_filepath, 'r') as f: + cifar10_keywords = json.load(f) + if unique_keywords and lists_for_unique: + result = [] + for entry in cifar10_keywords: + result.append([entry]) + else: + result = cifar10_keywords + assert len(result) == 60000 + return result + + +def load_distances_to_cifar10(version_string='v7'): + data_path = os.path.join(os.path.dirname(__file__), '../data/') + filename = 'tinyimage_cifar10_distances' + if version_string != '': + filename += '_' + version_string + filename += '.json' + filepath = os.path.abspath(os.path.join(data_path, filename)) + print('Loading distances from file {}'.format(filepath)) + assert pathlib.Path(filepath).is_file() + with open(filepath, 'r') as f: + tmp = json.load(f) + if version_string == 'v4': + assert len(tmp) == 372131 + elif version_string == 'v6': + assert len(tmp) == 1646248 + elif version_string == 'v7': + assert len(tmp) == 589711 + result = {} + for k, v in tmp.items(): + result[int(k)] = v + return result + + +def load_new_test_data_indices(version_string='v7'): + data_path = os.path.join(os.path.dirname(__file__), '../data/') + filename = 'cifar10.1' + if version_string == '': + version_string = 'v7' + if version_string in ['v4', 'v6', 'v7']: + filename += '_' + version_string + else: + raise ValueError('Unknown dataset version "{}".'.format(version_string)) + ti_indices_data_path = os.path.join(os.path.dirname(__file__), '../data/') + ti_indices_filename = 'cifar10.1_' + version_string + '_ti_indices.json' + ti_indices_filepath = os.path.abspath(os.path.join(ti_indices_data_path, ti_indices_filename)) + print('Loading Tiny Image indices from file {}'.format(ti_indices_filepath)) + assert pathlib.Path(ti_indices_filepath).is_file() + with open(ti_indices_filepath, 'r') as f: + tinyimage_indices = json.load(f) + assert type(tinyimage_indices) is list + if version_string == 'v6' or version_string == 'v7': + assert len(tinyimage_indices) == 2000 + elif version_string == 'v4': + assert len(tinyimage_indices) == 2021 + return tinyimage_indices diff --git a/Robustness_Eval/GAIRAT/__pycache__/GAIR.cpython-36.pyc b/Robustness_Eval/GAIRAT/__pycache__/GAIR.cpython-36.pyc new file mode 100644 index 0000000..e0cbdec Binary files /dev/null and b/Robustness_Eval/GAIRAT/__pycache__/GAIR.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/__pycache__/GAIR.cpython-37.pyc b/Robustness_Eval/GAIRAT/__pycache__/GAIR.cpython-37.pyc new file mode 100644 index 0000000..f91c61c Binary files /dev/null and b/Robustness_Eval/GAIRAT/__pycache__/GAIR.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/__pycache__/attack_generator.cpython-36.pyc b/Robustness_Eval/GAIRAT/__pycache__/attack_generator.cpython-36.pyc new file mode 100644 index 0000000..900ced1 Binary files /dev/null and b/Robustness_Eval/GAIRAT/__pycache__/attack_generator.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/__pycache__/attack_generator.cpython-37.pyc b/Robustness_Eval/GAIRAT/__pycache__/attack_generator.cpython-37.pyc new file mode 100644 index 0000000..0f4c7ac Binary files /dev/null and b/Robustness_Eval/GAIRAT/__pycache__/attack_generator.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/attack_cw.py b/Robustness_Eval/GAIRAT/attack_cw.py new file mode 100644 index 0000000..356a282 --- /dev/null +++ b/Robustness_Eval/GAIRAT/attack_cw.py @@ -0,0 +1,368 @@ +""" +Implementation of L_infty Carlini-Wagner attack based on the L2 implementation +in FoolBox v1.9 (with many dependencies on that pakage) +https://github.com/bethgelab/foolbox + +For PGD plus evaluation +Based on https://github.com/yaircarmon/semisup-adv +""" + +try: + from foolbox.models import PyTorchModel + from foolbox.attacks import Attack + from foolbox.attacks.base import call_decorator + from foolbox.utils import onehot_like + from foolbox.distances import Linf + HAVE_FOOLBOX = True +except ImportError: + HAVE_FOOLBOX = False + Attack = object + call_decorator = lambda x: x +import numpy as np +import logging + + +def cw(model, + X, + y, + binary_search_steps=5, + max_iterations=1000, + learning_rate=5E-3, + initial_const=1E-2, + tau_decrease_factor=0.9 + ): + if not HAVE_FOOLBOX: + raise ImportError('Could not import FoolBox') + + foolbox_model = PyTorchModel(model, bounds=(0, 1), num_classes=10) + attack = CarliniWagnerLIAttack(foolbox_model, distance=Linf) + linf_distances = [] + for i in range(len(X)): + logging.info('Example: %g', i) + image = X[i, :].detach().cpu().numpy() + label = y[i].cpu().numpy() + adversarial = attack(image, label, + binary_search_steps=binary_search_steps, + max_iterations=max_iterations, + learning_rate=learning_rate, + initial_const=initial_const, + tau_decrease_factor=tau_decrease_factor) + logging.info('Linf distance: %g', np.max(np.abs(adversarial - image))) + linf_distances.append(np.max(np.abs(adversarial - image))) + return linf_distances + + +class CarliniWagnerLIAttack(Attack): + """The Linf version of the Carlini & Wagner attack. + + This attack is described in [1]_. This implementation + is based on the reference implementation by Carlini [2]_. + For bounds \ne (0, 1), it differs from [2]_ because we + normalize the squared L2 loss with the bounds. + + References + ---------- + .. [1] Nicholas Carlini, David Wagner: "Towards Evaluating the + Robustness of Neural Networks", https://arxiv.org/abs/1608.04644 + .. [2] https://github.com/carlini/nn_robust_attacks + + """ + + @call_decorator + def __call__(self, input_or_adv, label=None, unpack=True, + binary_search_steps=5, + tau_decrease_factor=0.9, + max_iterations=1000, + confidence=0, learning_rate=5e-3, + initial_const=1e-2, abort_early=True): + + """The L_infty version of the Carlini & Wagner attack. + + Parameters + ---------- + input_or_adv : `numpy.ndarray` or :class:`Adversarial` + The original, unperturbed input as a `numpy.ndarray` or + an :class:`Adversarial` instance. + label : int + The reference label of the original input. Must be passed + if `a` is a `numpy.ndarray`, must not be passed if `a` is + an :class:`Adversarial` instance. + unpack : bool + If true, returns the adversarial input, otherwise returns + the Adversarial object. + binary_search_steps : int + The number of steps for the binary search used to + find the optimal tradeoff-constant between distance and confidence. + max_iterations : int + The maximum number of iterations. Larger values are more + accurate; setting it too small will require a large learning rate + and will produce poor results. + confidence : int or float + Confidence of adversarial examples: a higher value produces + adversarials that are further away, but more strongly classified + as adversarial. + learning_rate : float + The learning rate for the attack algorithm. Smaller values + produce better results but take longer to converge. + initial_const : float + The initial tradeoff-constant to use to tune the relative + importance of distance and confidence. If `binary_search_steps` + is large, the initial constant is not important. + abort_early : bool + If True, Adam will be aborted if the loss hasn't decreased + for some time (a tenth of max_iterations). + + """ + + a = input_or_adv + + del input_or_adv + del label + del unpack + + if not a.has_gradient(): + logging.fatal('Applied gradient-based attack to model that ' + 'does not provide gradients.') + return + + min_, max_ = a.bounds() + + def to_attack_space(x): + # map from [min_, max_] to [-1, +1] + a = (min_ + max_) / 2 + b = (max_ - min_) / 2 + x = (x - a) / b + + # from [-1, +1] to approx. (-1, +1) + x = x * 0.999999 + + # from (-1, +1) to (-inf, +inf) + return np.arctanh(x) + + def to_model_space(x): + """Transforms an input from the attack space + to the model space. This transformation and + the returned gradient are elementwise.""" + + # from (-inf, +inf) to (-1, +1) + x = np.tanh(x) + + grad = 1 - np.square(x) + + # map from (-1, +1) to (min_, max_) + a = (min_ + max_) / 2 + b = (max_ - min_) / 2 + x = x * b + a + + grad = grad * b + return x, grad + + # variables representing inputs in attack space will be + # prefixed with att_ + att_original = to_attack_space(a.original_image) + + # will be close but not identical to a.original_image + reconstructed_original, _ = to_model_space(att_original) + + # the binary search finds the smallest const for which we + # find an adversarial + const = initial_const + lower_bound = 0 + upper_bound = np.inf + true_logits, _ = a.predictions(reconstructed_original) + true_label = np.argmax(true_logits) + # Binary search for constant + start_tau = 1.0 + for binary_search_step in range(binary_search_steps): + if binary_search_step == binary_search_steps - 1 and \ + binary_search_steps >= 10: + # in the last binary search step, use the upper_bound instead + # TODO: find out why... it's not obvious why this is useful + const = upper_bound + + logging.info( + 'starting optimization with const = {}, best overall distance = {}'.format( + const, a.distance)) + + # found adv with the current const + + att_warmstart = att_original + tau = start_tau + while tau > 1. / 256: + found_adv = False + att_perturbation = np.zeros_like(att_original) + # create a new optimizer to minimize the perturbation + optimizer = AdamOptimizer(att_perturbation.shape) + loss_at_previous_check = np.inf + # logging.info('Running with const:{}, tau:{}'.format(const, tau)) + for iteration in range(max_iterations): + + x, dxdp = to_model_space(att_warmstart + att_perturbation) + logits, i = a.predictions(x) + false_label = np.argmax(logits) + is_adv = not (false_label == true_label) + + loss, dldx, adv_loss, distance = self.loss_function( + const, tau, a, x, logits, reconstructed_original, + confidence, min_, max_) + + check_loss = logits[true_label] - logits[false_label] + + # logging.info('Iteration:{}, loss: {}; adv_loss:{}; distance:{}'.format( + # iteration, loss, adv_loss, distance)) + # backprop the gradient of the loss w.r.t. x further + # to get the gradient of the loss w.r.t. att_perturbation + assert dldx.shape == x.shape + assert dxdp.shape == x.shape + # we can do a simple elementwise multiplication, because + # grad_x_wrt_p is a matrix of elementwise derivatives + # (i.e. each x[i] w.r.t. p[i] only, for all i) and + # grad_loss_wrt_x is a real gradient reshaped as a matrix + gradient = dldx * dxdp + + att_perturbation += optimizer(gradient, learning_rate) + x, dxdp = to_model_space(att_warmstart + att_perturbation) + + if is_adv: + # Tau + binary search was successful but continuing opt + found_adv = True + + if abort_early and \ + iteration % (np.ceil(max_iterations / 10)) == 0: + # after each tenth of the iterations, check progress + # logging.info('Iteration:{}, loss: {}; best overall distance: {}; is_adv:{}'.format( + # iteration, loss, a.distance, is_adv)) + if not (loss_at_previous_check - loss > 0.0001): + break # stop Adam if there has not been progress + loss_at_previous_check = loss + + if not found_adv: + # This constant is fine, just that we broke out of tau + if tau < start_tau: + found_adv = True + start_tau = tau + break + + else: + actualtau = np.max(np.abs(x - reconstructed_original)) + if actualtau < tau: + tau = actualtau + tau = tau * tau_decrease_factor + att_warmstart = to_attack_space(x) + + if found_adv: + # logging.info('found adversarial with const = {}'.format(const)) + upper_bound = const + else: + # logging.info('failed to find adversarial ' + # 'with const = {}'.format(const)) + lower_bound = const + + if upper_bound == np.inf: + # exponential search + const *= 10 + else: + # binary search + const = (lower_bound + upper_bound) / 2 + + @classmethod + def loss_function(cls, const, tau, a, x, logits, reconstructed_original, + confidence, min_, max_): + """Returns the loss and the gradient of the loss w.r.t. x, + assuming that logits = model(x).""" + + targeted = a.target_class() is not None + if targeted: + c_minimize = cls.best_other_class(logits, a.target_class()) + c_maximize = a.target_class() + else: + c_minimize = a.original_class + c_maximize = cls.best_other_class(logits, a.original_class) + + is_adv_loss = logits[c_minimize] - logits[c_maximize] + + # is_adv is True as soon as the is_adv_loss goes below 0 + # but sometimes we want additional confidence + is_adv_loss += confidence + is_adv_loss = max(0, is_adv_loss) + + s = max_ - min_ + # squared_l2_distance = np.sum((x - reconstructed_original)**2) / s**2 + linf_distance = np.sum( + np.maximum(np.abs(x - reconstructed_original) - tau, 0)) + + total_loss = linf_distance + const * is_adv_loss + + # calculate the gradient of total_loss w.r.t. x + logits_diff_grad = np.zeros_like(logits) + logits_diff_grad[c_minimize] = 1 + logits_diff_grad[c_maximize] = -1 + is_adv_loss_grad = a.backward(logits_diff_grad, x) + assert is_adv_loss >= 0 + if is_adv_loss == 0: + is_adv_loss_grad = 0 + + squared_l2_distance_grad = (2 / s ** 2) * (x - reconstructed_original) + linf_distance_grad = np.sign(x - reconstructed_original) * ( + np.abs(x - reconstructed_original) - tau > 0) + total_loss_grad = linf_distance_grad + const * is_adv_loss_grad + return total_loss, total_loss_grad, is_adv_loss, linf_distance + + @staticmethod + def best_other_class(logits, exclude): + """Returns the index of the largest logit, ignoring the class that + is passed as `exclude`.""" + other_logits = logits - onehot_like(logits, exclude, value=np.inf) + return np.argmax(other_logits) + + +class AdamOptimizer: + """Basic Adam optimizer implementation that can minimize w.r.t. + a single variable. + + Parameters + ---------- + shape : tuple + shape of the variable w.r.t. which the loss should be minimized + + """ + + def __init__(self, shape): + self.m = np.zeros(shape) + self.v = np.zeros(shape) + self.t = 0 + + def __call__(self, gradient, learning_rate, + beta1=0.9, beta2=0.999, epsilon=10e-8): + """Updates internal parameters of the optimizer and returns + the change that should be applied to the variable. + + Parameters + ---------- + gradient : `np.ndarray` + the gradient of the loss w.r.t. to the variable + learning_rate: float + the learning rate in the current iteration + beta1: float + decay rate for calculating the exponentially + decaying average of past gradients + beta2: float + decay rate for calculating the exponentially + decaying average of past squared gradients + epsilon: float + small value to avoid division by zero + + """ + + self.t += 1 + + self.m = beta1 * self.m + (1 - beta1) * gradient + self.v = beta2 * self.v + (1 - beta2) * gradient ** 2 + + bias_correction_1 = 1 - beta1 ** self.t + bias_correction_2 = 1 - beta2 ** self.t + + m_hat = self.m / bias_correction_1 + v_hat = self.v / bias_correction_2 + + return -learning_rate * m_hat / (np.sqrt(v_hat) + epsilon) diff --git a/Robustness_Eval/GAIRAT/attack_generator.py b/Robustness_Eval/GAIRAT/attack_generator.py new file mode 100644 index 0000000..cd5aefe --- /dev/null +++ b/Robustness_Eval/GAIRAT/attack_generator.py @@ -0,0 +1,85 @@ +import numpy as np +from models import * +from torch.autograd import Variable + +def cwloss(output, target,confidence=50, num_classes=10): + # Compute the probability of the label class versus the maximum other + # The same implementation as in repo CAT https://github.com/sunblaze-ucb/curriculum-adversarial-training-CAT + target = target.data + target_onehot = torch.zeros(target.size() + (num_classes,)) + target_onehot = target_onehot.cuda() + target_onehot.scatter_(1, target.unsqueeze(1), 1.) + target_var = Variable(target_onehot, requires_grad=False) + real = (target_var * output).sum(1) + other = ((1. - target_var) * output - target_var * 10000.).max(1)[0] + loss = -torch.clamp(real - other + confidence, min=0.) # equiv to max(..., 0.) + loss = torch.sum(loss) + return loss + +# Geometry-aware projected gradient descent (GA-PGD) +def GA_PGD(model, data, target, epsilon, step_size, num_steps,loss_fn,category,rand_init): + model.eval() + Kappa = torch.zeros(len(data)) + if category == "trades": + x_adv = data.detach() + 0.001 * torch.randn(data.shape).cuda().detach() if rand_init else data.detach() + nat_output = model(data) + if category == "Madry": + x_adv = data.detach() + torch.from_numpy(np.random.uniform(-epsilon, epsilon, data.shape)).float().cuda() if rand_init else data.detach() + x_adv = torch.clamp(x_adv, 0.0, 1.0) + for k in range(num_steps): + x_adv.requires_grad_() + output = model(x_adv) + predict = output.max(1, keepdim=True)[1] + # Update Kappa + for p in range(len(x_adv)): + if predict[p] == target[p]: + Kappa[p] += 1 + model.zero_grad() + with torch.enable_grad(): + if loss_fn == "cent": + loss_adv = nn.CrossEntropyLoss(reduction="mean")(output, target) + if loss_fn == "cw": + loss_adv = cwloss(output,target) + if loss_fn == "kl": + criterion_kl = nn.KLDivLoss(size_average=False).cuda() + loss_adv = criterion_kl(F.log_softmax(output, dim=1),F.softmax(nat_output, dim=1)) + loss_adv.backward() + eta = step_size * x_adv.grad.sign() + # Update adversarial data + x_adv = x_adv.detach() + eta + x_adv = torch.min(torch.max(x_adv, data - epsilon), data + epsilon) + x_adv = torch.clamp(x_adv, 0.0, 1.0) + x_adv = Variable(x_adv, requires_grad=False) + return x_adv, Kappa + +def eval_clean(model, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.cuda(), target.cuda() + output = model(data) + test_loss += F.cross_entropy(output, target, size_average=False).item() + pred = output.max(1, keepdim=True)[1] + correct += pred.eq(target.view_as(pred)).sum().item() + test_loss /= len(test_loader.dataset) + test_accuracy = correct / len(test_loader.dataset) + return test_loss, test_accuracy + +def eval_robust(model, test_loader, perturb_steps, epsilon, step_size, loss_fn, category, random): + model.eval() + test_loss = 0 + correct = 0 + with torch.enable_grad(): + for data, target in test_loader: + data, target = data.cuda(), target.cuda() + x_adv, _ = GA_PGD(model,data,target,epsilon,step_size,perturb_steps,loss_fn,category,rand_init=random) + output = model(x_adv) + test_loss += F.cross_entropy(output, target, size_average=False).item() + pred = output.max(1, keepdim=True)[1] + correct += pred.eq(target.view_as(pred)).sum().item() + test_loss /= len(test_loader.dataset) + test_accuracy = correct / len(test_loader.dataset) + return test_loss, test_accuracy + diff --git a/Robustness_Eval/GAIRAT/attack_pgd.py b/Robustness_Eval/GAIRAT/attack_pgd.py new file mode 100644 index 0000000..66a36e5 --- /dev/null +++ b/Robustness_Eval/GAIRAT/attack_pgd.py @@ -0,0 +1,58 @@ +### PGD implementation + +""" +For PGD plus evaluation +Based on https://github.com/yaircarmon/semisup-adv +""" + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from torch.autograd import Variable +import torch.optim as optim +import logging + + +def pgd(model, + X, + y, + epsilon=8 / 255, + num_steps=20, + step_size=0.01, + random_start=True): + out = model(X) + is_correct_natural = (out.max(1)[1] == y).float().cpu().numpy() + perturbation = torch.zeros_like(X, requires_grad=True) + + if random_start: + perturbation = torch.rand_like(X, requires_grad=True) + perturbation.data = perturbation.data * 2 * epsilon - epsilon + + is_correct_adv = [] + opt = optim.SGD([perturbation], lr=1e-3) # This is just to clear the grad + + for _ in range(num_steps): + opt.zero_grad() + + with torch.enable_grad(): + loss = nn.CrossEntropyLoss()(model(X + perturbation), y) + + loss.backward() + + perturbation.data = ( + perturbation + step_size * perturbation.grad.detach().sign()).clamp( + -epsilon, epsilon) + perturbation.data = torch.min(torch.max(perturbation.detach(), -X), + 1 - X) # clip X+delta to [0,1] + X_pgd = Variable(torch.clamp(X.data + perturbation.data, 0, 1.0), + requires_grad=False) + is_correct_adv.append(np.reshape( + (model(X_pgd).max(1)[1] == y).float().cpu().numpy(), + [-1, 1])) + + is_correct_adv = np.concatenate(is_correct_adv, axis=1) + return is_correct_natural, is_correct_adv + + diff --git a/Robustness_Eval/GAIRAT/datasets.py b/Robustness_Eval/GAIRAT/datasets.py new file mode 100644 index 0000000..52c6070 --- /dev/null +++ b/Robustness_Eval/GAIRAT/datasets.py @@ -0,0 +1,213 @@ +""" +Datasets with unlabeled (or pseudo-labeled) data for PGD plus evaluation +Based on https://github.com/yaircarmon/semisup-adv +""" + +from torchvision.datasets import CIFAR10, SVHN +from torch.utils.data import Sampler, Dataset +import torch +import numpy as np + +import os +import pickle + +import logging + +DATASETS = ['cifar10', 'svhn'] + + +class SemiSupervisedDataset(Dataset): + def __init__(self, + base_dataset='cifar10', + take_amount=None, + take_amount_seed=13, + add_svhn_extra=False, + aux_data_filename=None, + add_aux_labels=False, + aux_take_amount=None, + train=False, + **kwargs): + """A dataset with auxiliary pseudo-labeled data""" + + if base_dataset == 'cifar10': + self.dataset = CIFAR10(train=train, **kwargs) + elif base_dataset == 'svhn': + if train: + self.dataset = SVHN(split='train', **kwargs) + else: + self.dataset = SVHN(split='test', **kwargs) + # because torchvision is annoying + self.dataset.targets = self.dataset.labels + self.targets = list(self.targets) + + if train and add_svhn_extra: + svhn_extra = SVHN(split='extra', **kwargs) + self.data = np.concatenate([self.data, svhn_extra.data]) + self.targets.extend(svhn_extra.labels) + else: + raise ValueError('Dataset %s not supported' % base_dataset) + self.base_dataset = base_dataset + self.train = train + + if self.train: + if take_amount is not None: + rng_state = np.random.get_state() + np.random.seed(take_amount_seed) + take_inds = np.random.choice(len(self.sup_indices), + take_amount, replace=False) + np.random.set_state(rng_state) + + logger = logging.getLogger() + logger.info('Randomly taking only %d/%d examples from training' + ' set, seed=%d, indices=%s', + take_amount, len(self.sup_indices), + take_amount_seed, take_inds) + self.targets = self.targets[take_inds] + self.data = self.data[take_inds] + + self.sup_indices = list(range(len(self.targets))) + self.unsup_indices = [] + + if aux_data_filename is not None: + aux_path = os.path.join(kwargs['root'], aux_data_filename) + aux_path = os.path.expanduser(aux_path) + print("Loading data from %s" % aux_path) + with open(aux_path, 'rb') as f: + aux = pickle.load(f) + aux_data = aux['data'] + aux_targets = aux['extrapolated_targets'] + orig_len = len(self.data) + + if aux_take_amount is not None: + rng_state = np.random.get_state() + np.random.seed(take_amount_seed) + take_inds = np.random.choice(len(aux_data), + aux_take_amount, replace=False) + np.random.set_state(rng_state) + + logger = logging.getLogger() + logger.info( + 'Randomly taking only %d/%d examples from aux data' + ' set, seed=%d, indices=%s', + aux_take_amount, len(aux_data), + take_amount_seed, take_inds) + aux_data = aux_data[take_inds] + aux_targets = aux_targets[take_inds] + + self.data = np.concatenate((self.data, aux_data), axis=0) + + if not add_aux_labels: + self.targets.extend([-1] * len(aux_data)) + else: + self.targets.extend(aux_targets) + # note that we use unsup indices to track the labeled datapoints + # whose labels are "fake" + self.unsup_indices.extend( + range(orig_len, orig_len+len(aux_data))) + + logger = logging.getLogger() + logger.info("Training set") + logger.info("Number of training samples: %d", len(self.targets)) + logger.info("Number of supervised samples: %d", + len(self.sup_indices)) + logger.info("Number of unsup samples: %d", len(self.unsup_indices)) + logger.info("Label (and pseudo-label) histogram: %s", + tuple( + zip(*np.unique(self.targets, return_counts=True)))) + logger.info("Shape of training data: %s", np.shape(self.data)) + + # Test set + else: + self.sup_indices = list(range(len(self.targets))) + self.unsup_indices = [] + + logger = logging.getLogger() + logger.info("Test set") + logger.info("Number of samples: %d", len(self.targets)) + logger.info("Label histogram: %s", + tuple( + zip(*np.unique(self.targets, return_counts=True)))) + logger.info("Shape of data: %s", np.shape(self.data)) + + @property + def data(self): + return self.dataset.data + + @data.setter + def data(self, value): + self.dataset.data = value + + @property + def targets(self): + return self.dataset.targets + + @targets.setter + def targets(self, value): + self.dataset.targets = value + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, item): + self.dataset.labels = self.targets # because torchvision is annoying + return self.dataset[item] + + def __repr__(self): + fmt_str = 'Semisupervised Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Training: {}\n'.format(self.train) + fmt_str += ' Root Location: {}\n'.format(self.dataset.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.dataset.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.dataset.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + +class SemiSupervisedSampler(Sampler): + """Balanced sampling from the labeled and unlabeled data""" + def __init__(self, sup_inds, unsup_inds, batch_size, unsup_fraction=0.5, + num_batches=None): + if unsup_fraction is None or unsup_fraction < 0: + self.sup_inds = sup_inds + unsup_inds + unsup_fraction = 0.0 + else: + self.sup_inds = sup_inds + self.unsup_inds = unsup_inds + + self.batch_size = batch_size + unsup_batch_size = int(batch_size * unsup_fraction) + self.sup_batch_size = batch_size - unsup_batch_size + + if num_batches is not None: + self.num_batches = num_batches + else: + self.num_batches = int( + np.ceil(len(self.sup_inds) / self.sup_batch_size)) + + super().__init__(None) + + def __iter__(self): + batch_counter = 0 + while batch_counter < self.num_batches: + sup_inds_shuffled = [self.sup_inds[i] + for i in torch.randperm(len(self.sup_inds))] + for sup_k in range(0, len(self.sup_inds), self.sup_batch_size): + if batch_counter == self.num_batches: + break + batch = sup_inds_shuffled[sup_k:(sup_k + self.sup_batch_size)] + if self.sup_batch_size < self.batch_size: + batch.extend([self.unsup_inds[i] for i in + torch.randint(high=len(self.unsup_inds), + size=( + self.batch_size - len( + batch),), + dtype=torch.int64)]) + # this shuffle operation is very important, without it + # batch-norm / DataParallel hell ensues + np.random.shuffle(batch) + yield batch + batch_counter += 1 + + def __len__(self): + return self.num_batches diff --git a/Robustness_Eval/GAIRAT/earlystop.py b/Robustness_Eval/GAIRAT/earlystop.py new file mode 100644 index 0000000..3120a2e --- /dev/null +++ b/Robustness_Eval/GAIRAT/earlystop.py @@ -0,0 +1,107 @@ +from models import * +import torch +import numpy as np + +# Geometry-aware early stopped PGD +def GA_earlystop(model, data, target, step_size, epsilon, perturb_steps, tau, type, random, omega): + # Based on code from https://github.com/zjfheart/Friendly-Adversarial-Training + + model.eval() + K = perturb_steps + count = 0 + + output_target = [] + output_adv = [] + output_natural = [] + output_Kappa = [] + + control = torch.zeros(len(target)).cuda() + control += tau + Kappa = torch.zeros(len(data)).cuda() + + if random == False: + iter_adv = data.cuda().detach() + else: + + if type == "fat_for_trades" : + iter_adv = data.detach() + 0.001 * torch.randn(data.shape).cuda().detach() + iter_adv = torch.clamp(iter_adv, 0.0, 1.0) + if type == "fat" or "fat_for_mart": + iter_adv = data.detach() + torch.from_numpy(np.random.uniform(-epsilon, epsilon, data.shape)).float().cuda() + iter_adv = torch.clamp(iter_adv, 0.0, 1.0) + iter_clean_data = data.cuda().detach() + iter_target = target.cuda().detach() + output_iter_clean_data = model(data) + + while K>0: + iter_adv.requires_grad_() + output = model(iter_adv) + pred = output.max(1, keepdim=True)[1] + output_index = [] + iter_index = [] + + for idx in range(len(pred)): + if pred[idx] != target[idx]: + if control[idx]==0: + output_index.append(idx) + else: + control[idx]-=1 + iter_index.append(idx) + else: + # Update Kappa + Kappa[idx] += 1 + iter_index.append(idx) + + if (len(output_index)!=0): + if (len(output_target) == 0): + # incorrect adv data should not keep iterated + output_adv = iter_adv[output_index].reshape(-1, 3, 32, 32).cuda() + output_natural = iter_clean_data[output_index].reshape(-1, 3, 32, 32).cuda() + output_target = iter_target[output_index].reshape(-1).cuda() + output_Kappa = Kappa[output_index].reshape(-1).cuda() + else: + # incorrect adv data should not keep iterated + output_adv = torch.cat((output_adv, iter_adv[output_index].reshape(-1, 3, 32, 32).cuda()), dim=0) + output_natural = torch.cat((output_natural, iter_clean_data[output_index].reshape(-1, 3, 32, 32).cuda()), dim=0) + output_target = torch.cat((output_target, iter_target[output_index].reshape(-1).cuda()), dim=0) + output_Kappa = torch.cat((output_Kappa, Kappa[output_index].reshape(-1).cuda()), dim=0) + + model.zero_grad() + with torch.enable_grad(): + if type == "fat" or type == "fat_for_mart": + loss_adv = nn.CrossEntropyLoss()(output, iter_target) + if type == "fat_for_trades": + criterion_kl = nn.KLDivLoss(size_average=False).cuda() + loss_adv = criterion_kl(F.log_softmax(output, dim=1),F.softmax(output_iter_clean_data, dim=1)) + loss_adv.backward(retain_graph=True) + grad = iter_adv.grad + + if len(iter_index) != 0: + Kappa = Kappa[iter_index] + control = control[iter_index] + iter_adv = iter_adv[iter_index] + iter_clean_data = iter_clean_data[iter_index] + iter_target = iter_target[iter_index] + output_iter_clean_data = output_iter_clean_data[iter_index] + grad = grad[iter_index] + eta = step_size * grad.sign() + iter_adv = iter_adv.detach() + eta + omega * torch.randn(iter_adv.shape).detach().cuda() + iter_adv = torch.min(torch.max(iter_adv, iter_clean_data - epsilon), iter_clean_data + epsilon) + iter_adv = torch.clamp(iter_adv, 0, 1) + count += len(iter_target) + else: + return output_adv, output_target, output_natural, count, output_Kappa + K = K-1 + + if (len(output_target) == 0): + output_target = iter_target.reshape(-1).squeeze().cuda() + output_adv = iter_adv.reshape(-1, 3, 32, 32).cuda() + output_natural = iter_clean_data.reshape(-1, 3, 32, 32).cuda() + output_Kappa = Kappa.reshape(-1).cuda() + else: + output_adv = torch.cat((output_adv, iter_adv.reshape(-1, 3, 32, 32)), dim=0).cuda() + output_target = torch.cat((output_target, iter_target.reshape(-1)), dim=0).squeeze().cuda() + output_natural = torch.cat((output_natural, iter_clean_data.reshape(-1, 3, 32, 32).cuda()),dim=0).cuda() + output_Kappa = torch.cat((output_Kappa, Kappa.reshape(-1)),dim=0).squeeze().cuda() + + return output_adv, output_target, output_natural, count, output_Kappa diff --git a/Robustness_Eval/GAIRAT/eval_PGD_plus.py b/Robustness_Eval/GAIRAT/eval_PGD_plus.py new file mode 100644 index 0000000..621d506 --- /dev/null +++ b/Robustness_Eval/GAIRAT/eval_PGD_plus.py @@ -0,0 +1,246 @@ +""" +Evaluate robustness against specific attack. +Based on code from https://github.com/yaircarmon/semisup-adv +""" + +import os +import json +import numpy as np +import argparse +import logging +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from torch.autograd import Variable +from datasets import SemiSupervisedDataset, DATASETS +from torchvision import transforms +from attack_pgd import pgd +from attack_cw import cw +import torch.backends.cudnn as cudnn +from utils_eval import get_model + + +def eval_adv_test(model, device, test_loader, attack, attack_params, + results_dir, num_eval_batches): + """ + evaluate model by white-box attack + """ + model.eval() + if attack == 'pgd': + restarts_matrices = [] + for restart in range(attack_params['num_restarts']): + is_correct_adv_rows = [] + count = 0 + batch_num = 0 + natural_num_correct = 0 + for data, target in test_loader: + batch_num = batch_num + 1 + if num_eval_batches and batch_num > num_eval_batches: + break + data, target = data.to(device), target.to(device) + count += len(target) + X, y = Variable(data, requires_grad=True), Variable(target) + # is_correct_adv has batch_size*num_iterations dimensions + is_correct_natural, is_correct_adv = pgd( + model, X, y, + epsilon=attack_params['epsilon'], + num_steps=attack_params['num_steps'], + step_size=attack_params['step_size'], + random_start=attack_params['random_start']) + natural_num_correct += is_correct_natural.sum() + is_correct_adv_rows.append(is_correct_adv) + + is_correct_adv_matrix = np.concatenate(is_correct_adv_rows, axis=0) + restarts_matrices.append(is_correct_adv_matrix) + + is_correct_adv_over_restarts = np.stack(restarts_matrices, axis=-1) + num_correct_adv = is_correct_adv_over_restarts.prod( + axis=-1).prod(axis=-1).sum() + + logging.info("Accuracy after %d restarts: %.4g%%" % + (restart + 1, 100 * num_correct_adv / count)) + stats = {'attack': 'pgd', + 'count': count, + 'attack_params': attack_params, + 'natural_accuracy': natural_num_correct / count, + 'is_correct_adv_array': is_correct_adv_over_restarts, + 'robust_accuracy': num_correct_adv / count, + 'restart_num': restart + } + + np.save(os.path.join(results_dir, 'pgd_results.npy'), stats) + + elif attack == 'cw': + all_linf_distances = [] + count = 0 + for data, target in test_loader: + logging.info('Batch: %g', count) + count = count + 1 + if num_eval_batches and count > num_eval_batches: + break + data, target = data.to(device), target.to(device) + X, y = Variable(data, requires_grad=True), Variable(target) + batch_linf_distances = cw(model, X, y, + binary_search_steps=attack_params[ + 'binary_search_steps'], + max_iterations=attack_params[ + 'max_iterations'], + learning_rate=attack_params[ + 'learning_rate'], + initial_const=attack_params[ + 'initial_const'], + tau_decrease_factor=attack_params[ + 'tau_decrease_factor']) + all_linf_distances.append(batch_linf_distances) + stats = {'attack': 'cw', + 'attack_params': attack_params, + 'linf_distances': np.array(all_linf_distances), + } + np.save(os.path.join(results_dir, 'cw_results.npy'), stats) + + else: + raise ValueError('Unknown attack %s' % attack) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='PyTorch CIFAR Attack Evaluation') + parser.add_argument('--dataset', type=str, default='cifar10', + choices=DATASETS, + help='The dataset') + parser.add_argument('--model_path', + help='Model for attack evaluation') + parser.add_argument('--model', '-m', default='wrn-32-10', type=str, + help='Name of the model: wrn-XX-XX, resnet-XX, small-cnn') + parser.add_argument('--output_suffix', default='', type=str, + help='String to add to log filename') + parser.add_argument('--batch_size', type=int, default=128, metavar='N', + help='Input batch size for testing (default: 200)') + parser.add_argument('--no_cuda', action='store_true', default=False, + help='Disables CUDA training') + parser.add_argument('--epsilon', default=0.031, type=float, + help='Attack perturbation magnitude') + parser.add_argument('--attack', default='pgd', type=str, + help='Attack type (CW requires FoolBox)', + choices=('pgd', 'cw')) + parser.add_argument('--num_steps', default=40, type=int, + help='Number of PGD steps') + parser.add_argument('--step_size', default=0.01, type=float, + help='PGD step size') + parser.add_argument('--num_restarts', default=5, type=int, + help='Number of restarts for PGD attack') + parser.add_argument('--no_random_start', dest='random_start', + action='store_false', + help='Disable random PGD initialization') + parser.add_argument('--binary_search_steps', default=5, type=int, + help='Number of binary search steps for CW attack') + parser.add_argument('--max_iterations', default=1000, type=int, + help='Max number of Adam iterations in each CW' + ' optimization') + parser.add_argument('--learning_rate', default=5E-3, type=float, + help='Learning rate for CW attack') + parser.add_argument('--initial_const', default=1E-2, type=float, + help='Initial constant for CW attack') + parser.add_argument('--tau_decrease_factor', default=0.9, type=float, + help='Tau decrease factor for CW attack') + parser.add_argument('--random_seed', default=0, type=int, + help='Random seed for permutation of test instances') + parser.add_argument('--num_eval_batches', default=None, type=int, + help='Number of batches to run evalaution on') + parser.add_argument('--shuffle_testset', action='store_true', default=False, + help='Shuffles the test set') + + args = parser.parse_args() + torch.manual_seed(args.random_seed) + + output_dir, checkpoint_name = os.path.split(args.model_path) + + results_dir = os.path.join('./', args.output_suffix) + if not os.path.isdir(results_dir): + os.mkdir(results_dir) + #epoch = int(re.search('epoch(\d+)', checkpoint_name).group(1)) + epoch = 0 + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(message)s", + handlers=[ + logging.FileHandler(os.path.join(results_dir, + 'attack_epoch%d%s.log' % + (epoch, args.output_suffix))), + logging.StreamHandler() + ]) + logger = logging.getLogger() + + + + logging.info('Attack evaluation') + logging.info('Args: %s' % args) + + # settings + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + dl_kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} + + # set up data loader + transform_test = transforms.Compose([transforms.ToTensor(), ]) + testset = SemiSupervisedDataset(base_dataset=args.dataset, + train=False, root='./data/cifar-10', + download=True, + transform=transform_test) + + if args.shuffle_testset: + np.random.seed(123) + logging.info("Permuting testset") + permutation = np.random.permutation(len(testset)) + testset.data = testset.data[permutation, :] + testset.targets = [testset.targets[i] for i in permutation] + + test_loader = torch.utils.data.DataLoader(testset, + batch_size=args.batch_size, + shuffle=False, **dl_kwargs) + + checkpoint = torch.load(args.model_path) + state_dict = checkpoint.get('state_dict', checkpoint) + num_classes = checkpoint.get('num_classes', 10) + normalize_input = checkpoint.get('normalize_input', False) + model = get_model(args.model, num_classes=num_classes, + normalize_input=normalize_input) + if use_cuda: + model = torch.nn.DataParallel(model).cuda() + cudnn.benchmark = True + if not all([k.startswith('module') for k in state_dict]): + state_dict = {'module.' + k: v for k, v in state_dict.items()} + else: + def strip_data_parallel(s): + if s.startswith('module'): + return s[len('module.'):] + else: + return s + state_dict = {strip_data_parallel(k): v for k, v in state_dict.items()} + model.load_state_dict(state_dict) + + attack_params = { + 'epsilon': args.epsilon, + 'seed': args.random_seed + } + if args.attack == 'pgd': + attack_params.update({ + 'num_restarts': args.num_restarts, + 'step_size': args.step_size, + 'num_steps': args.num_steps, + 'random_start': args.random_start, + }) + elif args.attack == 'cw': + attack_params.update({ + 'binary_search_steps': args.binary_search_steps, + 'max_iterations': args.max_iterations, + 'learning_rate': args.learning_rate, + 'initial_const': args.initial_const, + 'tau_decrease_factor': args.tau_decrease_factor + }) + + logging.info('Running %s' % attack_params) + eval_adv_test(model, device, test_loader, attack=args.attack, + attack_params=attack_params, results_dir=results_dir, + num_eval_batches=args.num_eval_batches) diff --git a/Robustness_Eval/GAIRAT/images/GAIRAT_learning_obj.png b/Robustness_Eval/GAIRAT/images/GAIRAT_learning_obj.png new file mode 100644 index 0000000..3d60aa1 Binary files /dev/null and b/Robustness_Eval/GAIRAT/images/GAIRAT_learning_obj.png differ diff --git a/Robustness_Eval/GAIRAT/images/diff_net_error_white.png b/Robustness_Eval/GAIRAT/images/diff_net_error_white.png new file mode 100644 index 0000000..8c33a04 Binary files /dev/null and b/Robustness_Eval/GAIRAT/images/diff_net_error_white.png differ diff --git a/Robustness_Eval/GAIRAT/images/eps_error_white.png b/Robustness_Eval/GAIRAT/images/eps_error_white.png new file mode 100644 index 0000000..2ee64a0 Binary files /dev/null and b/Robustness_Eval/GAIRAT/images/eps_error_white.png differ diff --git a/Robustness_Eval/GAIRAT/images/motivation.png b/Robustness_Eval/GAIRAT/images/motivation.png new file mode 100644 index 0000000..c2e64f3 Binary files /dev/null and b/Robustness_Eval/GAIRAT/images/motivation.png differ diff --git a/Robustness_Eval/GAIRAT/images/pca01.png b/Robustness_Eval/GAIRAT/images/pca01.png new file mode 100644 index 0000000..f6b735d Binary files /dev/null and b/Robustness_Eval/GAIRAT/images/pca01.png differ diff --git a/Robustness_Eval/GAIRAT/models/__init__.py b/Robustness_Eval/GAIRAT/models/__init__.py new file mode 100644 index 0000000..3b308d4 --- /dev/null +++ b/Robustness_Eval/GAIRAT/models/__init__.py @@ -0,0 +1,5 @@ +from .resnet import * +from .preact_resnet import * +from .wide_resnet import * +from .small_cnn import * +from .wrn_madry import * diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/__init__.cpython-36.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..2fca48d Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/__init__.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/__init__.cpython-37.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..c724e89 Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/__init__.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/preact_resnet.cpython-36.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/preact_resnet.cpython-36.pyc new file mode 100644 index 0000000..7c7e1b1 Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/preact_resnet.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/preact_resnet.cpython-37.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/preact_resnet.cpython-37.pyc new file mode 100644 index 0000000..9a32489 Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/preact_resnet.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/resnet.cpython-36.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/resnet.cpython-36.pyc new file mode 100644 index 0000000..74a0523 Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/resnet.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/resnet.cpython-37.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/resnet.cpython-37.pyc new file mode 100644 index 0000000..3a79b02 Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/resnet.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/small_cnn.cpython-36.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/small_cnn.cpython-36.pyc new file mode 100644 index 0000000..3c93a8d Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/small_cnn.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/small_cnn.cpython-37.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/small_cnn.cpython-37.pyc new file mode 100644 index 0000000..ee9b5d9 Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/small_cnn.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/wide_resnet.cpython-36.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/wide_resnet.cpython-36.pyc new file mode 100644 index 0000000..ec36e90 Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/wide_resnet.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/wide_resnet.cpython-37.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/wide_resnet.cpython-37.pyc new file mode 100644 index 0000000..8bfe316 Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/wide_resnet.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/wrn_madry.cpython-36.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/wrn_madry.cpython-36.pyc new file mode 100644 index 0000000..234939a Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/wrn_madry.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/__pycache__/wrn_madry.cpython-37.pyc b/Robustness_Eval/GAIRAT/models/__pycache__/wrn_madry.cpython-37.pyc new file mode 100644 index 0000000..71e1b6e Binary files /dev/null and b/Robustness_Eval/GAIRAT/models/__pycache__/wrn_madry.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/models/preact_resnet.py b/Robustness_Eval/GAIRAT/models/preact_resnet.py new file mode 100644 index 0000000..aef2116 --- /dev/null +++ b/Robustness_Eval/GAIRAT/models/preact_resnet.py @@ -0,0 +1,120 @@ +'''Pre-activation ResNet in PyTorch. + +Reference: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Identity Mappings in Deep Residual Networks. arXiv:1603.05027 +''' +import torch +import torch.nn as nn +import torch.nn.functional as F + +from torch.autograd import Variable + + +class PreActBlock(nn.Module): + '''Pre-activation version of the BasicBlock.''' + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(PreActBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False) + ) + + def forward(self, x): + out = F.relu(self.bn1(x)) + shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x + out = self.conv1(out) + out = self.conv2(F.relu(self.bn2(out))) + out += shortcut + return out + + +class PreActBottleneck(nn.Module): + '''Pre-activation version of the original Bottleneck module.''' + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(PreActBottleneck, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) + + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False) + ) + + def forward(self, x): + out = F.relu(self.bn1(x)) + shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x + out = self.conv1(out) + out = self.conv2(F.relu(self.bn2(out))) + out = self.conv3(F.relu(self.bn3(out))) + out += shortcut + return out + + +class PreActResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(PreActResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512*block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = self.conv1(x) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def PreActResNet18(): + return PreActResNet(PreActBlock, [2,2,2,2]) + +def PreActResNet34(): + return PreActResNet(PreActBlock, [3,4,6,3]) + +def PreActResNet50(): + return PreActResNet(PreActBottleneck, [3,4,6,3]) + +def PreActResNet101(): + return PreActResNet(PreActBottleneck, [3,4,23,3]) + +def PreActResNet152(): + return PreActResNet(PreActBottleneck, [3,8,36,3]) + + +def test(): + net = PreActResNet18() + y = net(Variable(torch.randn(1,3,32,32))) + print(y.size()) + +# test() diff --git a/Robustness_Eval/GAIRAT/models/resnet.py b/Robustness_Eval/GAIRAT/models/resnet.py new file mode 100644 index 0000000..0dfbfb5 --- /dev/null +++ b/Robustness_Eval/GAIRAT/models/resnet.py @@ -0,0 +1,123 @@ +'''ResNet in PyTorch. + +For Pre-activation ResNet, see 'preact_resnet.py'. + +Reference: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 +''' +import torch +import torch.nn as nn +import torch.nn.functional as F + +from torch.autograd import Variable + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion*planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512*block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def ResNet18(num_classes=10): + return ResNet(BasicBlock, [2,2,2,2], num_classes=num_classes) + +def ResNet34(): + return ResNet(BasicBlock, [3,4,6,3]) + +def ResNet50(): + return ResNet(Bottleneck, [3,4,6,3]) + +def ResNet101(): + return ResNet(Bottleneck, [3,4,23,3]) + +def ResNet152(): + return ResNet(Bottleneck, [3,8,36,3]) + + +def test(): + net = ResNet18() + y = net(Variable(torch.randn(1,3,32,32))) + print(y.size()) + print(net) +# test() \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/models/small_cnn.py b/Robustness_Eval/GAIRAT/models/small_cnn.py new file mode 100644 index 0000000..c6a500c --- /dev/null +++ b/Robustness_Eval/GAIRAT/models/small_cnn.py @@ -0,0 +1,73 @@ +from collections import OrderedDict +import torch.nn as nn +import torch +from torch.autograd import Variable + +class SmallCNN(nn.Module): + def __init__(self, num_classes=10): + super(SmallCNN, self).__init__() + + self.block1_conv1 = nn.Conv2d(3, 64, 3, padding=1) + self.block1_conv2 = nn.Conv2d(64, 64, 3, padding=1) + self.block1_pool1 = nn.MaxPool2d(2, 2) + self.batchnorm1_1 = nn.BatchNorm2d(64) + self.batchnorm1_2 = nn.BatchNorm2d(64) + + self.block2_conv1 = nn.Conv2d(64, 128, 3, padding=1) + self.block2_conv2 = nn.Conv2d(128, 128, 3, padding=1) + self.block2_pool1 = nn.MaxPool2d(2, 2) + self.batchnorm2_1 = nn.BatchNorm2d(128) + self.batchnorm2_2 = nn.BatchNorm2d(128) + + self.block3_conv1 = nn.Conv2d(128, 196, 3, padding=1) + self.block3_conv2 = nn.Conv2d(196, 196, 3, padding=1) + self.block3_pool1 = nn.MaxPool2d(2, 2) + self.batchnorm3_1 = nn.BatchNorm2d(196) + self.batchnorm3_2 = nn.BatchNorm2d(196) + + self.activ = nn.ReLU() + + self.fc1 = nn.Linear(196*4*4,256) + self.fc2 = nn.Linear(256,num_classes) + + def forward(self, x): + #block1 + x = self.block1_conv1(x) + x = self.batchnorm1_1(x) + x = self.activ(x) + x = self.block1_conv2(x) + x = self.batchnorm1_2(x) + x = self.activ(x) + x = self.block1_pool1(x) + + #block2 + x = self.block2_conv1(x) + x = self.batchnorm2_1(x) + x = self.activ(x) + x = self.block2_conv2(x) + x = self.batchnorm2_2(x) + x = self.activ(x) + x = self.block2_pool1(x) + #block3 + x = self.block3_conv1(x) + x = self.batchnorm3_1(x) + x = self.activ(x) + x = self.block3_conv2(x) + x = self.batchnorm3_2(x) + x = self.activ(x) + x = self.block3_pool1(x) + + x = x.view(-1,196*4*4) + x = self.fc1(x) + x = self.activ(x) + x = self.fc2(x) + + return x + +def small_cnn(num_classes=10): + return SmallCNN(num_classes=num_classes) +def test(): + net = small_cnn() + y = net(Variable(torch.randn(1,3,32,32))) + print(y.size()) + print(net) \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/models/wide_resnet.py b/Robustness_Eval/GAIRAT/models/wide_resnet.py new file mode 100644 index 0000000..bcad5ae --- /dev/null +++ b/Robustness_Eval/GAIRAT/models/wide_resnet.py @@ -0,0 +1,98 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class Wide_ResNet(nn.Module): + def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0): + super(Wide_ResNet, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 4) % 6 == 0) + n = (depth - 4) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + return self.fc(out) +def test(): + net = Wide_ResNet() + y = net(Variable(torch.randn(1, 3, 32, 32))) + #print(y.size()) + print(net) +# test() \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/models/wrn_madry.py b/Robustness_Eval/GAIRAT/models/wrn_madry.py new file mode 100644 index 0000000..cbba758 --- /dev/null +++ b/Robustness_Eval/GAIRAT/models/wrn_madry.py @@ -0,0 +1,98 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class Wide_ResNet_Madry(nn.Module): + def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0): + super(Wide_ResNet_Madry, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 2) % 6 == 0) + n = (depth - 2) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + # self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + return self.fc(out) +def test(): + net = Wide_ResNet_Madry() + y = net(Variable(torch.randn(1, 3, 32, 32))) + #print(y.size()) + print(net) +# test() \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/utils/__init__.py b/Robustness_Eval/GAIRAT/utils/__init__.py new file mode 100644 index 0000000..bce5561 --- /dev/null +++ b/Robustness_Eval/GAIRAT/utils/__init__.py @@ -0,0 +1,10 @@ +"""Useful utils +""" +from .misc import * +from .logger import * +from .eval import * + +# progress bar +import os, sys +sys.path.append(os.path.join(os.path.dirname(__file__), "progress")) +#from progress.bar import Bar as Bar \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/utils/__pycache__/__init__.cpython-36.pyc b/Robustness_Eval/GAIRAT/utils/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..8089254 Binary files /dev/null and b/Robustness_Eval/GAIRAT/utils/__pycache__/__init__.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/utils/__pycache__/__init__.cpython-37.pyc b/Robustness_Eval/GAIRAT/utils/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..ca5c7b9 Binary files /dev/null and b/Robustness_Eval/GAIRAT/utils/__pycache__/__init__.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/utils/__pycache__/eval.cpython-36.pyc b/Robustness_Eval/GAIRAT/utils/__pycache__/eval.cpython-36.pyc new file mode 100644 index 0000000..ec72ac4 Binary files /dev/null and b/Robustness_Eval/GAIRAT/utils/__pycache__/eval.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/utils/__pycache__/eval.cpython-37.pyc b/Robustness_Eval/GAIRAT/utils/__pycache__/eval.cpython-37.pyc new file mode 100644 index 0000000..7c08422 Binary files /dev/null and b/Robustness_Eval/GAIRAT/utils/__pycache__/eval.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/utils/__pycache__/logger.cpython-36.pyc b/Robustness_Eval/GAIRAT/utils/__pycache__/logger.cpython-36.pyc new file mode 100644 index 0000000..d1c7fdb Binary files /dev/null and b/Robustness_Eval/GAIRAT/utils/__pycache__/logger.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/utils/__pycache__/logger.cpython-37.pyc b/Robustness_Eval/GAIRAT/utils/__pycache__/logger.cpython-37.pyc new file mode 100644 index 0000000..0e2a93a Binary files /dev/null and b/Robustness_Eval/GAIRAT/utils/__pycache__/logger.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/utils/__pycache__/misc.cpython-36.pyc b/Robustness_Eval/GAIRAT/utils/__pycache__/misc.cpython-36.pyc new file mode 100644 index 0000000..a2e915f Binary files /dev/null and b/Robustness_Eval/GAIRAT/utils/__pycache__/misc.cpython-36.pyc differ diff --git a/Robustness_Eval/GAIRAT/utils/__pycache__/misc.cpython-37.pyc b/Robustness_Eval/GAIRAT/utils/__pycache__/misc.cpython-37.pyc new file mode 100644 index 0000000..26ef7d5 Binary files /dev/null and b/Robustness_Eval/GAIRAT/utils/__pycache__/misc.cpython-37.pyc differ diff --git a/Robustness_Eval/GAIRAT/utils/eval.py b/Robustness_Eval/GAIRAT/utils/eval.py new file mode 100644 index 0000000..5051350 --- /dev/null +++ b/Robustness_Eval/GAIRAT/utils/eval.py @@ -0,0 +1,18 @@ +from __future__ import print_function, absolute_import + +__all__ = ['accuracy'] + +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/utils/logger.py b/Robustness_Eval/GAIRAT/utils/logger.py new file mode 100644 index 0000000..7eb5c67 --- /dev/null +++ b/Robustness_Eval/GAIRAT/utils/logger.py @@ -0,0 +1,127 @@ +# A simple torch style logger +# (C) Wei YANG 2017 +from __future__ import absolute_import +import matplotlib.pyplot as plt +import os +import sys +import numpy as np + +__all__ = ['Logger', 'LoggerMonitor', 'savefig'] + +def savefig(fname, dpi=None): + dpi = 150 if dpi == None else dpi + plt.savefig(fname, dpi=dpi) + +def plot_overlap(logger, names=None): + names = logger.names if names == None else names + numbers = logger.numbers + for _, name in enumerate(names): + x = np.arange(len(numbers[name])) + plt.plot(x, np.asarray(numbers[name])) + return [logger.title + '(' + name + ')' for name in names] + +class Logger(object): + '''Save training process to log file with simple plot function.''' + def __init__(self, fpath, title=None, resume=False): + self.file = None + self.resume = resume + self.title = '' if title == None else title + if fpath is not None: + if resume: + self.file = open(fpath, 'r') + name = self.file.readline() + self.names = name.rstrip().split('\t') + self.numbers = {} + for _, name in enumerate(self.names): + self.numbers[name] = [] + + for numbers in self.file: + numbers = numbers.rstrip().split('\t') + for i in range(0, len(numbers)): + self.numbers[self.names[i]].append(numbers[i]) + self.file.close() + self.file = open(fpath, 'a') + else: + self.file = open(fpath, 'w') + + def set_names(self, names): + if self.resume: + pass + # initialize numbers as empty list + self.numbers = {} + self.names = names + for _, name in enumerate(self.names): + self.file.write(name) + self.file.write('\t') + self.numbers[name] = [] + self.file.write('\n') + self.file.flush() + + + def append(self, numbers): + assert len(self.names) == len(numbers), 'Numbers do not match names' + for index, num in enumerate(numbers): + self.file.write("{0:.6f}".format(num)) + self.file.write('\t') + self.numbers[self.names[index]].append(num) + self.file.write('\n') + self.file.flush() + + def plot(self, names=None): + names = self.names if names == None else names + numbers = self.numbers + for _, name in enumerate(names): + x = np.arange(len(numbers[name])) + plt.plot(x, np.asarray(numbers[name])) + plt.legend([self.title + '(' + name + ')' for name in names]) + plt.grid(True) + + def close(self): + if self.file is not None: + self.file.close() + +class LoggerMonitor(object): + '''Load and visualize multiple logs.''' + def __init__ (self, paths): + '''paths is a distionary with {name:filepath} pair''' + self.loggers = [] + for title, path in paths.items(): + logger = Logger(path, title=title, resume=True) + self.loggers.append(logger) + + def plot(self, names=None): + plt.figure() + plt.subplot(121) + legend_text = [] + for logger in self.loggers: + legend_text += plot_overlap(logger, names) + plt.legend(legend_text, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) + plt.grid(True) + +if __name__ == '__main__': + # # Example + # logger = Logger('test.txt') + # logger.set_names(['Train loss', 'Valid loss','Test loss']) + + # length = 100 + # t = np.arange(length) + # train_loss = np.exp(-t / 10.0) + np.random.rand(length) * 0.1 + # valid_loss = np.exp(-t / 10.0) + np.random.rand(length) * 0.1 + # test_loss = np.exp(-t / 10.0) + np.random.rand(length) * 0.1 + + # for i in range(0, length): + # logger.append([train_loss[i], valid_loss[i], test_loss[i]]) + # logger.plot() + + # Example: logger monitor + paths = { + 'resadvnet20':'/home/wyang/code/pytorch-classification/checkpoint/cifar10/resadvnet20/log.txt', + 'resadvnet32':'/home/wyang/code/pytorch-classification/checkpoint/cifar10/resadvnet32/log.txt', + 'resadvnet44':'/home/wyang/code/pytorch-classification/checkpoint/cifar10/resadvnet44/log.txt', + } + + field = ['Valid Acc.'] + + monitor = LoggerMonitor(paths) + monitor.plot(names=field) + savefig('test.eps') \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/utils/misc.py b/Robustness_Eval/GAIRAT/utils/misc.py new file mode 100644 index 0000000..d387f59 --- /dev/null +++ b/Robustness_Eval/GAIRAT/utils/misc.py @@ -0,0 +1,76 @@ +'''Some helper functions for PyTorch, including: + - get_mean_and_std: calculate the mean and std value of dataset. + - msr_init: net parameter initialization. + - progress_bar: progress bar mimic xlua.progress. +''' +import errno +import os +import sys +import time +import math + +import torch.nn as nn +import torch.nn.init as init +from torch.autograd import Variable + +__all__ = ['get_mean_and_std', 'init_params', 'mkdir_p', 'AverageMeter'] + + +def get_mean_and_std(dataset): + '''Compute the mean and std value of dataset.''' + dataloader = trainloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=2) + + mean = torch.zeros(3) + std = torch.zeros(3) + print('==> Computing mean and std..') + for inputs, targets in dataloader: + for i in range(3): + mean[i] += inputs[:,i,:,:].mean() + std[i] += inputs[:,i,:,:].std() + mean.div_(len(dataset)) + std.div_(len(dataset)) + return mean, std + +def init_params(net): + '''Init layer parameters.''' + for m in net.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal(m.weight, mode='fan_out') + if m.bias: + init.constant(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant(m.weight, 1) + init.constant(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal(m.weight, std=1e-3) + if m.bias: + init.constant(m.bias, 0) + +def mkdir_p(path): + '''make dir if not exist''' + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +class AverageMeter(object): + """Computes and stores the average and current value + Imported from https://github.com/pytorch/examples/blob/master/imagenet/main.py#L247-L262 + """ + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count \ No newline at end of file diff --git a/Robustness_Eval/GAIRAT/utils_eval.py b/Robustness_Eval/GAIRAT/utils_eval.py new file mode 100644 index 0000000..de39a27 --- /dev/null +++ b/Robustness_Eval/GAIRAT/utils_eval.py @@ -0,0 +1,47 @@ +""" +Utilities for PGD plus evaluation. +Based on code from https://github.com/yaircarmon/semisup-adv +""" +import os +import numpy as np + +from models.wrn_madry import Wide_ResNet_Madry +from models.resnet import * +from models.small_cnn import SmallCNN + +import torch +from torch.nn import Sequential, Module + + +def get_model(name, num_classes=10, normalize_input=False): + name_parts = name.split('-') + if name_parts[0] == 'wrn': + depth = int(name_parts[1]) + widen = int(name_parts[2]) + model = Wide_ResNet_Madry( + depth=depth, num_classes=num_classes, widen_factor=widen) + + elif name_parts[0] == 'small': + model = SmallCNN() + elif name_parts[0] == 'resnet': + model = ResNet18() + else: + raise ValueError('Could not parse model name %s' % name) + + if normalize_input: + model = Sequential(NormalizeInput(), model) + + return model + + +class NormalizeInput(Module): + def __init__(self, mean=(0.4914, 0.4822, 0.4465), + std=(0.2023, 0.1994, 0.2010)): + super().__init__() + + self.register_buffer('mean', torch.Tensor(mean).reshape(1, -1, 1, 1)) + self.register_buffer('std', torch.Tensor(std).reshape(1, -1, 1, 1)) + + def forward(self, x): + return (x - self.mean) / self.std + diff --git a/Robustness_Eval/augmix/LICENSE b/Robustness_Eval/augmix/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/Robustness_Eval/augmix/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/Robustness_Eval/augmix/allconv.py b/Robustness_Eval/augmix/allconv.py new file mode 100644 index 0000000..89d8c7b --- /dev/null +++ b/Robustness_Eval/augmix/allconv.py @@ -0,0 +1,77 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""AllConv implementation (https://arxiv.org/abs/1412.6806).""" +import math +import torch +import torch.nn as nn + + +class GELU(nn.Module): + + def forward(self, x): + return torch.sigmoid(1.702 * x) * x + + +def make_layers(cfg): + """Create a single layer.""" + layers = [] + in_channels = 3 + for v in cfg: + if v == 'Md': + layers += [nn.MaxPool2d(kernel_size=2, stride=2), nn.Dropout(p=0.5)] + elif v == 'A': + layers += [nn.AvgPool2d(kernel_size=8)] + elif v == 'NIN': + conv2d = nn.Conv2d(in_channels, in_channels, kernel_size=1, padding=1) + layers += [conv2d, nn.BatchNorm2d(in_channels), GELU()] + elif v == 'nopad': + conv2d = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=0) + layers += [conv2d, nn.BatchNorm2d(in_channels), GELU()] + else: + conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) + layers += [conv2d, nn.BatchNorm2d(v), GELU()] + in_channels = v + return nn.Sequential(*layers) + + +class AllConvNet(nn.Module): + """AllConvNet main class.""" + + def __init__(self, num_classes): + super(AllConvNet, self).__init__() + + self.num_classes = num_classes + self.width1, w1 = 96, 96 + self.width2, w2 = 192, 192 + + self.features = make_layers( + [w1, w1, w1, 'Md', w2, w2, w2, 'Md', 'nopad', 'NIN', 'NIN', 'A']) + self.classifier = nn.Linear(self.width2, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) # He initialization + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + x = self.features(x) + x = x.view(x.size(0), -1) + x = self.classifier(x) + return x diff --git a/Robustness_Eval/augmix/augment_and_mix.py b/Robustness_Eval/augmix/augment_and_mix.py new file mode 100644 index 0000000..c9a1d91 --- /dev/null +++ b/Robustness_Eval/augmix/augment_and_mix.py @@ -0,0 +1,70 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Reference implementation of AugMix's data augmentation method in numpy.""" +import augmentations +import numpy as np +from PIL import Image + +# CIFAR-10 constants +MEAN = [0.4914, 0.4822, 0.4465] +STD = [0.2023, 0.1994, 0.2010] + + +def normalize(image): + """Normalize input image channel-wise to zero mean and unit variance.""" + image = image.transpose(2, 0, 1) # Switch to channel-first + mean, std = np.array(MEAN), np.array(STD) + image = (image - mean[:, None, None]) / std[:, None, None] + return image.transpose(1, 2, 0) + + +def apply_op(image, op, severity): + image = np.clip(image * 255., 0, 255).astype(np.uint8) + pil_img = Image.fromarray(image) # Convert to PIL.Image + pil_img = op(pil_img, severity) + return np.asarray(pil_img) / 255. + + +def augment_and_mix(image, severity=3, width=3, depth=-1, alpha=1.): + """Perform AugMix augmentations and compute mixture. + + Args: + image: Raw input image as float32 np.ndarray of shape (h, w, c) + severity: Severity of underlying augmentation operators (between 1 to 10). + width: Width of augmentation chain + depth: Depth of augmentation chain. -1 enables stochastic depth uniformly + from [1, 3] + alpha: Probability coefficient for Beta and Dirichlet distributions. + + Returns: + mixed: Augmented and mixed image. + """ + ws = np.float32( + np.random.dirichlet([alpha] * width)) + m = np.float32(np.random.beta(alpha, alpha)) + + mix = np.zeros_like(image) + for i in range(width): + image_aug = image.copy() + d = depth if depth > 0 else np.random.randint(1, 4) + for _ in range(d): + op = np.random.choice(augmentations.augmentations) + image_aug = apply_op(image_aug, op, severity) + # Preprocessing commutes since all coefficients are convex + mix += ws[i] * normalize(image_aug) + + mixed = (1 - m) * normalize(image) + m * mix + return mixed + diff --git a/Robustness_Eval/augmix/augmentations.py b/Robustness_Eval/augmix/augmentations.py new file mode 100644 index 0000000..f02ef43 --- /dev/null +++ b/Robustness_Eval/augmix/augmentations.py @@ -0,0 +1,152 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Base augmentations operators.""" + +import numpy as np +from PIL import Image, ImageOps, ImageEnhance + +# ImageNet code should change this value +IMAGE_SIZE = 32 + + +def int_parameter(level, maxval): + """Helper function to scale `val` between 0 and maxval . + + Args: + level: Level of the operation that will be between [0, `PARAMETER_MAX`]. + maxval: Maximum value that the operation can have. This will be scaled to + level/PARAMETER_MAX. + + Returns: + An int that results from scaling `maxval` according to `level`. + """ + return int(level * maxval / 10) + + +def float_parameter(level, maxval): + """Helper function to scale `val` between 0 and maxval. + + Args: + level: Level of the operation that will be between [0, `PARAMETER_MAX`]. + maxval: Maximum value that the operation can have. This will be scaled to + level/PARAMETER_MAX. + + Returns: + A float that results from scaling `maxval` according to `level`. + """ + return float(level) * maxval / 10. + + +def sample_level(n): + return np.random.uniform(low=0.1, high=n) + + +def autocontrast(pil_img, _): + return ImageOps.autocontrast(pil_img) + + +def equalize(pil_img, _): + return ImageOps.equalize(pil_img) + + +def posterize(pil_img, level): + level = int_parameter(sample_level(level), 4) + return ImageOps.posterize(pil_img, 4 - level) + + +def rotate(pil_img, level): + degrees = int_parameter(sample_level(level), 30) + if np.random.uniform() > 0.5: + degrees = -degrees + return pil_img.rotate(degrees, resample=Image.BILINEAR) + + +def solarize(pil_img, level): + level = int_parameter(sample_level(level), 256) + return ImageOps.solarize(pil_img, 256 - level) + + +def shear_x(pil_img, level): + level = float_parameter(sample_level(level), 0.3) + if np.random.uniform() > 0.5: + level = -level + return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), + Image.AFFINE, (1, level, 0, 0, 1, 0), + resample=Image.BILINEAR) + + +def shear_y(pil_img, level): + level = float_parameter(sample_level(level), 0.3) + if np.random.uniform() > 0.5: + level = -level + return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), + Image.AFFINE, (1, 0, 0, level, 1, 0), + resample=Image.BILINEAR) + + +def translate_x(pil_img, level): + level = int_parameter(sample_level(level), IMAGE_SIZE / 3) + if np.random.random() > 0.5: + level = -level + return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), + Image.AFFINE, (1, 0, level, 0, 1, 0), + resample=Image.BILINEAR) + + +def translate_y(pil_img, level): + level = int_parameter(sample_level(level), IMAGE_SIZE / 3) + if np.random.random() > 0.5: + level = -level + return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), + Image.AFFINE, (1, 0, 0, 0, 1, level), + resample=Image.BILINEAR) + + +# operation that overlaps with ImageNet-C's test set +def color(pil_img, level): + level = float_parameter(sample_level(level), 1.8) + 0.1 + return ImageEnhance.Color(pil_img).enhance(level) + + +# operation that overlaps with ImageNet-C's test set +def contrast(pil_img, level): + level = float_parameter(sample_level(level), 1.8) + 0.1 + return ImageEnhance.Contrast(pil_img).enhance(level) + + +# operation that overlaps with ImageNet-C's test set +def brightness(pil_img, level): + level = float_parameter(sample_level(level), 1.8) + 0.1 + return ImageEnhance.Brightness(pil_img).enhance(level) + + +# operation that overlaps with ImageNet-C's test set +def sharpness(pil_img, level): + level = float_parameter(sample_level(level), 1.8) + 0.1 + return ImageEnhance.Sharpness(pil_img).enhance(level) + + +augmentations_none = [] + + +augmentations = [ + autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y, + translate_x, translate_y +] + +augmentations_all = [ + autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y, + translate_x, translate_y, color, contrast, brightness, sharpness +] diff --git a/Robustness_Eval/augmix/cifar.py b/Robustness_Eval/augmix/cifar.py new file mode 100644 index 0000000..2fad7af --- /dev/null +++ b/Robustness_Eval/augmix/cifar.py @@ -0,0 +1,533 @@ + + +""" +The code is adapted from the google-research's repo: +https://github.com/google-research/augmix/blob/master/cifar.py +""" + + + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Main script to launch AugMix training on CIFAR-10/100. +Supports WideResNet, AllConv, ResNeXt models on CIFAR-10 and CIFAR-100 as well +as evaluation on CIFAR-10-C and CIFAR-100-C. +Example usage: + `python cifar.py` +""" +from __future__ import print_function + +import argparse +import os +import shutil +import time + +import augmentations +from models.cifar.allconv import AllConvNet +import numpy as np +from third_party.ResNeXt_DenseNet.models.densenet import densenet +from third_party.ResNeXt_DenseNet.models.resnext import resnext29 +from third_party.WideResNet_pytorch.wideresnet import WideResNet + +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F +from torchvision import datasets +from torchvision import transforms + +parser = argparse.ArgumentParser( + description='Trains a CIFAR Classifier', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument( + '--dataset', + type=str, + default='cifar10', + choices=['cifar10', 'cifar100'], + help='Choose between CIFAR-10, CIFAR-100.') +parser.add_argument( + '--model', + '-m', + type=str, + default='wrn', + choices=['wrn', 'allconv', 'densenet', 'resnext'], + help='Choose architecture.') +# Optimization options +parser.add_argument( + '--epochs', '-e', type=int, default=100, help='Number of epochs to train.') +parser.add_argument( + '--learning-rate', + '-lr', + type=float, + default=0.1, + help='Initial learning rate.') +parser.add_argument( + '--batch-size', '-b', type=int, default=128, help='Batch size.') +parser.add_argument('--eval-batch-size', type=int, default=1000) +parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.') +parser.add_argument( + '--decay', + '-wd', + type=float, + default=0.0005, + help='Weight decay (L2 penalty).') +# WRN Architecture options +parser.add_argument( + '--layers', default=40, type=int, help='total number of layers') +parser.add_argument('--widen-factor', default=2, type=int, help='Widen factor') +parser.add_argument( + '--droprate', default=0.0, type=float, help='Dropout probability') +# AugMix options +parser.add_argument( + '--mixture-width', + default=3, + type=int, + help='Number of augmentation chains to mix per augmented example') +parser.add_argument( + '--mixture-depth', + default=-1, + type=int, + help='Depth of augmentation chains. -1 denotes stochastic depth in [1, 3]') +parser.add_argument( + '--aug-severity', + default=3, + type=int, + help='Severity of base augmentation operators') +parser.add_argument( + '--no-jsd', + '-nj', + action='store_true', + help='Turn off JSD consistency loss.') +parser.add_argument( + '--all-ops', + '-all', + action='store_true', + help='Turn on all operations (+brightness,contrast,color,sharpness).') +# Checkpointing options +parser.add_argument( + '--save', + '-s', + type=str, + default='./snapshots', + help='Folder to save checkpoints.') +parser.add_argument( + '--resume', + '-r', + type=str, + default='', + help='Checkpoint path for resume / test.') +parser.add_argument('--evaluate', action='store_true', help='Eval only.') +parser.add_argument( + '--print-freq', + type=int, + default=50, + help='Training loss print frequency (batches).') +# Acceleration +parser.add_argument( + '--num-workers', + type=int, + default=4, + help='Number of pre-fetching threads.') + +######################################################### +parser.add_argument( + '--mix_off', + action='store_true', + help='Turn off augmix') +######################################################### + +args = parser.parse_args() + +CORRUPTIONS = [ + 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', + 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', + 'brightness', 'contrast', 'elastic_transform', 'pixelate', + 'jpeg_compression' +] + + +def get_lr(step, total_steps, lr_max, lr_min): + """Compute learning rate according to cosine annealing schedule.""" + return lr_min + (lr_max - lr_min) * 0.5 * (1 + + np.cos(step / total_steps * np.pi)) + +######################################################################################################################################################################################################################## +def aug(image, preprocess): + """Perform AugMix augmentations and compute mixture. + Args: + image: PIL.Image input image + preprocess: Preprocessing function which should return a torch tensor. + Returns: + mixed: Augmented and mixed image. + """ + aug_list = augmentations.augmentations + if args.all_ops: + aug_list = augmentations.augmentations_all + if args.mix_off: + mixed = preprocess(image) + return mixed + + ws = np.float32(np.random.dirichlet([1] * args.mixture_width)) + m = np.float32(np.random.beta(1, 1)) + + mix = torch.zeros_like(preprocess(image)) + for i in range(args.mixture_width): + image_aug = image.copy() + depth = args.mixture_depth if args.mixture_depth > 0 else np.random.randint( + 1, 4) + for _ in range(depth): + op = np.random.choice(aug_list) + image_aug = op(image_aug, args.aug_severity) + # Preprocessing commutes since all coefficients are convex + mix += ws[i] * preprocess(image_aug) + + mixed = (1 - m) * preprocess(image) + m * mix + return mixed + + +class AugMixDataset(torch.utils.data.Dataset): + """Dataset wrapper to perform AugMix augmentation.""" + + def __init__(self, dataset, preprocess, no_jsd=False): + self.dataset = dataset + self.preprocess = preprocess + self.no_jsd = no_jsd + + def __getitem__(self, i): + x, y = self.dataset[i] + if self.no_jsd: + return aug(x, self.preprocess), y + elif args.mix_off: + return aug(x, self.preprocess), y + else: + im_tuple = (self.preprocess(x), aug(x, self.preprocess), + aug(x, self.preprocess)) + return im_tuple, y + + def __len__(self): + return len(self.dataset) + +######################################################################################################################################################################################################################## + + + +def train(net, train_loader, optimizer, scheduler, location): + """Train for one epoch.""" + net.train() + loss_ema = 0. + for i, (images, targets) in enumerate(train_loader): + optimizer.zero_grad() + + if args.no_jsd: + images = images.cuda() + targets = targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + logits1 = logits.cpu().detach().numpy() + images1 = images.cpu().reshape((len(images.cpu()), np.prod(images.cpu().shape[1:]))) + np.savetxt("./{}/train_corrupted_input.csv".format(location),images1) + np.savetxt("./{}/train_corrupted_labels.csv".format(location),targets.cpu()) + np.savetxt("./{}/train_corrupted_output.csv".format(location),logits1) +#################changed here############### + elif args.mix_off: + images = images.cuda() + targets = targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + logits1 = logits.cpu().detach().numpy() + images1 = images.cpu().reshape((len(images.cpu()), np.prod(images.cpu().shape[1:]))) + np.savetxt("./{}/train_clean_input.csv".format(location),images1) + np.savetxt("./{}/train_clean_labels.csv".format(location),targets.cpu()) + np.savetxt("./{}/train_clean_output.csv".format(location),logits1) +############################################ + else: + images_all = torch.cat(images, 0).cuda() + targets = targets.cuda() + logits_all = net(images_all) + logits_all1 = logits_all.cpu().detach().numpy() + images1 = images_all.cpu().reshape((len(images_all.cpu()), np.prod(images_all.cpu().shape[1:]))) + np.savetxt("./{}/train_corrupted_input.csv".format(location),images1) + np.savetxt("./{}/train_corrupted_labels.csv".format(location),targets.cpu()) + np.savetxt("./{}/train_corrupted_output.csv".format(location),logits_all1) + logits_clean, logits_aug1, logits_aug2 = torch.split( + logits_all, images[0].size(0)) + + # Cross-entropy is only computed on clean images + loss = F.cross_entropy(logits_clean, targets) + + p_clean, p_aug1, p_aug2 = F.softmax( + logits_clean, dim=1), F.softmax( + logits_aug1, dim=1), F.softmax( + logits_aug2, dim=1) + + # Clamp mixture distribution to avoid exploding KL divergence + p_mixture = torch.clamp((p_clean + p_aug1 + p_aug2) / 3., 1e-7, 1).log() + loss += 12 * (F.kl_div(p_mixture, p_clean, reduction='batchmean') + + F.kl_div(p_mixture, p_aug1, reduction='batchmean') + + F.kl_div(p_mixture, p_aug2, reduction='batchmean')) / 3. + + loss.backward() + optimizer.step() + scheduler.step() + loss_ema = loss_ema * 0.9 + float(loss) * 0.1 + if i % args.print_freq == 0: + print('Train Loss {:.3f}'.format(loss_ema)) + + return loss_ema + + +def test(net, test_loader,location): + """Evaluate network on given dataset.""" + net.eval() + total_loss = 0. + total_correct = 0 + with torch.no_grad(): + for images, targets in test_loader: + images, targets = images.cuda(), targets.cuda() + logits = net(images) + images1 = images.cpu().reshape((len(images.cpu()), np.prod(images.cpu().shape[1:]))) + np.savetxt("./{}/test_clean_input.csv".format(location),images1) + np.savetxt("./{}/test_clean_labels.csv".format(location),targets.cpu()) + np.savetxt("./{}/test_clean_output.csv".format(location),logits.cpu()) + loss = F.cross_entropy(logits, targets) + pred = logits.data.max(1)[1] + total_loss += float(loss.data) + total_correct += pred.eq(targets.data).sum().item() + + + return total_loss / len(test_loader.dataset), total_correct / len( + test_loader.dataset) + + +################################################################################# +def my_test(net, test_loader,location, corruption): + """Evaluate network on corrupted dataset.""" + net.eval() + total_loss = 0. + total_correct = 0 + with torch.no_grad(): + for images, targets in test_loader: + images, targets = images.cuda(), targets.cuda() + logits = net(images) + images1 = images.cpu().reshape((len(images.cpu()), np.prod(images.cpu().shape[1:]))) + np.savetxt("./{}/{}_test_corrupted_input.csv".format(location,corruption),images1) + np.savetxt("./{}/{}_test_corrupted_labels.csv".format(location,corruption),targets.cpu()) + np.savetxt("./{}/{}_test_corrupted_output.csv".format(location,corruption),logits.cpu()) + loss = F.cross_entropy(logits, targets) + pred = logits.data.max(1)[1] + total_loss += float(loss.data) + total_correct += pred.eq(targets.data).sum().item() +################################################################################### + + return total_loss / len(test_loader.dataset), total_correct / len( + test_loader.dataset) + + + +def test_c(net, test_data, base_path): + """Evaluate network on given corrupted dataset.""" + corruption_accs = [] + for corruption in CORRUPTIONS: + # Reference to original data is mutated + test_data.data = np.load(base_path + corruption + '.npy') + test_data.targets = torch.LongTensor(np.load(base_path + 'labels.npy')) + + test_loader = torch.utils.data.DataLoader( + test_data, + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + test_loss, test_acc = my_test(net, test_loader, args.save, corruption) + + + + + + corruption_accs.append(test_acc) + print('{}\n\tTest Loss {:.3f} | Test Error {:.3f}'.format( + corruption, test_loss, 100 - 100. * test_acc)) + + return np.mean(corruption_accs) + + +def main(): + torch.manual_seed(1) + np.random.seed(1) + + # Load datasets + train_transform = transforms.Compose( + [transforms.RandomHorizontalFlip(), + transforms.RandomCrop(32, padding=4)]) + + ###################################related to augmixdataset################################ + preprocess = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize([0.5] * 3, [0.5] * 3)]) + ########################################################################################### + + + + test_transform = preprocess + + if args.dataset == 'cifar10': + train_data = datasets.CIFAR10( + './data/cifar', train=True, transform=train_transform, download=True) + test_data = datasets.CIFAR10( + './data/cifar', train=False, transform=test_transform, download=True) + base_c_path = './data/cifar/CIFAR-10-C/' + num_classes = 10 + else: + train_data = datasets.CIFAR100( + './data/cifar', train=True, transform=train_transform, download=True) + test_data = datasets.CIFAR100( + './data/cifar', train=False, transform=test_transform, download=True) + base_c_path = './data/cifar/CIFAR-100-C/' + num_classes = 100 + + + + + + + ###################################change here########################################## + train_data = AugMixDataset(train_data, preprocess, args.no_jsd) + ######################################################################################## + + + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=args.batch_size, + shuffle=True, + num_workers=args.num_workers, + pin_memory=True) + + test_loader = torch.utils.data.DataLoader( + test_data, + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + # Create model + if args.model == 'densenet': + net = densenet(num_classes=num_classes) + elif args.model == 'wrn': + net = WideResNet(args.layers, num_classes, args.widen_factor, args.droprate) + elif args.model == 'allconv': + net = AllConvNet(num_classes) + elif args.model == 'resnext': + net = resnext29(num_classes=num_classes) + + optimizer = torch.optim.SGD( + net.parameters(), + args.learning_rate, + momentum=args.momentum, + weight_decay=args.decay, + nesterov=True) + + # Distribute model across all visible GPUs + net = torch.nn.DataParallel(net).cuda() + cudnn.benchmark = True + + start_epoch = 0 + + if args.resume: + if os.path.isfile(args.resume): + checkpoint = torch.load(args.resume) + start_epoch = checkpoint['epoch'] + 1 + best_acc = checkpoint['best_acc'] + net.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print('Model restored from epoch:', start_epoch) + + if args.evaluate: + # Evaluate clean accuracy first because test_c mutates underlying data + test_loss, test_acc = test(net, test_loader, args.save) + print('Clean\n\tTest Loss {:.3f} | Test Error {:.2f}'.format( + test_loss, 100 - 100. * test_acc)) + + #test_c_acc = test_c(net, test_data, base_c_path) + #print('Mean Corruption Error: {:.3f}'.format(100 - 100. * test_c_acc)) + return + + scheduler = torch.optim.lr_scheduler.LambdaLR( + optimizer, + lr_lambda=lambda step: get_lr( # pylint: disable=g-long-lambda + step, + args.epochs * len(train_loader), + 1, # lr_lambda computes multiplicative factor + 1e-6 / args.learning_rate)) + + if not os.path.exists(args.save): + os.makedirs(args.save) + if not os.path.isdir(args.save): + raise Exception('%s is not a dir' % args.save) + + log_path = os.path.join(args.save, + args.dataset + '_' + args.model + '_training_log.csv') + with open(log_path, 'w') as f: + f.write('epoch,time(s),train_loss,test_loss,test_error(%)\n') + + best_acc = 0 + print('Beginning training from epoch:', start_epoch + 1) + for epoch in range(start_epoch, args.epochs): + begin_time = time.time() + + train_loss_ema = train(net, train_loader, optimizer, scheduler, args.save) + test_loss, test_acc = test(net, test_loader, args.save) + + is_best = test_acc > best_acc + best_acc = max(test_acc, best_acc) + checkpoint = { + 'epoch': epoch, + 'dataset': args.dataset, + 'model': args.model, + 'state_dict': net.state_dict(), + 'best_acc': best_acc, + 'optimizer': optimizer.state_dict(), + } + + save_path = os.path.join(args.save, 'checkpoint.pth.tar') + torch.save(checkpoint, save_path) + if is_best: + shutil.copyfile(save_path, os.path.join(args.save, 'model_best.pth.tar')) + + with open(log_path, 'a') as f: + f.write('%03d,%05d,%0.6f,%0.5f,%0.2f\n' % ( + (epoch + 1), + time.time() - begin_time, + train_loss_ema, + test_loss, + 100 - 100. * test_acc, + )) + + print( + 'Epoch {0:3d} | Time {1:5d} | Train Loss {2:.4f} | Test Loss {3:.3f} |' + ' Test Error {4:.2f}' + .format((epoch + 1), int(time.time() - begin_time), train_loss_ema, + test_loss, 100 - 100. * test_acc)) + + test_c_acc = test_c(net, test_data, base_c_path) + print('Mean Corruption Error: {:.3f}'.format(100 - 100. * test_c_acc)) + + with open(log_path, 'a') as f: + f.write('%03d,%05d,%0.6f,%0.5f,%0.2f\n' % + (args.epochs + 1, 0, 0, 0, 100 - 100 * test_c_acc)) + + +if __name__ == '__main__': + main() + diff --git a/Robustness_Eval/augmix/imagenet.py b/Robustness_Eval/augmix/imagenet.py new file mode 100644 index 0000000..d88431a --- /dev/null +++ b/Robustness_Eval/augmix/imagenet.py @@ -0,0 +1,493 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Main script to launch AugMix training on ImageNet. + +Currently only supports ResNet-50 training. + +Example usage: + `python imagenet.py ` +""" +from __future__ import print_function + +import argparse +import os +import shutil +import time + +import augmentations + +import numpy as np +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F +from torchvision import datasets +from torchvision import models +from torchvision import transforms + +augmentations.IMAGE_SIZE = 224 + +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith('__') and + callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='Trains an ImageNet Classifier') +parser.add_argument( + 'clean_data', metavar='DIR', help='path to clean ImageNet dataset') +parser.add_argument( + 'corrupted_data', metavar='DIR_C', help='path to ImageNet-C dataset') +parser.add_argument( + '--model', + '-m', + default='resnet50', + choices=model_names, + help='model architecture: ' + ' | '.join(model_names) + + ' (default: resnet50)') +# Optimization options +parser.add_argument( + '--epochs', '-e', type=int, default=90, help='Number of epochs to train.') +parser.add_argument( + '--learning-rate', + '-lr', + type=float, + default=0.1, + help='Initial learning rate.') +parser.add_argument( + '--batch-size', '-b', type=int, default=256, help='Batch size.') +parser.add_argument('--eval-batch-size', type=int, default=1000) +parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.') +parser.add_argument( + '--decay', + '-wd', + type=float, + default=0.0001, + help='Weight decay (L2 penalty).') +# AugMix options +parser.add_argument( + '--mixture-width', + default=3, + type=int, + help='Number of augmentation chains to mix per augmented example') +parser.add_argument( + '--mixture-depth', + default=-1, + type=int, + help='Depth of augmentation chains. -1 denotes stochastic depth in [1, 3]') +parser.add_argument( + '--aug-severity', + default=1, + type=int, + help='Severity of base augmentation operators') +parser.add_argument( + '--aug-prob-coeff', + default=1., + type=float, + help='Probability distribution coefficients') +parser.add_argument( + '--no-jsd', + '-nj', + action='store_true', + help='Turn off JSD consistency loss.') +parser.add_argument( + '--all-ops', + '-all', + action='store_true', + help='Turn on all operations (+brightness,contrast,color,sharpness).') +# Checkpointing options +parser.add_argument( + '--save', + '-s', + type=str, + default='./snapshots', + help='Folder to save checkpoints.') +parser.add_argument( + '--resume', + '-r', + type=str, + default='', + help='Checkpoint path for resume / test.') +parser.add_argument('--evaluate', action='store_true', help='Eval only.') +parser.add_argument( + '--print-freq', + type=int, + default=10, + help='Training loss print frequency (batches).') +parser.add_argument( + '--pretrained', + dest='pretrained', + action='store_true', + help='use pre-trained model') +# Acceleration +parser.add_argument( + '--num-workers', + type=int, + default=4, + help='Number of pre-fetching threads.') + +args = parser.parse_args() + +CORRUPTIONS = [ + 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', + 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', + 'brightness', 'contrast', 'elastic_transform', 'pixelate', + 'jpeg_compression' +] + +# Raw AlexNet errors taken from https://github.com/hendrycks/robustness +ALEXNET_ERR = [ + 0.886428, 0.894468, 0.922640, 0.819880, 0.826268, 0.785948, 0.798360, + 0.866816, 0.826572, 0.819324, 0.564592, 0.853204, 0.646056, 0.717840, + 0.606500 +] + + +def adjust_learning_rate(optimizer, epoch): + """Sets the learning rate to the initial LR (linearly scaled to batch size) decayed by 10 every n / 3 epochs.""" + b = args.batch_size / 256. + k = args.epochs // 3 + if epoch < k: + m = 1 + elif epoch < 2 * k: + m = 0.1 + else: + m = 0.01 + lr = args.learning_rate * m * b + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k.""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def compute_mce(corruption_accs): + """Compute mCE (mean Corruption Error) normalized by AlexNet performance.""" + mce = 0. + for i in range(len(CORRUPTIONS)): + avg_err = 1 - np.mean(corruption_accs[CORRUPTIONS[i]]) + ce = 100 * avg_err / ALEXNET_ERR[i] + mce += ce / 15 + return mce + + +def aug(image, preprocess): + """Perform AugMix augmentations and compute mixture. + + Args: + image: PIL.Image input image + preprocess: Preprocessing function which should return a torch tensor. + + Returns: + mixed: Augmented and mixed image. + """ + aug_list = augmentations.augmentations + if args.all_ops: + aug_list = augmentations.augmentations_all + + ws = np.float32( + np.random.dirichlet([args.aug_prob_coeff] * args.mixture_width)) + m = np.float32(np.random.beta(args.aug_prob_coeff, args.aug_prob_coeff)) + + mix = torch.zeros_like(preprocess(image)) + for i in range(args.mixture_width): + image_aug = image.copy() + depth = args.mixture_depth if args.mixture_depth > 0 else np.random.randint( + 1, 4) + for _ in range(depth): + op = np.random.choice(aug_list) + image_aug = op(image_aug, args.aug_severity) + # Preprocessing commutes since all coefficients are convex + mix += ws[i] * preprocess(image_aug) + + mixed = (1 - m) * preprocess(image) + m * mix + return mixed + + +class AugMixDataset(torch.utils.data.Dataset): + """Dataset wrapper to perform AugMix augmentation.""" + + def __init__(self, dataset, preprocess, no_jsd=False): + self.dataset = dataset + self.preprocess = preprocess + self.no_jsd = no_jsd + + def __getitem__(self, i): + x, y = self.dataset[i] + if self.no_jsd: + return aug(x, self.preprocess), y + else: + im_tuple = (self.preprocess(x), aug(x, self.preprocess), + aug(x, self.preprocess)) + return im_tuple, y + + def __len__(self): + return len(self.dataset) + + +def train(net, train_loader, optimizer): + """Train for one epoch.""" + net.train() + data_ema = 0. + batch_ema = 0. + loss_ema = 0. + acc1_ema = 0. + acc5_ema = 0. + + end = time.time() + for i, (images, targets) in enumerate(train_loader): + # Compute data loading time + data_time = time.time() - end + optimizer.zero_grad() + + if args.no_jsd: + images = images.cuda() + targets = targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + acc1, acc5 = accuracy(logits, targets, topk=(1, 5)) # pylint: disable=unbalanced-tuple-unpacking + else: + images_all = torch.cat(images, 0).cuda() + targets = targets.cuda() + logits_all = net(images_all) + logits_clean, logits_aug1, logits_aug2 = torch.split( + logits_all, images[0].size(0)) + + # Cross-entropy is only computed on clean images + loss = F.cross_entropy(logits_clean, targets) + + p_clean, p_aug1, p_aug2 = F.softmax( + logits_clean, dim=1), F.softmax( + logits_aug1, dim=1), F.softmax( + logits_aug2, dim=1) + + # Clamp mixture distribution to avoid exploding KL divergence + p_mixture = torch.clamp((p_clean + p_aug1 + p_aug2) / 3., 1e-7, 1).log() + loss += 12 * (F.kl_div(p_mixture, p_clean, reduction='batchmean') + + F.kl_div(p_mixture, p_aug1, reduction='batchmean') + + F.kl_div(p_mixture, p_aug2, reduction='batchmean')) / 3. + acc1, acc5 = accuracy(logits_clean, targets, topk=(1, 5)) # pylint: disable=unbalanced-tuple-unpacking + + loss.backward() + optimizer.step() + + # Compute batch computation time and update moving averages. + batch_time = time.time() - end + end = time.time() + + data_ema = data_ema * 0.1 + float(data_time) * 0.9 + batch_ema = batch_ema * 0.1 + float(batch_time) * 0.9 + loss_ema = loss_ema * 0.1 + float(loss) * 0.9 + acc1_ema = acc1_ema * 0.1 + float(acc1) * 0.9 + acc5_ema = acc5_ema * 0.1 + float(acc5) * 0.9 + + if i % args.print_freq == 0: + print( + 'Batch {}/{}: Data Time {:.3f} | Batch Time {:.3f} | Train Loss {:.3f} | Train Acc1 ' + '{:.3f} | Train Acc5 {:.3f}'.format(i, len(train_loader), data_ema, + batch_ema, loss_ema, acc1_ema, + acc5_ema)) + + return loss_ema, acc1_ema, batch_ema + + +def test(net, test_loader): + """Evaluate network on given dataset.""" + net.eval() + total_loss = 0. + total_correct = 0 + with torch.no_grad(): + for images, targets in test_loader: + images, targets = images.cuda(), targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + pred = logits.data.max(1)[1] + total_loss += float(loss.data) + total_correct += pred.eq(targets.data).sum().item() + + return total_loss / len(test_loader.dataset), total_correct / len( + test_loader.dataset) + + +def test_c(net, test_transform): + """Evaluate network on given corrupted dataset.""" + corruption_accs = {} + for c in CORRUPTIONS: + print(c) + for s in range(1, 6): + valdir = os.path.join(args.corrupted_data, c, str(s)) + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, test_transform), + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + loss, acc1 = test(net, val_loader) + if c in corruption_accs: + corruption_accs[c].append(acc1) + else: + corruption_accs[c] = [acc1] + + print('\ts={}: Test Loss {:.3f} | Test Acc1 {:.3f}'.format( + s, loss, 100. * acc1)) + + return corruption_accs + + +def main(): + torch.manual_seed(1) + np.random.seed(1) + + # Load datasets + mean = [0.485, 0.456, 0.406] + std = [0.229, 0.224, 0.225] + train_transform = transforms.Compose( + [transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip()]) + preprocess = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize(mean, std)]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + preprocess, + ]) + + traindir = os.path.join(args.clean_data, 'train') + valdir = os.path.join(args.clean_data, 'val') + train_dataset = datasets.ImageFolder(traindir, train_transform) + train_dataset = AugMixDataset(train_dataset, preprocess) + train_loader = torch.utils.data.DataLoader( + train_dataset, + batch_size=args.batch_size, + shuffle=True, + num_workers=args.num_workers) + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, test_transform), + batch_size=args.batch_size, + shuffle=False, + num_workers=args.num_workers) + + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.model)) + net = models.__dict__[args.model](pretrained=True) + else: + print("=> creating model '{}'".format(args.model)) + net = models.__dict__[args.model]() + + optimizer = torch.optim.SGD( + net.parameters(), + args.learning_rate, + momentum=args.momentum, + weight_decay=args.decay) + + # Distribute model across all visible GPUs + net = torch.nn.DataParallel(net).cuda() + cudnn.benchmark = True + + start_epoch = 0 + + if args.resume: + if os.path.isfile(args.resume): + checkpoint = torch.load(args.resume) + start_epoch = checkpoint['epoch'] + 1 + best_acc1 = checkpoint['best_acc1'] + net.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print('Model restored from epoch:', start_epoch) + + if args.evaluate: + test_loss, test_acc1 = test(net, val_loader) + print('Clean\n\tTest Loss {:.3f} | Test Acc1 {:.3f}'.format( + test_loss, 100 * test_acc1)) + + corruption_accs = test_c(net, test_transform) + for c in CORRUPTIONS: + print('\t'.join([c] + map(str, corruption_accs[c]))) + + print('mCE (normalized by AlexNet): ', compute_mce(corruption_accs)) + return + + if not os.path.exists(args.save): + os.makedirs(args.save) + if not os.path.isdir(args.save): + raise Exception('%s is not a dir' % args.save) + + log_path = os.path.join(args.save, + 'imagenet_{}_training_log.csv'.format(args.model)) + with open(log_path, 'w') as f: + f.write( + 'epoch,batch_time,train_loss,train_acc1(%),test_loss,test_acc1(%)\n') + + best_acc1 = 0 + print('Beginning training from epoch:', start_epoch + 1) + for epoch in range(start_epoch, args.epochs): + adjust_learning_rate(optimizer, epoch) + + train_loss_ema, train_acc1_ema, batch_ema = train(net, train_loader, + optimizer) + test_loss, test_acc1 = test(net, val_loader) + + is_best = test_acc1 > best_acc1 + best_acc1 = max(test_acc1, best_acc1) + checkpoint = { + 'epoch': epoch, + 'model': args.model, + 'state_dict': net.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + } + + save_path = os.path.join(args.save, 'checkpoint.pth.tar') + torch.save(checkpoint, save_path) + if is_best: + shutil.copyfile(save_path, os.path.join(args.save, 'model_best.pth.tar')) + + with open(log_path, 'a') as f: + f.write('%03d,%0.3f,%0.6f,%0.2f,%0.5f,%0.2f\n' % ( + (epoch + 1), + batch_ema, + train_loss_ema, + 100. * train_acc1_ema, + test_loss, + 100. * test_acc1, + )) + + print( + 'Epoch {:3d} | Train Loss {:.4f} | Test Loss {:.3f} | Test Acc1 ' + '{:.2f}' + .format((epoch + 1), train_loss_ema, test_loss, 100. * test_acc1)) + + corruption_accs = test_c(net, test_transform) + for c in CORRUPTIONS: + print('\t'.join(map(str, [c] + corruption_accs[c]))) + + print('mCE (normalized by AlexNet):', compute_mce(corruption_accs)) + + +if __name__ == '__main__': + main() diff --git a/Robustness_Eval/augmix/models/cifar/__pycache__/allconv.cpython-37.pyc b/Robustness_Eval/augmix/models/cifar/__pycache__/allconv.cpython-37.pyc new file mode 100644 index 0000000..0173016 Binary files /dev/null and b/Robustness_Eval/augmix/models/cifar/__pycache__/allconv.cpython-37.pyc differ diff --git a/Robustness_Eval/augmix/models/cifar/allconv.py b/Robustness_Eval/augmix/models/cifar/allconv.py new file mode 100644 index 0000000..89d8c7b --- /dev/null +++ b/Robustness_Eval/augmix/models/cifar/allconv.py @@ -0,0 +1,77 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""AllConv implementation (https://arxiv.org/abs/1412.6806).""" +import math +import torch +import torch.nn as nn + + +class GELU(nn.Module): + + def forward(self, x): + return torch.sigmoid(1.702 * x) * x + + +def make_layers(cfg): + """Create a single layer.""" + layers = [] + in_channels = 3 + for v in cfg: + if v == 'Md': + layers += [nn.MaxPool2d(kernel_size=2, stride=2), nn.Dropout(p=0.5)] + elif v == 'A': + layers += [nn.AvgPool2d(kernel_size=8)] + elif v == 'NIN': + conv2d = nn.Conv2d(in_channels, in_channels, kernel_size=1, padding=1) + layers += [conv2d, nn.BatchNorm2d(in_channels), GELU()] + elif v == 'nopad': + conv2d = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=0) + layers += [conv2d, nn.BatchNorm2d(in_channels), GELU()] + else: + conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) + layers += [conv2d, nn.BatchNorm2d(v), GELU()] + in_channels = v + return nn.Sequential(*layers) + + +class AllConvNet(nn.Module): + """AllConvNet main class.""" + + def __init__(self, num_classes): + super(AllConvNet, self).__init__() + + self.num_classes = num_classes + self.width1, w1 = 96, 96 + self.width2, w2 = 192, 192 + + self.features = make_layers( + [w1, w1, w1, 'Md', w2, w2, w2, 'Md', 'nopad', 'NIN', 'NIN', 'A']) + self.classifier = nn.Linear(self.width2, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) # He initialization + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + x = self.features(x) + x = x.view(x.size(0), -1) + x = self.classifier(x) + return x diff --git a/Robustness_Eval/augmix/original_cifar.py b/Robustness_Eval/augmix/original_cifar.py new file mode 100644 index 0000000..9b316de --- /dev/null +++ b/Robustness_Eval/augmix/original_cifar.py @@ -0,0 +1,436 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Main script to launch AugMix training on CIFAR-10/100. +Supports WideResNet, AllConv, ResNeXt models on CIFAR-10 and CIFAR-100 as well +as evaluation on CIFAR-10-C and CIFAR-100-C. +Example usage: + `python cifar.py` +""" +from __future__ import print_function + +import argparse +import os +import shutil +import time + +import augmentations +from models.cifar.allconv import AllConvNet +import numpy as np +from third_party.ResNeXt_DenseNet.models.densenet import densenet +from third_party.ResNeXt_DenseNet.models.resnext import resnext29 +from third_party.WideResNet_pytorch.wideresnet import WideResNet + +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F +from torchvision import datasets +from torchvision import transforms + +parser = argparse.ArgumentParser( + description='Trains a CIFAR Classifier', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument( + '--dataset', + type=str, + default='cifar10', + choices=['cifar10', 'cifar100'], + help='Choose between CIFAR-10, CIFAR-100.') +parser.add_argument( + '--model', + '-m', + type=str, + default='wrn', + choices=['wrn', 'allconv', 'densenet', 'resnext'], + help='Choose architecture.') +# Optimization options +parser.add_argument( + '--epochs', '-e', type=int, default=100, help='Number of epochs to train.') +parser.add_argument( + '--learning-rate', + '-lr', + type=float, + default=0.1, + help='Initial learning rate.') +parser.add_argument( + '--batch-size', '-b', type=int, default=128, help='Batch size.') +parser.add_argument('--eval-batch-size', type=int, default=1000) +parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.') +parser.add_argument( + '--decay', + '-wd', + type=float, + default=0.0005, + help='Weight decay (L2 penalty).') +# WRN Architecture options +parser.add_argument( + '--layers', default=40, type=int, help='total number of layers') +parser.add_argument('--widen-factor', default=2, type=int, help='Widen factor') +parser.add_argument( + '--droprate', default=0.0, type=float, help='Dropout probability') +# AugMix options +parser.add_argument( + '--mixture-width', + default=3, + type=int, + help='Number of augmentation chains to mix per augmented example') +parser.add_argument( + '--mixture-depth', + default=-1, + type=int, + help='Depth of augmentation chains. -1 denotes stochastic depth in [1, 3]') +parser.add_argument( + '--aug-severity', + default=3, + type=int, + help='Severity of base augmentation operators') +parser.add_argument( + '--no-jsd', + '-nj', + action='store_true', + help='Turn off JSD consistency loss.') +parser.add_argument( + '--all-ops', + '-all', + action='store_true', + help='Turn on all operations (+brightness,contrast,color,sharpness).') +# Checkpointing options +parser.add_argument( + '--save', + '-s', + type=str, + default='./snapshots', + help='Folder to save checkpoints.') +parser.add_argument( + '--resume', + '-r', + type=str, + default='', + help='Checkpoint path for resume / test.') +parser.add_argument('--evaluate', action='store_true', help='Eval only.') +parser.add_argument( + '--print-freq', + type=int, + default=50, + help='Training loss print frequency (batches).') +# Acceleration +parser.add_argument( + '--num-workers', + type=int, + default=4, + help='Number of pre-fetching threads.') + +args = parser.parse_args() + +CORRUPTIONS = [ + 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', + 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', + 'brightness', 'contrast', 'elastic_transform', 'pixelate', + 'jpeg_compression' +] + + +def get_lr(step, total_steps, lr_max, lr_min): + """Compute learning rate according to cosine annealing schedule.""" + return lr_min + (lr_max - lr_min) * 0.5 * (1 + + np.cos(step / total_steps * np.pi)) + + +def aug(image, preprocess): + """Perform AugMix augmentations and compute mixture. + Args: + image: PIL.Image input image + preprocess: Preprocessing function which should return a torch tensor. + Returns: + mixed: Augmented and mixed image. + """ + aug_list = augmentations.augmentations + if args.all_ops: + aug_list = augmentations.augmentations_all + + ws = np.float32(np.random.dirichlet([1] * args.mixture_width)) + m = np.float32(np.random.beta(1, 1)) + + mix = torch.zeros_like(preprocess(image)) + for i in range(args.mixture_width): + image_aug = image.copy() + depth = args.mixture_depth if args.mixture_depth > 0 else np.random.randint( + 1, 4) + for _ in range(depth): + op = np.random.choice(aug_list) + image_aug = op(image_aug, args.aug_severity) + # Preprocessing commutes since all coefficients are convex + mix += ws[i] * preprocess(image_aug) + + mixed = (1 - m) * preprocess(image) + m * mix + return mixed + + +class AugMixDataset(torch.utils.data.Dataset): + """Dataset wrapper to perform AugMix augmentation.""" + + def __init__(self, dataset, preprocess, no_jsd=False): + self.dataset = dataset + self.preprocess = preprocess + self.no_jsd = no_jsd + + def __getitem__(self, i): + x, y = self.dataset[i] + if self.no_jsd: + return aug(x, self.preprocess), y + else: + im_tuple = (self.preprocess(x), aug(x, self.preprocess), + aug(x, self.preprocess)) + return im_tuple, y + + def __len__(self): + return len(self.dataset) + + +def train(net, train_loader, optimizer, scheduler): + """Train for one epoch.""" + net.train() + loss_ema = 0. + for i, (images, targets) in enumerate(train_loader): + optimizer.zero_grad() + + if args.no_jsd: + images = images.cuda() + targets = targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + else: + images_all = torch.cat(images, 0).cuda() + targets = targets.cuda() + logits_all = net(images_all) + logits_clean, logits_aug1, logits_aug2 = torch.split( + logits_all, images[0].size(0)) + + # Cross-entropy is only computed on clean images + loss = F.cross_entropy(logits_clean, targets) + + p_clean, p_aug1, p_aug2 = F.softmax( + logits_clean, dim=1), F.softmax( + logits_aug1, dim=1), F.softmax( + logits_aug2, dim=1) + + # Clamp mixture distribution to avoid exploding KL divergence + p_mixture = torch.clamp((p_clean + p_aug1 + p_aug2) / 3., 1e-7, 1).log() + loss += 12 * (F.kl_div(p_mixture, p_clean, reduction='batchmean') + + F.kl_div(p_mixture, p_aug1, reduction='batchmean') + + F.kl_div(p_mixture, p_aug2, reduction='batchmean')) / 3. + + loss.backward() + optimizer.step() + scheduler.step() + loss_ema = loss_ema * 0.9 + float(loss) * 0.1 + if i % args.print_freq == 0: + print('Train Loss {:.3f}'.format(loss_ema)) + + return loss_ema + + +def test(net, test_loader): + """Evaluate network on given dataset.""" + net.eval() + total_loss = 0. + total_correct = 0 + with torch.no_grad(): + for images, targets in test_loader: + images, targets = images.cuda(), targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + pred = logits.data.max(1)[1] + total_loss += float(loss.data) + total_correct += pred.eq(targets.data).sum().item() + + return total_loss / len(test_loader.dataset), total_correct / len( + test_loader.dataset) + + +def test_c(net, test_data, base_path): + """Evaluate network on given corrupted dataset.""" + corruption_accs = [] + for corruption in CORRUPTIONS: + # Reference to original data is mutated + test_data.data = np.load(base_path + corruption + '.npy') + test_data.targets = torch.LongTensor(np.load(base_path + 'labels.npy')) + + test_loader = torch.utils.data.DataLoader( + test_data, + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + test_loss, test_acc = test(net, test_loader) + corruption_accs.append(test_acc) + print('{}\n\tTest Loss {:.3f} | Test Error {:.3f}'.format( + corruption, test_loss, 100 - 100. * test_acc)) + + return np.mean(corruption_accs) + + +def main(): + torch.manual_seed(1) + np.random.seed(1) + + # Load datasets + train_transform = transforms.Compose( + [transforms.RandomHorizontalFlip(), + transforms.RandomCrop(32, padding=4)]) + preprocess = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize([0.5] * 3, [0.5] * 3)]) + test_transform = preprocess + + if args.dataset == 'cifar10': + train_data = datasets.CIFAR10( + './data/cifar', train=True, transform=train_transform, download=True) + test_data = datasets.CIFAR10( + './data/cifar', train=False, transform=test_transform, download=True) + base_c_path = './data/cifar/CIFAR-10-C/' + num_classes = 10 + else: + train_data = datasets.CIFAR100( + './data/cifar', train=True, transform=train_transform, download=True) + test_data = datasets.CIFAR100( + './data/cifar', train=False, transform=test_transform, download=True) + base_c_path = './data/cifar/CIFAR-100-C/' + num_classes = 100 + + train_data = AugMixDataset(train_data, preprocess, args.no_jsd) + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=args.batch_size, + shuffle=True, + num_workers=args.num_workers, + pin_memory=True) + + test_loader = torch.utils.data.DataLoader( + test_data, + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + # Create model + if args.model == 'densenet': + net = densenet(num_classes=num_classes) + elif args.model == 'wrn': + net = WideResNet(args.layers, num_classes, args.widen_factor, args.droprate) + elif args.model == 'allconv': + net = AllConvNet(num_classes) + elif args.model == 'resnext': + net = resnext29(num_classes=num_classes) + + optimizer = torch.optim.SGD( + net.parameters(), + args.learning_rate, + momentum=args.momentum, + weight_decay=args.decay, + nesterov=True) + + # Distribute model across all visible GPUs + net = torch.nn.DataParallel(net).cuda() + cudnn.benchmark = True + + start_epoch = 0 + + if args.resume: + if os.path.isfile(args.resume): + checkpoint = torch.load(args.resume) + start_epoch = checkpoint['epoch'] + 1 + best_acc = checkpoint['best_acc'] + net.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print('Model restored from epoch:', start_epoch) + + if args.evaluate: + # Evaluate clean accuracy first because test_c mutates underlying data + test_loss, test_acc = test(net, test_loader) + print('Clean\n\tTest Loss {:.3f} | Test Error {:.2f}'.format( + test_loss, 100 - 100. * test_acc)) + + test_c_acc = test_c(net, test_data, base_c_path) + print('Mean Corruption Error: {:.3f}'.format(100 - 100. * test_c_acc)) + return + + scheduler = torch.optim.lr_scheduler.LambdaLR( + optimizer, + lr_lambda=lambda step: get_lr( # pylint: disable=g-long-lambda + step, + args.epochs * len(train_loader), + 1, # lr_lambda computes multiplicative factor + 1e-6 / args.learning_rate)) + + if not os.path.exists(args.save): + os.makedirs(args.save) + if not os.path.isdir(args.save): + raise Exception('%s is not a dir' % args.save) + + log_path = os.path.join(args.save, + args.dataset + '_' + args.model + '_training_log.csv') + with open(log_path, 'w') as f: + f.write('epoch,time(s),train_loss,test_loss,test_error(%)\n') + + best_acc = 0 + print('Beginning training from epoch:', start_epoch + 1) + for epoch in range(start_epoch, args.epochs): + begin_time = time.time() + + train_loss_ema = train(net, train_loader, optimizer, scheduler) + test_loss, test_acc = test(net, test_loader) + + is_best = test_acc > best_acc + best_acc = max(test_acc, best_acc) + checkpoint = { + 'epoch': epoch, + 'dataset': args.dataset, + 'model': args.model, + 'state_dict': net.state_dict(), + 'best_acc': best_acc, + 'optimizer': optimizer.state_dict(), + } + + save_path = os.path.join(args.save, 'checkpoint.pth.tar') + torch.save(checkpoint, save_path) + if is_best: + shutil.copyfile(save_path, os.path.join(args.save, 'model_best.pth.tar')) + + with open(log_path, 'a') as f: + f.write('%03d,%05d,%0.6f,%0.5f,%0.2f\n' % ( + (epoch + 1), + time.time() - begin_time, + train_loss_ema, + test_loss, + 100 - 100. * test_acc, + )) + + print( + 'Epoch {0:3d} | Time {1:5d} | Train Loss {2:.4f} | Test Loss {3:.3f} |' + ' Test Error {4:.2f}' + .format((epoch + 1), int(time.time() - begin_time), train_loss_ema, + test_loss, 100 - 100. * test_acc)) + + test_c_acc = test_c(net, test_data, base_c_path) + print('Mean Corruption Error: {:.3f}'.format(100 - 100. * test_c_acc)) + + with open(log_path, 'a') as f: + f.write('%03d,%05d,%0.6f,%0.5f,%0.2f\n' % + (args.epochs + 1, 0, 0, 0, 100 - 100 * test_c_acc)) + + +if __name__ == '__main__': + main() diff --git a/Robustness_Eval/augmix/requirements.txt b/Robustness_Eval/augmix/requirements.txt new file mode 100644 index 0000000..f36c578 --- /dev/null +++ b/Robustness_Eval/augmix/requirements.txt @@ -0,0 +1,4 @@ +numpy>=1.15.0 +Pillow>=6.1.0 +torch==1.2.0 +torchvision==0.2.2 diff --git a/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/LICENSE b/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/LICENSE new file mode 100644 index 0000000..d05cea3 --- /dev/null +++ b/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Xuanyi Dong + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/METADATA b/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/METADATA new file mode 100644 index 0000000..43dc477 --- /dev/null +++ b/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/METADATA @@ -0,0 +1,12 @@ +name: "ResNeXt-DenseNet" +description: "PyTorch implementations of ResNeXt and DenseNet." + +third_party { + url { + type: GIT + value: "https://github.com/D-X-Y/ResNeXt-DenseNet" + } + version: "0de9a8c8fd095b37eb60945f8dafefdbfe1cef6b" + last_upgrade_date { year: 2019 month: 12 day: 4 } + license_type: PERMISSIVE +} diff --git a/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/models/densenet.py b/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/models/densenet.py new file mode 100644 index 0000000..2c52a2b --- /dev/null +++ b/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/models/densenet.py @@ -0,0 +1,125 @@ +"""DenseNet implementation (https://arxiv.org/abs/1608.06993).""" +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Bottleneck(nn.Module): + """Bottleneck block for DenseNet.""" + + def __init__(self, n_channels, growth_rate): + super(Bottleneck, self).__init__() + inter_channels = 4 * growth_rate + self.bn1 = nn.BatchNorm2d(n_channels) + self.conv1 = nn.Conv2d( + n_channels, inter_channels, kernel_size=1, bias=False) + self.bn2 = nn.BatchNorm2d(inter_channels) + self.conv2 = nn.Conv2d( + inter_channels, growth_rate, kernel_size=3, padding=1, bias=False) + + def forward(self, x): + out = self.conv1(F.relu(self.bn1(x))) + out = self.conv2(F.relu(self.bn2(out))) + out = torch.cat((x, out), 1) + return out + + +class SingleLayer(nn.Module): + """Layer container for blocks.""" + + def __init__(self, n_channels, growth_rate): + super(SingleLayer, self).__init__() + self.bn1 = nn.BatchNorm2d(n_channels) + self.conv1 = nn.Conv2d( + n_channels, growth_rate, kernel_size=3, padding=1, bias=False) + + def forward(self, x): + out = self.conv1(F.relu(self.bn1(x))) + out = torch.cat((x, out), 1) + return out + + +class Transition(nn.Module): + """Transition block.""" + + def __init__(self, n_channels, n_out_channels): + super(Transition, self).__init__() + self.bn1 = nn.BatchNorm2d(n_channels) + self.conv1 = nn.Conv2d( + n_channels, n_out_channels, kernel_size=1, bias=False) + + def forward(self, x): + out = self.conv1(F.relu(self.bn1(x))) + out = F.avg_pool2d(out, 2) + return out + + +class DenseNet(nn.Module): + """DenseNet main class.""" + + def __init__(self, growth_rate, depth, reduction, n_classes, bottleneck): + super(DenseNet, self).__init__() + + if bottleneck: + n_dense_blocks = int((depth - 4) / 6) + else: + n_dense_blocks = int((depth - 4) / 3) + + n_channels = 2 * growth_rate + self.conv1 = nn.Conv2d(3, n_channels, kernel_size=3, padding=1, bias=False) + + self.dense1 = self._make_dense(n_channels, growth_rate, n_dense_blocks, + bottleneck) + n_channels += n_dense_blocks * growth_rate + n_out_channels = int(math.floor(n_channels * reduction)) + self.trans1 = Transition(n_channels, n_out_channels) + + n_channels = n_out_channels + self.dense2 = self._make_dense(n_channels, growth_rate, n_dense_blocks, + bottleneck) + n_channels += n_dense_blocks * growth_rate + n_out_channels = int(math.floor(n_channels * reduction)) + self.trans2 = Transition(n_channels, n_out_channels) + + n_channels = n_out_channels + self.dense3 = self._make_dense(n_channels, growth_rate, n_dense_blocks, + bottleneck) + n_channels += n_dense_blocks * growth_rate + + self.bn1 = nn.BatchNorm2d(n_channels) + self.fc = nn.Linear(n_channels, n_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def _make_dense(self, n_channels, growth_rate, n_dense_blocks, bottleneck): + layers = [] + for _ in range(int(n_dense_blocks)): + if bottleneck: + layers.append(Bottleneck(n_channels, growth_rate)) + else: + layers.append(SingleLayer(n_channels, growth_rate)) + n_channels += growth_rate + return nn.Sequential(*layers) + + def forward(self, x): + out = self.conv1(x) + out = self.trans1(self.dense1(out)) + out = self.trans2(self.dense2(out)) + out = self.dense3(out) + out = torch.squeeze(F.avg_pool2d(F.relu(self.bn1(out)), 8)) + out = self.fc(out) + return out + + +def densenet(growth_rate=12, depth=40, num_classes=10): + model = DenseNet(growth_rate, depth, 1., num_classes, False) + return model diff --git a/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/models/resnext.py b/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/models/resnext.py new file mode 100644 index 0000000..2863a7a --- /dev/null +++ b/Robustness_Eval/augmix/third_party/ResNeXt_DenseNet/models/resnext.py @@ -0,0 +1,144 @@ +"""ResNeXt implementation (https://arxiv.org/abs/1611.05431).""" +import math +import torch.nn as nn +from torch.nn import init +import torch.nn.functional as F + + +class ResNeXtBottleneck(nn.Module): + """ResNeXt Bottleneck Block type C (https://github.com/facebookresearch/ResNeXt/blob/master/models/resnext.lua).""" + expansion = 4 + + def __init__(self, + inplanes, + planes, + cardinality, + base_width, + stride=1, + downsample=None): + super(ResNeXtBottleneck, self).__init__() + + dim = int(math.floor(planes * (base_width / 64.0))) + + self.conv_reduce = nn.Conv2d( + inplanes, + dim * cardinality, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn_reduce = nn.BatchNorm2d(dim * cardinality) + + self.conv_conv = nn.Conv2d( + dim * cardinality, + dim * cardinality, + kernel_size=3, + stride=stride, + padding=1, + groups=cardinality, + bias=False) + self.bn = nn.BatchNorm2d(dim * cardinality) + + self.conv_expand = nn.Conv2d( + dim * cardinality, + planes * 4, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn_expand = nn.BatchNorm2d(planes * 4) + + self.downsample = downsample + + def forward(self, x): + residual = x + + bottleneck = self.conv_reduce(x) + bottleneck = F.relu(self.bn_reduce(bottleneck), inplace=True) + + bottleneck = self.conv_conv(bottleneck) + bottleneck = F.relu(self.bn(bottleneck), inplace=True) + + bottleneck = self.conv_expand(bottleneck) + bottleneck = self.bn_expand(bottleneck) + + if self.downsample is not None: + residual = self.downsample(x) + + return F.relu(residual + bottleneck, inplace=True) + + +class CifarResNeXt(nn.Module): + """ResNext optimized for the Cifar dataset, as specified in https://arxiv.org/pdf/1611.05431.pdf.""" + + def __init__(self, block, depth, cardinality, base_width, num_classes): + super(CifarResNeXt, self).__init__() + + # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + assert (depth - 2) % 9 == 0, 'depth should be one of 29, 38, 47, 56, 101' + layer_blocks = (depth - 2) // 9 + + self.cardinality = cardinality + self.base_width = base_width + self.num_classes = num_classes + + self.conv_1_3x3 = nn.Conv2d(3, 64, 3, 1, 1, bias=False) + self.bn_1 = nn.BatchNorm2d(64) + + self.inplanes = 64 + self.stage_1 = self._make_layer(block, 64, layer_blocks, 1) + self.stage_2 = self._make_layer(block, 128, layer_blocks, 2) + self.stage_3 = self._make_layer(block, 256, layer_blocks, 2) + self.avgpool = nn.AvgPool2d(8) + self.classifier = nn.Linear(256 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + init.kaiming_normal(m.weight) + m.bias.data.zero_() + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, self.cardinality, self.base_width, stride, + downsample)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block(self.inplanes, planes, self.cardinality, self.base_width)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv_1_3x3(x) + x = F.relu(self.bn_1(x), inplace=True) + x = self.stage_1(x) + x = self.stage_2(x) + x = self.stage_3(x) + x = self.avgpool(x) + x = x.view(x.size(0), -1) + return self.classifier(x) + + +def resnext29(num_classes=10, cardinality=4, base_width=32): + model = CifarResNeXt(ResNeXtBottleneck, 29, cardinality, base_width, + num_classes) + return model diff --git a/Robustness_Eval/mnist/LICENSE b/Robustness_Eval/mnist/LICENSE new file mode 100644 index 0000000..c1b92dd --- /dev/null +++ b/Robustness_Eval/mnist/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Aleksander Madry, Aleksandar Makelov, Ludwig Schmidt, Dimitris Tsipras, and Adrian Vladu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Robustness_Eval/mnist/config.json b/Robustness_Eval/mnist/config.json new file mode 100644 index 0000000..af5efb1 --- /dev/null +++ b/Robustness_Eval/mnist/config.json @@ -0,0 +1,25 @@ +{ + "_comment": "===== MODEL CONFIGURATION =====", + "model_dir": "models/a_very_robust_model", + + "_comment": "===== TRAINING CONFIGURATION =====", + "random_seed": 4557077, + "max_num_training_steps": 100000, + "num_output_steps": 100, + "num_summary_steps": 100, + "num_checkpoint_steps": 300, + "training_batch_size": 50, + + "_comment": "===== EVAL CONFIGURATION =====", + "num_eval_examples": 10000, + "eval_batch_size": 200, + "eval_on_cpu": true, + + "_comment": "=====ADVERSARIAL EXAMPLES CONFIGURATION=====", + "epsilon": 0.3, + "k": 40, + "a": 0.01, + "random_start": true, + "loss_func": "xent", + "store_adv_path": "attack.npy" +} diff --git a/Robustness_Eval/mnist/eval.py b/Robustness_Eval/mnist/eval.py new file mode 100644 index 0000000..20b8e5e --- /dev/null +++ b/Robustness_Eval/mnist/eval.py @@ -0,0 +1,158 @@ +""" +Infinite evaluation loop going through the checkpoints in the model directory +as they appear and evaluating them. Accuracy and average loss are printed and +added as tensorboard summaries. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import json +import math +import os +import sys +import time + +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data + +from model import Model +from pgd_attack import LinfPGDAttack + +# Global constants +with open('config.json') as config_file: + config = json.load(config_file) +num_eval_examples = config['num_eval_examples'] +eval_batch_size = config['eval_batch_size'] +eval_on_cpu = config['eval_on_cpu'] + +model_dir = config['model_dir'] + +# Set upd the data, hyperparameters, and the model +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) + +if eval_on_cpu: + with tf.device("/cpu:0"): + model = Model() + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) +else: + model = Model() + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + +global_step = tf.contrib.framework.get_or_create_global_step() + +# Setting up the Tensorboard and checkpoint outputs +if not os.path.exists(model_dir): + os.makedirs(model_dir) +eval_dir = os.path.join(model_dir, 'eval') +if not os.path.exists(eval_dir): + os.makedirs(eval_dir) + +last_checkpoint_filename = '' +already_seen_state = False + +saver = tf.train.Saver() +summary_writer = tf.summary.FileWriter(eval_dir) + +# A function for evaluating a single checkpoint +def evaluate_checkpoint(filename): + with tf.Session() as sess: + # Restore the checkpoint + saver.restore(sess, filename) + + # Iterate over the samples batch-by-batch + num_batches = int(math.ceil(num_eval_examples / eval_batch_size)) + total_xent_nat = 0. + total_xent_adv = 0. + total_corr_nat = 0 + total_corr_adv = 0 + + for ibatch in range(num_batches): + bstart = ibatch * eval_batch_size + bend = min(bstart + eval_batch_size, num_eval_examples) + + x_batch = mnist.test.images[bstart:bend, :] + y_batch = mnist.test.labels[bstart:bend] + + dict_nat = {model.x_input: x_batch, + model.y_input: y_batch} + + x_batch_adv = attack.perturb(x_batch, y_batch, sess) + + dict_adv = {model.x_input: x_batch_adv, + model.y_input: y_batch} + + cur_corr_nat, cur_xent_nat = sess.run( + [model.num_correct,model.xent], + feed_dict = dict_nat) + cur_corr_adv, cur_xent_adv = sess.run( + [model.num_correct,model.xent], + feed_dict = dict_adv) + + total_xent_nat += cur_xent_nat + total_xent_adv += cur_xent_adv + total_corr_nat += cur_corr_nat + total_corr_adv += cur_corr_adv + + avg_xent_nat = total_xent_nat / num_eval_examples + avg_xent_adv = total_xent_adv / num_eval_examples + acc_nat = total_corr_nat / num_eval_examples + acc_adv = total_corr_adv / num_eval_examples + + summary = tf.Summary(value=[ + tf.Summary.Value(tag='xent adv eval', simple_value= avg_xent_adv), + tf.Summary.Value(tag='xent adv', simple_value= avg_xent_adv), + tf.Summary.Value(tag='xent nat', simple_value= avg_xent_nat), + tf.Summary.Value(tag='accuracy adv eval', simple_value= acc_adv), + tf.Summary.Value(tag='accuracy adv', simple_value= acc_adv), + tf.Summary.Value(tag='accuracy nat', simple_value= acc_nat)]) + summary_writer.add_summary(summary, global_step.eval(sess)) + + print('natural: {:.2f}%'.format(100 * acc_nat)) + print('adversarial: {:.2f}%'.format(100 * acc_adv)) + print('avg nat loss: {:.4f}'.format(avg_xent_nat)) + print('avg adv loss: {:.4f}'.format(avg_xent_adv)) + +# Infinite eval loop +while True: + cur_checkpoint = tf.train.latest_checkpoint(model_dir) + + # Case 1: No checkpoint yet + if cur_checkpoint is None: + if not already_seen_state: + print('No checkpoint yet, waiting ...', end='') + already_seen_state = True + else: + print('.', end='') + sys.stdout.flush() + time.sleep(10) + # Case 2: Previously unseen checkpoint + elif cur_checkpoint != last_checkpoint_filename: + print('\nCheckpoint {}, evaluating ... ({})'.format(cur_checkpoint, + datetime.now())) + sys.stdout.flush() + last_checkpoint_filename = cur_checkpoint + already_seen_state = False + evaluate_checkpoint(cur_checkpoint) + # Case 3: Previously evaluated checkpoint + else: + if not already_seen_state: + print('Waiting for the next checkpoint ... ({}) '.format( + datetime.now()), + end='') + already_seen_state = True + else: + print('.', end='') + sys.stdout.flush() + time.sleep(10) diff --git a/Robustness_Eval/mnist/fetch_model.py b/Robustness_Eval/mnist/fetch_model.py new file mode 100644 index 0000000..ab08e38 --- /dev/null +++ b/Robustness_Eval/mnist/fetch_model.py @@ -0,0 +1,46 @@ +"""Downloads a model, computes its SHA256 hash and unzips it + at the proper location.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import zipfile +import hashlib + +if len(sys.argv) != 2 or sys.argv[1] not in ['natural', + 'adv_trained', + 'secret']: + print('Usage: python fetch_model.py [natural, adv_trained, secret]') + sys.exit(1) + +if sys.argv[1] == 'natural': + url = 'https://github.com/MadryLab/mnist_challenge_models/raw/master/natural.zip' +elif sys.argv[1] == 'secret': + url = 'https://github.com/MadryLab/mnist_challenge_models/raw/master/secret.zip' +else: # fetch adv_trained model + url = 'https://github.com/MadryLab/mnist_challenge_models/raw/master/adv_trained.zip' + +fname = url.split('/')[-1] # get the name of the file + +# model download +print('Downloading models') +if sys.version_info >= (3,): + import urllib.request + urllib.request.urlretrieve(url, fname) +else: + import urllib + urllib.urlretrieve(url, fname) + +# computing model hash +sha256 = hashlib.sha256() +with open(fname, 'rb') as f: + data = f.read() + sha256.update(data) +print('SHA256 hash: {}'.format(sha256.hexdigest())) + +# extracting model +print('Extracting model') +with zipfile.ZipFile(fname, 'r') as model_zip: + model_zip.extractall() + print('Extracted model in {}'.format(model_zip.namelist()[0])) diff --git a/Robustness_Eval/mnist/gbt_eval.py b/Robustness_Eval/mnist/gbt_eval.py new file mode 100644 index 0000000..fd82e36 --- /dev/null +++ b/Robustness_Eval/mnist/gbt_eval.py @@ -0,0 +1,161 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import json +import math +import os +import sys +import time + +import numpy as np +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data +from pgd_attack import LinfPGDAttack + +from tensorflow.python import pywrap_tensorflow + +from model import Model +# from pgd_attack import LinfPGDAttack + +print("acc"+"b") + +with open('config.json') as config_file: + config = json.load(config_file) +num_eval_examples = config['num_eval_examples'] +eval_batch_size = config['eval_batch_size'] +eval_on_cpu = config['eval_on_cpu'] + +model = Model() +eps = config['epsilon'] +model_dir = config['model_dir'] + +# ============= set output file path ============== +path_folder = "./mnist_data_"+str(eps)+"_dl/"; + +saver = tf.train.Saver() + +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) +use_set = mnist.train + +if eval_on_cpu: + with tf.device("/cpu:0"): + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) +else: + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + + +global_step = tf.contrib.framework.get_or_create_global_step() + + + +with tf.Session() as sess: + print(sess.list_devices()) + + # model_dir = 'models/mnist-'+str(eps) + model_dir = config['model_dir'] + + # for eps in np.arange(0.4, 0.6, 0.1): + cur_checkpoint = tf.train.latest_checkpoint(model_dir) + # cur_checkpoint = 'models/mnist-0.3-dl/checkpoint-99900' + print('restoring '+ cur_checkpoint) + saver = tf.train.Saver() + saver.restore(sess, cur_checkpoint) + + + num_batches = int(math.ceil(use_set.labels.shape[0] / eval_batch_size)) + if not os.path.exists(path_folder): + os.mkdir(path_folder) + + batch_index = 0 + op_data = np.zeros((1,10)) + for ibatch in range(num_batches): + bstart = ibatch * eval_batch_size + bend = min(bstart + eval_batch_size, use_set.labels.shape[0]) + + x_batch = use_set.images[bstart:bend, :] + y_batch = use_set.labels[bstart:bend] + + dict_nat = {model.x_input: x_batch, + model.y_input: y_batch} + # x_batch_adv = attack.perturb(x_batch, y_batch, sess) + # dict_adv = {model.x_input: x_batch_adv, + # model.y_input: y_batch} + + # h_conv1, h_conv2, h_fc1, pred_softmax = sess.run([model.h_conv1, model.h_conv2, model.h_fc1, model.pre_softmax], + # feed_dict = dict_nat) + h_fc1, y_xent, xent, pred_softmax = sess.run([model.h_fc1, model.y_xent, model.xent, model.pre_softmax], + feed_dict = dict_nat) + + # shp_conv1 = h_conv1.shape + # shp_conv2 = h_conv2.shape + shp_fc1 = h_fc1.shape + + print(y_xent.shape) + print(xent.shape) + + op_data = np.vstack((op_data, pred_softmax)) + + # pred_softmax = sess.run(model.pre_softmax, + # feed_dict = dict_nat) + + # print('conv1') + # print(np.reshape(h_conv1, (eval_batch_size, shp_conv1[1]*shp_conv1[2]*shp_conv1[3])).shape) + + + # fp_out = open("I:/process/mltest/mnist_data_0.3/gbt_output_conv1.csv","ab"); + # np.savetxt(fp_out, np.reshape(h_conv1, (eval_batch_size, shp_conv1[1]*shp_conv1[2]*shp_conv1[3]))) + # fp_out.close() + + # fp_out = open("I:/process/mltest/mnist_data_0.3/gbt_output_conv2.csv","ab"); + # np.savetxt(fp_out, np.reshape(h_conv2, (eval_batch_size, shp_conv2[1]*shp_conv2[2]*shp_conv2[3]))) + # fp_out.close() + + # fp_out = open(path_folder+"/gbt_output_fc1.csv","ab"); + # np.savetxt(fp_out, h_fc1, fmt="%.3f"); + # fp_out.close(); + + fp_out = open(path_folder+"gbt_output_out.csv","ab"); + np.savetxt(fp_out, pred_softmax, fmt="%.4f") + fp_out.close() + + fp_in = open(path_folder+"gbt_input.csv", "ab") + np.savetxt(fp_in, x_batch, fmt="%.4f") + fp_in.close() + + fp_label = open(path_folder+"gbt_label.csv", "ab") + np.savetxt(fp_label, y_batch, fmt="%d") + fp_label.close() + + # break; + # batch_index=batch_index+1 + # print(batch_index) + # if batch_index>=2: + # break + print(ibatch) + # op_data = op_data[1:] + # print(np.sum(np.argmax(op_data, axis=1) == mnist.test.labels)/mnist.test.labels.shape[0]) + # print(mnist.test.images[0]) + + + # =========for debug=========== + # ckpt = tf.train.get_checkpoint_state('models/mnist-0') + # print(ckpt.model_checkpoint_path) + # 'models/mnist-0\checkpint-19800' + # reader = pywrap_tensorflow.NewCheckpointReader(cur_checkpoint) + # var_to_shape_map = reader.get_variable_to_shape_map() + # for key in var_to_shape_map: + # print(key) + + diff --git a/Robustness_Eval/mnist/model.py b/Robustness_Eval/mnist/model.py new file mode 100644 index 0000000..65ed350 --- /dev/null +++ b/Robustness_Eval/mnist/model.py @@ -0,0 +1,76 @@ +""" +The model is adapted from the tensorflow tutorial: +https://www.tensorflow.org/get_started/mnist/pros +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +class Model(object): + def __init__(self): + self.x_input = tf.placeholder(tf.float32, shape = [None, 784]) + self.y_input = tf.placeholder(tf.int64, shape = [None]) + + self.x_image = tf.reshape(self.x_input, [-1, 28, 28, 1]) + + # first convolutional layer + W_conv1 = self._weight_variable([5,5,1,32]) + b_conv1 = self._bias_variable([32]) + + h_conv1 = tf.nn.relu(self._conv2d(self.x_image, W_conv1) + b_conv1) + h_pool1 = self._max_pool_2x2(h_conv1) + + # second convolutional layer + W_conv2 = self._weight_variable([5,5,32,64]) + b_conv2 = self._bias_variable([64]) + + h_conv2 = tf.nn.relu(self._conv2d(h_pool1, W_conv2) + b_conv2) + h_pool2 = self._max_pool_2x2(h_conv2) + + # first fully connected layer + W_fc1 = self._weight_variable([7 * 7 * 64, 1024]) + b_fc1 = self._bias_variable([1024]) + + h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) + h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) + + # output layer + W_fc2 = self._weight_variable([1024,10]) + b_fc2 = self._bias_variable([10]) + + self.pre_softmax = tf.matmul(h_fc1, W_fc2) + b_fc2 + + y_xent = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=self.y_input, logits=self.pre_softmax) + + self.xent = tf.reduce_sum(y_xent) + + self.y_pred = tf.argmax(self.pre_softmax, 1) + + correct_prediction = tf.equal(self.y_pred, self.y_input) + + self.num_correct = tf.reduce_sum(tf.cast(correct_prediction, tf.int64)) + self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + + @staticmethod + def _weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + + @staticmethod + def _bias_variable(shape): + initial = tf.constant(0.1, shape = shape) + return tf.Variable(initial) + + @staticmethod + def _conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME') + + @staticmethod + def _max_pool_2x2( x): + return tf.nn.max_pool(x, + ksize = [1,2,2,1], + strides=[1,2,2,1], + padding='SAME') diff --git a/Robustness_Eval/mnist/model_converter.py b/Robustness_Eval/mnist/model_converter.py new file mode 100644 index 0000000..8bb3603 --- /dev/null +++ b/Robustness_Eval/mnist/model_converter.py @@ -0,0 +1,99 @@ +import tensorflow as tf +from tensorflow import keras +import numpy as np +import os +import sys +from model import Model + +from tensorflow.contrib.keras.api.keras.models import Sequential +from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout, Activation, Flatten, Lambda +from tensorflow.contrib.keras.api.keras.layers import Conv2D, MaxPooling2D +from tensorflow.contrib.keras.api.keras.models import load_model +from tensorflow.contrib.keras.api.keras.optimizers import SGD, Adam + +from tensorflow.python import pywrap_tensorflow + +# read checkpoint +with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + +# set path of model in and out + model_name = 'mnist-0.3-selftrained' + saver = tf.train.import_meta_graph('models/mnist-0.3/checkpoint-19800.meta') + saver.restore(sess, 'models/mnist-0.3/checkpoint-19800') + + vars_global = tf.global_variables() + model_vars = {} + var_shapes = [] + var_names = [] + + for var in vars_global: + # try: + model_vars[var.name] = var.eval() + var_shapes.append((model_vars[var.name].shape)) + var_names.append(var.name) + # except: + # print("For var={}, an exception occurred".format(var.name)) + +labels = tf.placeholder(tf.int64, shape=[None], name='labels') +logits = tf.placeholder(tf.float32, shape=[None, 10], name='logits') + +# create keras model according to the model structure +model = Sequential() +conv1 = Conv2D(32, (5,5), input_shape=(28,28,1), activation='relu', padding='SAME', +use_bias=True +) +model.add(conv1) +model.add(MaxPooling2D(pool_size=(2,2), + padding='SAME')) +conv2 = Conv2D(64, (5,5), activation='relu', padding='SAME', +use_bias=True +) +model.add(conv2) +model.add(MaxPooling2D(pool_size=(2,2), + padding='SAME')) +model.add(Flatten()) + +fc1 = Dense(1024, activation='relu', use_bias=True) +model.add(fc1) +fc2 = Dense(10, activation='softmax', use_bias=True) +model.add(fc2) + +def fn(correct, predicted): + return tf.nn.softmax_cross_entropy_with_logits(labels=correct, + logits=predicted) +sgd = keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) +model.compile(optimizer=sgd, loss=fn, metrics=['accuracy']) + +# match layers of two models +model.layers[0].set_weights([model_vars['Variable:0'], model_vars['Variable_1:0']]) +model.layers[2].set_weights([model_vars['Variable_2:0'], model_vars['Variable_3:0']]) +model.layers[5].set_weights([model_vars['Variable_4:0'], model_vars['Variable_5:0']]) +model.layers[6].set_weights([model_vars['Variable_6:0'], model_vars['Variable_7:0']]) + + +model.save('models/'+model_name+'.h5', overwrite=True) + +# validate the accuracy of models + +dataset = keras.datasets.mnist.load_data() + +# mnist +test_data = dataset[1][0] +test_labels = dataset[1][1] + +extra_bias = 0 + +batch_size = 100 +op_data = np.zeros((1,10)) +for i in range(int(test_data.shape[0]/batch_size)): + # x = test_data[i*batch_size:(i+1)*batch_size] + x = test_data[i*batch_size:(i+1)*batch_size]/255 - extra_bias + x = np.reshape(x, (batch_size, test_data.shape[1], test_data.shape[2], 1)) + # op = sess.run(model.output, feed_dict = {model.input: x}) + op = model.predict(x, batch_size=batch_size) + op_data = np.vstack((op_data, op)) +op_data = op_data[1:] +print('accuracy is:') +print(np.sum(np.argmax(op_data, axis=1) == test_labels)/op_data.shape[0]) + diff --git a/Robustness_Eval/mnist/model_robustml.py b/Robustness_Eval/mnist/model_robustml.py new file mode 100644 index 0000000..3b3792c --- /dev/null +++ b/Robustness_Eval/mnist/model_robustml.py @@ -0,0 +1,45 @@ +import robustml +import tensorflow as tf + +import model + +class Model(robustml.model.Model): + def __init__(self, sess): + self._model = model.Model() + + saver = tf.train.Saver() + checkpoint = tf.train.latest_checkpoint('models/secret') + saver.restore(sess, checkpoint) + + self._sess = sess + self._input = self._model.x_input + self._logits = self._model.pre_softmax + self._predictions = self._model.y_pred + self._dataset = robustml.dataset.MNIST() + self._threat_model = robustml.threat_model.Linf(epsilon=0.3) + + @property + def dataset(self): + return self._dataset + + @property + def threat_model(self): + return self._threat_model + + def classify(self, x): + return self._sess.run(self._predictions, + {self._input: x})[0] + + # expose attack interface + + @property + def input(self): + return self._input + + @property + def logits(self): + return self._logits + + @property + def predictions(self): + return self._predictions diff --git a/Robustness_Eval/mnist/pgd_attack.py b/Robustness_Eval/mnist/pgd_attack.py new file mode 100644 index 0000000..88186db --- /dev/null +++ b/Robustness_Eval/mnist/pgd_attack.py @@ -0,0 +1,121 @@ +""" +Implementation of attack methods. Running this file as a program will +apply the attack to the model specified by the config file and store +the examples in an .npy file. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +import numpy as np + + +class LinfPGDAttack: + def __init__(self, model, epsilon, k, a, random_start, loss_func): + """Attack parameter initialization. The attack performs k steps of + size a, while always staying within epsilon from the initial + point.""" + self.model = model + self.epsilon = epsilon + self.k = k + self.a = a + self.rand = random_start + + if loss_func == 'xent': + loss = model.xent + elif loss_func == 'cw': + label_mask = tf.one_hot(model.y_input, + 10, + on_value=1.0, + off_value=0.0, + dtype=tf.float32) + correct_logit = tf.reduce_sum(label_mask * model.pre_softmax, axis=1) + wrong_logit = tf.reduce_max((1-label_mask) * model.pre_softmax + - 1e4*label_mask, axis=1) + loss = -tf.nn.relu(correct_logit - wrong_logit + 50) + else: + print('Unknown loss function. Defaulting to cross-entropy') + loss = model.xent + + self.grad = tf.gradients(loss, model.x_input)[0] + + def perturb(self, x_nat, y, sess): + """Given a set of examples (x_nat, y), returns a set of adversarial + examples within epsilon of x_nat in l_infinity norm.""" + if self.rand: + x = x_nat + np.random.uniform(-self.epsilon, self.epsilon, x_nat.shape) + x = np.clip(x, 0, 1) # ensure valid pixel range + else: + x = np.copy(x_nat) + + for i in range(self.k): + grad = sess.run(self.grad, feed_dict={self.model.x_input: x, + self.model.y_input: y}) + + x += self.a * np.sign(grad) + + x = np.clip(x, x_nat - self.epsilon, x_nat + self.epsilon) + x = np.clip(x, 0, 1) # ensure valid pixel range + + return x + + +if __name__ == '__main__': + import json + import sys + import math + + from tensorflow.examples.tutorials.mnist import input_data + + from model import Model + + with open('config.json') as config_file: + config = json.load(config_file) + + model_file = tf.train.latest_checkpoint(config['model_dir']) + if model_file is None: + print('No model found') + sys.exit() + + model = Model() + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + saver = tf.train.Saver() + + mnist = input_data.read_data_sets('MNIST_data', one_hot=False) + + with tf.Session() as sess: + # Restore the checkpoint + saver.restore(sess, model_file) + + # Iterate over the samples batch-by-batch + num_eval_examples = config['num_eval_examples'] + eval_batch_size = config['eval_batch_size'] + num_batches = int(math.ceil(num_eval_examples / eval_batch_size)) + + x_adv = [] # adv accumulator + + print('Iterating over {} batches'.format(num_batches)) + + for ibatch in range(num_batches): + bstart = ibatch * eval_batch_size + bend = min(bstart + eval_batch_size, num_eval_examples) + print('batch size: {}'.format(bend - bstart)) + + x_batch = mnist.test.images[bstart:bend, :] + y_batch = mnist.test.labels[bstart:bend] + + x_batch_adv = attack.perturb(x_batch, y_batch, sess) + + x_adv.append(x_batch_adv) + + print('Storing examples') + path = config['store_adv_path'] + x_adv = np.concatenate(x_adv, axis=0) + np.save(path, x_adv) + print('Examples stored in {}'.format(path)) diff --git a/Robustness_Eval/mnist/run_attack.py b/Robustness_Eval/mnist/run_attack.py new file mode 100644 index 0000000..2a0f680 --- /dev/null +++ b/Robustness_Eval/mnist/run_attack.py @@ -0,0 +1,93 @@ +"""Evaluates a model against examples from a .npy file as specified + in config.json""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import json +import math +import os +import sys +import time + +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data + +import numpy as np + +from model import Model + +def run_attack(checkpoint, x_adv, epsilon): + mnist = input_data.read_data_sets('MNIST_data', one_hot=False) + + model = Model() + + saver = tf.train.Saver() + + num_eval_examples = 10000 + eval_batch_size = 64 + + num_batches = int(math.ceil(num_eval_examples / eval_batch_size)) + total_corr = 0 + + x_nat = mnist.test.images + l_inf = np.amax(np.abs(x_nat - x_adv)) + + if l_inf > epsilon + 0.0001: + print('maximum perturbation found: {}'.format(l_inf)) + print('maximum perturbation allowed: {}'.format(epsilon)) + return + + y_pred = [] # label accumulator + + with tf.Session() as sess: + # Restore the checkpoint + saver.restore(sess, checkpoint) + + # Iterate over the samples batch-by-batch + for ibatch in range(num_batches): + bstart = ibatch * eval_batch_size + bend = min(bstart + eval_batch_size, num_eval_examples) + + x_batch = x_adv[bstart:bend, :] + y_batch = mnist.test.labels[bstart:bend] + + dict_adv = {model.x_input: x_batch, + model.y_input: y_batch} + cur_corr, y_pred_batch = sess.run([model.num_correct, model.y_pred], + feed_dict=dict_adv) + + total_corr += cur_corr + y_pred.append(y_pred_batch) + + accuracy = total_corr / num_eval_examples + + print('Accuracy: {:.2f}%'.format(100.0 * accuracy)) + y_pred = np.concatenate(y_pred, axis=0) + np.save('pred.npy', y_pred) + print('Output saved at pred.npy') + +if __name__ == '__main__': + import json + + with open('config.json') as config_file: + config = json.load(config_file) + + model_dir = config['model_dir'] + + checkpoint = tf.train.latest_checkpoint(model_dir) + x_adv = np.load(config['store_adv_path']) + + if checkpoint is None: + print('No checkpoint found') + elif x_adv.shape != (10000, 784): + print('Invalid shape: expected (10000,784), found {}'.format(x_adv.shape)) + elif np.amax(x_adv) > 1.0001 or \ + np.amin(x_adv) < -0.0001 or \ + np.isnan(np.amax(x_adv)): + print('Invalid pixel range. Expected [0, 1], found [{}, {}]'.format( + np.amin(x_adv), + np.amax(x_adv))) + else: + run_attack(checkpoint, x_adv, config['epsilon']) diff --git a/Robustness_Eval/mnist/train.py b/Robustness_Eval/mnist/train.py new file mode 100644 index 0000000..3844284 --- /dev/null +++ b/Robustness_Eval/mnist/train.py @@ -0,0 +1,122 @@ +""" +The code is adapted from the MadryLab's repo: +https://github.com/MadryLab/mnist_challenge/train.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import json +import os +import shutil +from timeit import default_timer as timer + +import tensorflow as tf +import numpy as np +from tensorflow.examples.tutorials.mnist import input_data + +from model import Model +from pgd_attack import LinfPGDAttack + +with open('config.json') as config_file: + config = json.load(config_file) + +# Setting up training parameters +tf.set_random_seed(config['random_seed']) + +max_num_training_steps = config['max_num_training_steps'] +num_output_steps = config['num_output_steps'] +num_summary_steps = config['num_summary_steps'] +num_checkpoint_steps = config['num_checkpoint_steps'] + +batch_size = config['training_batch_size'] + +# Setting up the data and the model +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) +global_step = tf.contrib.framework.get_or_create_global_step() +model = Model() + +# Setting up the optimizer +train_step = tf.train.AdamOptimizer(1e-4).minimize(model.xent, + global_step=global_step) + +# Set up adversary +attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + +# Setting up the Tensorboard and checkpoint outputs +model_dir = config['model_dir'] +if not os.path.exists(model_dir): + os.makedirs(model_dir) + +# We add accuracy and xent twice so we can easily make three types of +# comparisons in Tensorboard: +# - train vs eval (for a single run) +# - train of different runs +# - eval of different runs + +saver = tf.train.Saver(max_to_keep=3) +tf.summary.scalar('accuracy adv train', model.accuracy) +tf.summary.scalar('accuracy adv', model.accuracy) +tf.summary.scalar('xent adv train', model.xent / batch_size) +tf.summary.scalar('xent adv', model.xent / batch_size) +tf.summary.image('images adv train', model.x_image) +merged_summaries = tf.summary.merge_all() + +shutil.copy('config.json', model_dir) + +with tf.Session() as sess: + # Initialize the summary writer, global variables, and our time counter. + summary_writer = tf.summary.FileWriter(model_dir, sess.graph) + sess.run(tf.global_variables_initializer()) + training_time = 0.0 + + # Main training loop + for ii in range(max_num_training_steps): + x_batch, y_batch = mnist.train.next_batch(batch_size) + + # Compute Adversarial Perturbations + start = timer() + x_batch_adv = attack.perturb(x_batch, y_batch, sess) + end = timer() + training_time += end - start + + nat_dict = {model.x_input: x_batch, + model.y_input: y_batch} + + adv_dict = {model.x_input: x_batch_adv, + model.y_input: y_batch} + + # Output to stdout + if ii % num_output_steps == 0: + nat_acc = sess.run(model.accuracy, feed_dict=nat_dict) + adv_acc = sess.run(model.accuracy, feed_dict=adv_dict) + print('Step {}: ({})'.format(ii, datetime.now())) + print(' training nat accuracy {:.4}%'.format(nat_acc * 100)) + print(' training adv accuracy {:.4}%'.format(adv_acc * 100)) + if ii != 0: + print(' {} examples per second'.format( + num_output_steps * batch_size / training_time)) + training_time = 0.0 + # Tensorboard summaries + if ii % num_summary_steps == 0: + summary = sess.run(merged_summaries, feed_dict=adv_dict) + summary_writer.add_summary(summary, global_step.eval(sess)) + + # Write a checkpoint + if ii % num_checkpoint_steps == 0: + saver.save(sess, + os.path.join(model_dir, 'checkpoint'), + global_step=global_step) + + # Actual training step + start = timer() + sess.run(train_step, feed_dict=adv_dict) + end = timer() + training_time += end - start diff --git a/Robustness_Eval/readme.md b/Robustness_Eval/readme.md new file mode 100644 index 0000000..c8a3a9b --- /dev/null +++ b/Robustness_Eval/readme.md @@ -0,0 +1,48 @@ +SPADE_evaluation +=============================== + +SPADE: A Spectral Method for Black-Box Adversarial Robustness Evaluation + +Usage +----- + +**SPADE-Guided Robustness Evaluation** +1. To reproduce our experiment, extract data from following Models: + +`mnist(modified from https://github.com/MadryLab/mnist_challenge)` + + 1. config parameters and paths in config.json, mainly model_path and eps + 2. run train.py to train tensorflow model for mnist dataset(with adversial examples), the models are saved in checkpoints in ./models folder + 2b. model with eps=0 and 0.3 can also be downloaded using `fetch_model.py`, in this way: `python fetch_model.py natural`(or `adv_trained`) + 3a. when analyzing characteristic of model, run `gbt_eval.py` to evaluate with trained model through the train/test set(output paths can be set in the py file, while input and other config in config.json) + 3b. when runing for clever score, run model_converter.py to get the .h5 file containing structure and weights of the model. Paths are set at the head of the py file. + by default .5h model files are also stored in ./models folder + +`robustness(modified from https://github.com/MadryLab/robustness)` + + 1. install robustness package using pip or other similar tools + 2. download model from github page https://github.com/MadryLab/robustness, + or train model with robustness, e.g. + `python -m robustness.main --dataset restrict_imagenet --adv-train 0 --arch resnet50 --out-dir logs/checkpoints/dir/` + for a naturally trained resnet-50 model for cifar dataset, + more parameters see + https://robustness.readthedocs.io/en/latest/example_usage/cli_usage.html#training-a-standard-nonrobust-model + 3. evaluate through the model trained using `run.py`. + by default .pt model files are put together with `run.py`, and output files are stored in `./train_eval_results` foler + + +`CLEVER(modified from https://github.com/huanzhang12/CLEVER)` + + 1. run `python3 collect_gradients.py --data mnist --model_name 2-layer --target_type 16 --numimg 10 -s ./your data storage location` to get gredients. Data set options: `mnist` and `cifar`. Model_name to "2-layer" (MLP), "normal" (7-layer CNN), "distilled" (7-layer CNN with defensive distillation). For `mnist`, two more Model_name are `mnist_0` and `mnist_03` + 1b(SPADE-Guided CLEVER training). to calculate SPADE-Guided gredients, an example is adding `--ids ./normal_spade_nodelist/vanilla/nodes_cifar_cnn.csv` to 1. you can switch all csv file to ids to calculate different network gredients. nodes rankings are calculated with 10NN, `vanilla` stands for top ranked nodes, `vanilla_reverse` stands for reversed top ranked nodes. + 2. run `python3 clever.py --untargeted ./your_data_storage_location/` to get clever score. + + `AugMix(modified from https://github.com/google-research/augmix)` + + 1. run `python3 cifar.py -s ./your_data_storage_location/ -m networks_name --no-jsd ` for AugMix trained data + 2. run `python3 cifar.py -s ./your_data_storage_location/ -m networks_name --mix_off --no-jsd ` for standard trained data + networks options have `allconv`, `resnext`, `densenet`, and `WRN` + +`GAIRAT(modified from https://github.com/zjfheart/Geometry-aware-Instance-reweighted-Adversarial-Training.git)` + + 1. run `python3 GAIRAT.py --epsilon 0.25 --net 'resnet18' --out-dir './your_data_save_location' ` diff --git a/Robustness_Eval/robustness/run.py b/Robustness_Eval/robustness/run.py new file mode 100644 index 0000000..bcf663e --- /dev/null +++ b/Robustness_Eval/robustness/run.py @@ -0,0 +1,133 @@ +import torch +import torchvision.models as models +from robustness import model_utils, datasets, train, defaults +from robustness.datasets import CIFAR +from robustness.datasets import CINIC +from robustness.model_utils import make_and_restore_model +import numpy as np + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + +# read dataset from file +dataset = CIFAR() + +# read trained model +model_rn_0, _ = model_utils.make_and_restore_model(arch='resnet50', dataset=dataset, resume_path='cifar_nat_0.pt') +model_rn_025, _ = model_utils.make_and_restore_model(arch='resnet50', dataset=dataset, resume_path='cifar_l2_0_25.pt') +model_rn_05, _ = model_utils.make_and_restore_model(arch='resnet50', dataset=dataset, resume_path='cifar_l2_0_5.pt') +model_rn_1, _ = model_utils.make_and_restore_model(arch='resnet50', dataset=dataset, resume_path='cifar_l2_1_0.pt') + + +# img = img.to(device) +batch_size = 10 +element_size = 3*32*32 + +inp_data = np.zeros((1,3*32*32)) +op_data = np.zeros((1,10)) +label_data = np.zeros((1,1)) +acc_data = np.array([]) + +count_beat = 0 + +# create dataset loader +train_loader, _ = dataset.make_loaders(workers=0, batch_size=10) + + +for img, label in train_loader: + img = img.to(device) + op = model_rn_0(img) + op_data = np.vstack((op_data, op[0].detach().cpu().numpy())) + inp_data = np.vstack((inp_data, img.cpu().numpy().reshape(batch_size, element_size))) + label_data = np.append(label_data, label.cpu().numpy()) + count_beat = count_beat + 1 + print(count_beat) + + # if count_beat == 1: break; + +inp_data = inp_data[1:,:] +op_data = op_data[1:,:] +label_data = label_data[1:] +acc_data = np.append(acc_data, np.sum(np.argmax(op_data,1) == label_data)/op_data.shape[0]) + +np.savetxt('train_eval_results/testset_input_0.csv', inp_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_output_0.csv', op_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_label_0.csv', label_data, fmt="%d") + + +# 0.25 ========================= +inp_data = np.zeros((1,3*32*32)) +op_data = np.zeros((1,10)) +label_data = np.zeros((1,1)) +train_loader, _ = dataset.make_loaders(workers=0, batch_size=10) + +count_beat = 0 + +for img, label in train_loader: + img = img.to(device) + # target_label = (label + torch.randint_like(label, high=3))%10 + op = model_rn_025(img) + # adv_op = model_rn_025(img, target_label, **attack_kwargs) + inp_data = np.vstack((inp_data, img.cpu().numpy().reshape(batch_size, element_size))) + op_data = np.vstack((op_data, op[0].detach().cpu().numpy())) + # op_data = np.vstack((op_data, adv_op[0].detach().cpu().numpy())) + label_data = np.append(label_data, label.cpu().numpy()) + count_beat = count_beat + 1 + print(count_beat) + +inp_data = inp_data[1:,:] +op_data = op_data[1:,:] +label_data = label_data[1:] +acc_data = np.append(acc_data, np.sum(np.argmax(op_data,1) == label_data)/op_data.shape[0]) +np.savetxt('train_eval_results/testset_input_025.csv', inp_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_output_025.csv', op_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_label_025.csv', label_data, fmt="%d") + +# # 0.5 ================================= +inp_data = np.zeros((1,3*32*32)) +op_data = np.zeros((1,10)) +label_data = np.zeros((1,1)) +train_loader, _ = dataset.make_loaders(workers=0, batch_size=10) + +count_beat = 0 + +for img, label in train_loader: + img = img.to(device) + op = model_rn_05(img) + op_data = np.vstack((op_data, op[0].detach().cpu().numpy())) + inp_data = np.vstack((inp_data, img.cpu().numpy().reshape(batch_size, element_size))) + label_data = np.append(label_data, label.cpu().numpy()) + count_beat = count_beat + 1 + print(count_beat) + +inp_data = inp_data[1:,:] +op_data = op_data[1:,:] +label_data = label_data[1:] +acc_data = np.append(acc_data, np.sum(np.argmax(op_data,1) == label_data)/op_data.shape[0]) +np.savetxt('train_eval_results/testset_input_05.csv', inp_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_output_05.csv', op_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_label_05.csv', label_data, fmt="%d") + +# # # 1.0 ================================= +inp_data = np.zeros((1,3*32*32)) +op_data = np.zeros((1,10)) +label_data = np.zeros((1,1)) +train_loader, _ = dataset.make_loaders(workers=0, batch_size=10) + +count_beat = 0 + +for img, label in train_loader: + img = img.to(device) + op = model_rn_1(img) + op_data = np.vstack((op_data, op[0].detach().cpu().numpy())) + inp_data = np.vstack((inp_data, img.cpu().numpy().reshape(batch_size, element_size))) + label_data = np.append(label_data, label.cpu().numpy()) + count_beat = count_beat + 1 + print(count_beat) + +inp_data = inp_data[1:,:] +op_data = op_data[1:,:] +label_data = label_data[1:] +acc_data = np.append(acc_data,np.sum(np.argmax(op_data,1) == label_data)/op_data.shape[0]) +np.savetxt('train_eval_results/testset_input_1.csv', inp_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_output_1.csv', op_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_label_1.csv', label_data, fmt="%d") diff --git a/SPADE_score/SPADE.py b/SPADE_score/SPADE.py new file mode 100644 index 0000000..c3d733f --- /dev/null +++ b/SPADE_score/SPADE.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[1]: + + +import time +class Timer: + def __init__(self): + self._start_time = None + + def start(self): + """Start a new timer""" + if self._start_time is not None: + raise TimerError(f"Timer is running. Use .stop() to stop it") + + self._start_time = time.perf_counter() + + def stop(self): + """Stop the timer, and report the elapsed time""" + if self._start_time is None: + raise TimerError(f"Timer is not running. Use .start() to start it") + + elapsed_time = time.perf_counter() - self._start_time + self._start_time = None + print(f"Elapsed time: {elapsed_time:0.4f} seconds") + +# function [TopEig, TopEdgeList, TopNodeList]=RiemannianDist(Gx,Gy,num_eigs,gnd) +def GetRiemannianDist(Gx, Gy, num_eigs, gnd, one_class, julia): + + import scipy.sparse.linalg as sla + import numpy as np + import scipy as sp + import networkx as nx + import math + + Lx= nx.laplacian_matrix(Gx, nodelist=None, weight=None) + Ly= nx.laplacian_matrix(Gy, nodelist=None, weight=None) + + Lx = Lx.asfptype() + Ly = Ly.asfptype() + + if julia: + from julia.api import Julia + jl = Julia(compiled_modules=False) + from julia import Main + Main.include("./eigen.jl") + print('Generate eigenpairs') + [Dxy, Uxy] = Main.main(l_in, l_out, k) + else: + [Dxy, Uxy] = sla.eigs(Lx, num_eigs, Ly) + + num_node_tot=Uxy.shape[0] + num_top_node=10; + dist=np.zeros((1,1)); + num_cluster=5; + + TopEig=max(Dxy) + NodeDegree=Lx.diagonal() + + num_edge_tot=len(Gx.edges) # number of total edges + + Zpq=np.zeros((num_edge_tot,));# edge embedding distance + + p = np.array(Gx.edges)[:,0];# one end node of each edge + q = np.array(Gx.edges)[:,1];# another end node of each edge + + density=math.ceil(num_edge_tot/num_node_tot) + + for i in np.arange(0,num_eigs): + Zpq = Zpq + np.power(Uxy[p,i]-Uxy[q,i], 2)*Dxy[i] + + Zpq = Zpq/max(Zpq) + node_score=np.zeros((num_node_tot,)) + + if not one_class: + label_score=np.zeros(np.unique(gnd).shape,) + else: + label_score=np.zeros(10,) + + for i in np.arange(0,num_edge_tot): + node_score[p[i]]=node_score[p[i]]+Zpq[i] + node_score[q[i]]=node_score[q[i]]+Zpq[i] + node_score=node_score/NodeDegree + node_score=node_score/np.amax(node_score) + + for i in np.arange(0,num_node_tot): + idx = int(gnd[i]); + label_score[idx]=label_score[idx]+node_score[i]; + + label_score=label_score/max(label_score); + + II = np.flip(node_score.argsort(axis=0)) + YY = node_score + YY.sort(axis=0) + YY = np.flip(YY) + TopNodeList=II + node_top=II[0:num_top_node] + gndSort=gnd[II]; + I = np.flip(Zpq.argsort(axis=0)) + Y = node_score + Y.sort(axis=0) + Y = np.flip(Y) + TopEdgeList=np.column_stack((p,q))[I,:] + + return TopEig, TopEdgeList, TopNodeList + +def hnsw(data, k): + import hnswlib + import numpy as np + import pickle + + dim = data.shape[1] + num_elements = data.shape[0] + data_labels = np.arange(num_elements) + + # Declaring index + p = hnswlib.Index(space = 'l2', dim = dim) # possible options are l2, cosine or ip + + # Initializing index - the maximum number of elements should be known beforehand + p.init_index(max_elements = num_elements, ef_construction = 200, M = 16) + + # Element insertion (can be called several times): + p.add_items(data, data_labels) + + # Controlling the recall by setting ef: + p.set_ef(50) # ef should always be > k + + # Query dataset, k - number of closest elements (returns 2 numpy arrays) + labels, distances = p.knn_query(data, k) + + return labels, distances + +def Spade(data_input, data_output, data_labels, k, num_eigs, massive_data=True, full_function=True, The_faiss=True, graph=False, one_class=False, julia = False): + import numpy as np + import pandas as pd + import networkx as nx + import scipy.linalg as la + from scipy.linalg import eigh + import scipy.sparse.linalg as sla + import numpy as np + from scipy.sparse import lil_matrix + import scipy.sparse as sp + + t = Timer() + t.start() + + if data_input.shape[0] == data_output.shape[0] is False: + return('input and output must have the same sample numbers') + + # use faiss + if The_faiss: + import faiss + data_input = data_input.astype('float32') + num_elements = data_input.shape[0] + dim = data_input.shape[1] + index = faiss.IndexFlatL2(dim) # build the index + index.add(data_input) # add vectors to the index + print("totual elements:{}".format(index.ntotal)) + _, I_in = index.search(data_input, k) # actual search + + + data_output = data_output.astype('float32') + num_elements = data_output.shape[0] + dim = data_output.shape[1] + index = faiss.IndexFlatL2(dim) # build the index + index.add(data_output) # add vectors to the index + print("totual elements:{}".format(index.ntotal)) + _, I_out = index.search(data_output, k) # actual search + + # use hnsw + else: + I_in,_ = hnsw(data_input, k) + I_out,_ = hnsw(data_output, k) + + if massive_data: + A_in = lil_matrix((data_input.shape[0], data_input.shape[0])) + A_out = lil_matrix((data_input.shape[0], data_input.shape[0])) + for i in range(0, data_input.shape[0]): + for j_in in I_in[i][1:]: + A_in[i,j_in] = 1 + + for j_out in I_out[i][1:]: + A_out[i,j_out] = 1 + Gx = nx.from_scipy_sparse_matrix(A_in) + Gy = nx.from_scipy_sparse_matrix(A_out) + + else: + A_in = np.zeros((data_input.shape[0], data_input.shape[0])) + A_out = np.zeros((data_output.shape[0], data_output.shape[0])) + for i in range(0, data_input.shape[0]): + for j_in in I_in[i][1:]: + A_in[i][j_in] = 1 + for j_out in I_out[i][1:]: + A_out[i][j_out] = 1 + + A_in = pd.DataFrame(A_in) + A_out = pd.DataFrame(A_out) + Gx = nx.from_pandas_adjacency(A_in) + Gy = nx.from_pandas_adjacency(A_out) + + print('Is input graph connected?:',nx.is_connected(Gx)) + print('Is output graph connected?:',nx.is_connected(Gy)) + + if full_function: + TopEig, TopEdgeList, TopNodeList = GetRiemannianDist(Gx, Gy, num_eigs, data_labels,one_class, julia)# full function + t.stop() + if graph: + return TopEig, TopEdgeList, TopNodeList, Gx, Gy + else: + return TopEig, TopEdgeList, TopNodeList + + else: + input_ = nx.laplacian_matrix(Gx, nodelist=None, weight=None)#.todense() + output_ = nx.laplacian_matrix(Gy, nodelist=None, weight=None)#.todense() + input_ = input_.asfptype()# use it if choose hnsw(data, k) + output_ = output_.asfptype()#use it if choose hnsw(data, k) + if julia: + from julia.api import Julia + jl = Julia(compiled_modules=False) + from julia import Main + Main.include("./eigen.jl") + print('Generate eigenpairs') + [V,D] = Main.main(input_, output_, num_eigs) + + else: + [V,D] = sla.eigs(input_, num_eigs, output_)#for eigs only + t.stop() + + if graph: + return V, Gx, Gy + else: + return V + + + + + + diff --git a/SPADE_score/eigen.jl b/SPADE_score/eigen.jl new file mode 100644 index 0000000..144f27c --- /dev/null +++ b/SPADE_score/eigen.jl @@ -0,0 +1,35 @@ +using LinearAlgebra +using LinearMaps +using MAT +using SparseArrays +using Arpack + +using PyCall, SparseArrays + +function scipyCSC_to_julia(A) + m, n = A.shape + colPtr = Int[i+1 for i in PyArray(A."indptr")] + rowVal = Int[i+1 for i in PyArray(A."indices")] + nzVal = Vector{Float64}(PyArray(A."data")) + B = SparseMatrixCSC{Float64,Int}(m, n, colPtr, rowVal, nzVal) + return PyCall.pyjlwrap_new(B) +end + + +function main(PyX, PyY, k::Int64) + m, n = PyX.shape + colPtr = Int[i+1 for i in PyArray(PyX."indptr")] + rowVal = Int[i+1 for i in PyArray(PyX."indices")] + nzVal = Vector{Float64}(PyArray(PyX."data")) + X = SparseMatrixCSC{Float64,Int}(m, n, colPtr, rowVal, nzVal) + + m, n = PyY.shape + colPtr = Int[i+1 for i in PyArray(PyY."indptr")] + rowVal = Int[i+1 for i in PyArray(PyY."indices")] + nzVal = Vector{Float64}(PyArray(PyY."data")) + Y = SparseMatrixCSC{Float64,Int}(m, n, colPtr, rowVal, nzVal) + + (Λ, V) = eigs(X, Y, nev=k, tol=1e-6, which=:LM) + + return Λ, V +end diff --git a/SPADE_score/readme.md b/SPADE_score/readme.md new file mode 100644 index 0000000..1b37fad --- /dev/null +++ b/SPADE_score/readme.md @@ -0,0 +1,28 @@ +SPADE_score +=============================== + +SPADE: A Spectral Method for Black-Box Adversarial Robustness Evaluation + + +Requirements +----- +hnswlib == 0.3.4 +or +faiss == 1.7.0 + +Usage +----- + +**SPADE-Score Evaluation Usage** +Download SPADE.py file. In your Python code, using: +1. `from SPADE import Spade` +2. `TopEig, TopEdgeList, TopNodeList = Spade(data_input, data_output, data_labels, k, num_eigs)` +`data_input` and ` data_output` requie flattening. Please do image flattening if your data is multidimensional array +3. Options default: full_function=True, The_faiss=True, graph=False, one_class=False, julia = False +4. Different graph-based manifold constructions can be chose by `The_faiss`, `The_faiss=Flse` import hnswlib, `The_faiss=True` import faiss +5. Quick eigenvalues calculation using `EigenValues = Spade(data_input, data_output, data_labels, k, num_eigs, full_function=False)` +6. We noticed in some rare case, `scipy.sparse.linalg.eigs` cannot compute eigenvalues correctly(like maximum eigenvalue is negative) if you experience the same case, `julia = True` may helps but it requires Julia installation and download file `eigen.jl` + +**SPADE-Guided Robustness Evaluation** +1. You can extract data from your own model then repeat `SPADE-SCORE Evaluation Usage` +2. To reproduce our experiment, `cd evaluation/` diff --git a/adv_train/README.md b/adv_train/README.md new file mode 100644 index 0000000..d08ad85 --- /dev/null +++ b/adv_train/README.md @@ -0,0 +1,36 @@ +SPADE-adv\_train +=============================== + +The adversarial training application of SPADE. For MNIST, we adapt the Tensorflow code from [MadryLab's repo](https://github.com/MadryLab/mnist_challenge/). As for CIFAR10, we adapt the PyTorch code from [locuslab's repo](https://github.com/locuslab/fast_adversarial/tree/master/CIFAR10) to speedup the adversarial training from 3 days to 2 hours on a single GPU. + + +Requirements +------------ +* tensorflow <= 1.15 (Only required for MNIST) +* googledrivedownloader (Only required if you fetch the pre-trained models for MNIST) +* [Requirements in locuslab/fast_adversarial](https://github.com/locuslab/fast_adversarial/tree/master/CIFAR10) (Only required for CIFAR10) +* Julia (Only required for computing node SPADE score on CIFAR10) +* hnswlib (Only required for computing node SPADE score on CIFAR10) + + +Usage +----- + +**MNIST Usage** + +1. `cd mnist/` + +2. `python fetch_model.py --method spade` (Only required if you fetch the pre-trained model) + +3. `python train.py --device 0 --method spade` (Only required if you train the model from scratch) + +4. `python eval.py --device 0 --method spade` + + +**CIFAR10 Usage** + +1. `cd cifar10/` + +2. `python train.py --method spade` (Only required if you train the model from scratch) + +3. `python eval.py --method spade` diff --git a/adv_train/cifar10/eigen.jl b/adv_train/cifar10/eigen.jl new file mode 100644 index 0000000..144f27c --- /dev/null +++ b/adv_train/cifar10/eigen.jl @@ -0,0 +1,35 @@ +using LinearAlgebra +using LinearMaps +using MAT +using SparseArrays +using Arpack + +using PyCall, SparseArrays + +function scipyCSC_to_julia(A) + m, n = A.shape + colPtr = Int[i+1 for i in PyArray(A."indptr")] + rowVal = Int[i+1 for i in PyArray(A."indices")] + nzVal = Vector{Float64}(PyArray(A."data")) + B = SparseMatrixCSC{Float64,Int}(m, n, colPtr, rowVal, nzVal) + return PyCall.pyjlwrap_new(B) +end + + +function main(PyX, PyY, k::Int64) + m, n = PyX.shape + colPtr = Int[i+1 for i in PyArray(PyX."indptr")] + rowVal = Int[i+1 for i in PyArray(PyX."indices")] + nzVal = Vector{Float64}(PyArray(PyX."data")) + X = SparseMatrixCSC{Float64,Int}(m, n, colPtr, rowVal, nzVal) + + m, n = PyY.shape + colPtr = Int[i+1 for i in PyArray(PyY."indptr")] + rowVal = Int[i+1 for i in PyArray(PyY."indices")] + nzVal = Vector{Float64}(PyArray(PyY."data")) + Y = SparseMatrixCSC{Float64,Int}(m, n, colPtr, rowVal, nzVal) + + (Λ, V) = eigs(X, Y, nev=k, tol=1e-6, which=:LM) + + return Λ, V +end diff --git a/adv_train/cifar10/eval.py b/adv_train/cifar10/eval.py new file mode 100644 index 0000000..1ded892 --- /dev/null +++ b/adv_train/cifar10/eval.py @@ -0,0 +1,71 @@ +import argparse +import logging +import os +import sys +import time + +import apex.amp as amp +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from preact_resnet import PreActResNet18 +from utils import (upper_limit, lower_limit, std, clamp, get_loaders, + evaluate_pgd, evaluate_standard) +from spade import * + +logger = logging.getLogger(__name__) + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--batch-size', default=128, type=int) + parser.add_argument('--data-dir', default='./cifar-data', type=str) + parser.add_argument('--eval_epsilon', default=8, type=int) + parser.add_argument('--epsilon', default=14, type=int) + parser.add_argument('--small_epsilon', default=12, type=int) + parser.add_argument('--iters', default=50, type=int, help='Attack iterations') + parser.add_argument('--restarts', default=10, type=int) + parser.add_argument('--method', default='spade', type=str, choices=['spade', 'pgd', 'random']) + parser.add_argument('--out-dir', default='log', type=str, help='Output directory') + parser.add_argument('--seed', default=0, type=int, help='Random seed') + return parser.parse_args() + + +def main(): + args = get_args() + if args.method == "pgd": + args.out_dir = args.method + "_{}".format(args.epsilon) + else: + args.out_dir = args.method + "_{}_{}".format(args.small_epsilon, args.epsilon) + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir, exist_ok=True) + logfile = os.path.join(args.out_dir, f'output.log') + + logging.basicConfig( + format='[%(asctime)s] - %(message)s', + datefmt='%Y/%m/%d %H:%M:%S', + level=logging.INFO, + filename=logfile) + logger.info(args) + + np.random.seed(args.seed) + torch.manual_seed(args.seed) + torch.cuda.manual_seed(args.seed) + + _, test_loader, _ = get_loaders(args.data_dir, args.batch_size) + + # Evaluation + model_test = PreActResNet18().cuda() + model_test.load_state_dict(torch.load(os.path.join(args.out_dir, 'model.pth'))) + model_test.float() + model_test.eval() + + pgd_loss, pgd_acc = evaluate_pgd(test_loader, model_test, args.iters, args.restarts, args.eval_epsilon) + test_loss, test_acc = evaluate_standard(test_loader, model_test) + + logger.info('Test Loss \t Test Acc \t PGD Loss \t PGD Acc') + logger.info('%.4f \t \t %.4f \t %.4f \t %.4f', test_loss, test_acc, pgd_loss, pgd_acc) + +if __name__ == "__main__": + main() diff --git a/adv_train/cifar10/knn_graph.gpickle b/adv_train/cifar10/knn_graph.gpickle new file mode 100644 index 0000000..ec0cc25 Binary files /dev/null and b/adv_train/cifar10/knn_graph.gpickle differ diff --git a/adv_train/cifar10/l_in.npz b/adv_train/cifar10/l_in.npz new file mode 100644 index 0000000..d6b076a Binary files /dev/null and b/adv_train/cifar10/l_in.npz differ diff --git a/adv_train/cifar10/pgd_12/model.pth b/adv_train/cifar10/pgd_12/model.pth new file mode 100644 index 0000000..6ae8d57 Binary files /dev/null and b/adv_train/cifar10/pgd_12/model.pth differ diff --git a/adv_train/cifar10/pgd_14/model.pth b/adv_train/cifar10/pgd_14/model.pth new file mode 100644 index 0000000..dc73b94 Binary files /dev/null and b/adv_train/cifar10/pgd_14/model.pth differ diff --git a/adv_train/cifar10/pgd_8/model.pth b/adv_train/cifar10/pgd_8/model.pth new file mode 100644 index 0000000..699fbd3 Binary files /dev/null and b/adv_train/cifar10/pgd_8/model.pth differ diff --git a/adv_train/cifar10/preact_resnet.py b/adv_train/cifar10/preact_resnet.py new file mode 100644 index 0000000..3d41634 --- /dev/null +++ b/adv_train/cifar10/preact_resnet.py @@ -0,0 +1,93 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class PreActBlock(nn.Module): + '''Pre-activation version of the BasicBlock.''' + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(PreActBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False) + ) + + def forward(self, x): + out = F.relu(self.bn1(x)) + shortcut = self.shortcut(x) if hasattr(self, 'shortcut') else x + out = self.conv1(out) + out = self.conv2(F.relu(self.bn2(out))) + out += shortcut + return out + + +class PreActBottleneck(nn.Module): + '''Pre-activation version of the original Bottleneck module.''' + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(PreActBottleneck, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) + + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False) + ) + + def forward(self, x): + out = F.relu(self.bn1(x)) + shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x + out = self.conv1(out) + out = self.conv2(F.relu(self.bn2(out))) + out = self.conv3(F.relu(self.bn3(out))) + out += shortcut + return out + + +class PreActResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(PreActResNet, self).__init__() + self.in_planes = 64 + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.bn = nn.BatchNorm2d(512 * block.expansion) + self.linear = nn.Linear(512 * block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = self.conv1(x) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.relu(self.bn(out)) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def PreActResNet18(): + return PreActResNet(PreActBlock, [2,2,2,2]) diff --git a/adv_train/cifar10/random_12_14/model.pth b/adv_train/cifar10/random_12_14/model.pth new file mode 100644 index 0000000..aa86e92 Binary files /dev/null and b/adv_train/cifar10/random_12_14/model.pth differ diff --git a/adv_train/cifar10/spade.py b/adv_train/cifar10/spade.py new file mode 100644 index 0000000..8436b0f --- /dev/null +++ b/adv_train/cifar10/spade.py @@ -0,0 +1,114 @@ +import hnswlib +import numpy as np +from scipy.io import loadmat +import scipy.sparse +from scipy.sparse import coo_matrix, diags, identity, csr_matrix +from scipy.sparse.linalg import eigsh +import networkx as nx +from numpy import linalg as LA +import sklearn +import time +import sys +from scipy.io import mmwrite + +def eigs(l_in, l_out, k): + from julia.api import Julia + jl = Julia(compiled_modules=False) + from julia import Main + Main.include("./eigen.jl") + print('Generate eigenpairs') + eigenvalues, eigenvectors = Main.main(l_in, l_out, k) + + return eigenvalues.real, eigenvectors.real + +def node_score(l_in, l_out): + print("computing eigens") + eigenvalues, eigenvectors = eigs(l_in, l_out, 2) + eigenvalues = np.sqrt(eigenvalues).reshape(1,-1) + embeddings = eigenvalues * eigenvectors + + G_in = nx.read_gpickle("knn_graph.gpickle") + + print("computing weighted degree") + for e in G_in.edges(): + src = e[0] + dst = e[1] + diff = embeddings[src] - embeddings[dst] + G_in[src][dst]['weight'] = np.inner(diff, diff) + + degree = {pair[0]: pair[1] for pair in G_in.degree()} + weighted_degree = {pair[0]: pair[1] for pair in G_in.degree(weight='weight')} + + print("computing node spade score") + num_nodes = len(degree) + node_spade_score = np.zeros(num_nodes) + for i in range(num_nodes): + if degree[i] == 0: + print("isolated node, pls check the knn graph!!!!!!") + sys.exit() + node_spade_score[i] = weighted_degree[i]/degree[i] + + node_spade_score = (1.0/np.amax(node_spade_score)) * node_spade_score + + return node_spade_score + +def construct_adj(neighs): + dim = neighs.shape[0] + k = neighs.shape[1] - 1 + + idx0 = np.asarray(list(range(dim))) + idx1 = neighs[:,0] + mismatch_idx = ~np.isclose(idx0, idx1, rtol=1e-6) + neighs[mismatch_idx, 1:] = neighs[mismatch_idx, :k] + row = (np.repeat(idx0.reshape(-1,1), k, axis=1)).reshape(-1,) + col = neighs[:,1:].reshape(-1,) + all_row = np.concatenate((row, col), axis=0) + all_col = np.concatenate((col, row), axis=0) + data = np.ones(all_row.shape[0]) + adj = csr_matrix((data, (all_row, all_col)), shape=(dim, dim)) + adj.data[:] = 1 + + return adj + +def adj2laplacian(A): + D = diags(np.squeeze(np.asarray(A.sum(axis=1))), 0) + L = D - A + identity(A.shape[0]).multiply(1e-6) + + return L + +def hnsw(features, k=10, ef=100, M=48, save_index_file=None, gen_graph=None): + print("constructing hnsw") + num_samples, dim = features.shape + p = hnswlib.Index(space='l2', dim=dim) + p.init_index(max_elements=num_samples, ef_construction=ef, M=M) + labels_index = np.arange(num_samples) + p.add_items(features, labels_index) + + p.set_ef(ef) + + if save_index_file: + p.save_index(save_index_file) + + print("constructing knn") + neighs, _ = p.knn_query(features, k+1) + adj = construct_adj(neighs) + + if gen_graph: + print("construct graph!!!!!!") + G = nx.from_scipy_sparse_matrix(adj) + nx.write_gpickle(G, "knn_graph.gpickle") + + laplacian = adj2laplacian(adj) + #scipy.sparse.save_npz('laplacian.npz', laplacian) + + return laplacian + +def spade(feat_in, feat_out, k=10): + #l_in = hnsw(feat_in, k=k, gen_graph=True) + #scipy.sparse.save_npz("l_in.npz", l_in) + #sys.exit() + l_in = scipy.sparse.load_npz("l_in.npz") + l_out = hnsw(feat_out, k=k) + node_spade_score = node_score(l_in, l_out) + + return node_spade_score diff --git a/adv_train/cifar10/spade_12_14/model.pth b/adv_train/cifar10/spade_12_14/model.pth new file mode 100644 index 0000000..a7b4b01 Binary files /dev/null and b/adv_train/cifar10/spade_12_14/model.pth differ diff --git a/adv_train/cifar10/train.py b/adv_train/cifar10/train.py new file mode 100644 index 0000000..86b7240 --- /dev/null +++ b/adv_train/cifar10/train.py @@ -0,0 +1,186 @@ +import argparse +import logging +import os +import sys +import time + +import apex.amp as amp +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from preact_resnet import PreActResNet18 +from utils import (upper_limit, lower_limit, std, clamp, get_loaders, + evaluate_pgd, evaluate_standard) +from spade import * + +logger = logging.getLogger(__name__) + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--topk', default=45000, type=int) + parser.add_argument('--batch-size', default=128, type=int) + parser.add_argument('--data-dir', default='./cifar-data', type=str) + parser.add_argument('--epochs', default=30, type=int) + parser.add_argument('--lr-schedule', default='cyclic', type=str, choices=['cyclic', 'multistep']) + parser.add_argument('--lr-min', default=0., type=float) + parser.add_argument('--lr-max', default=0.2, type=float) + parser.add_argument('--weight-decay', default=5e-4, type=float) + parser.add_argument('--momentum', default=0.9, type=float) + parser.add_argument('--epsilon', default=14, type=int) + parser.add_argument('--small_epsilon', default=12, type=int) + parser.add_argument('--eval_epsilon', default=8, type=int) + parser.add_argument('--attack-iters', default=7, type=int, help='Attack iterations') + parser.add_argument('--iters', default=50, type=int, help='Eval Attack iterations') + parser.add_argument('--restarts', default=10, type=int) + parser.add_argument('--alpha', default=2, type=int, help='Step size') + parser.add_argument('--delta-init', default='random', choices=['zero', 'random'], + help='Perturbation initialization method') + parser.add_argument('--out-dir', default='log', type=str, help='Output directory') + parser.add_argument('--seed', default=0, type=int, help='Random seed') + parser.add_argument('--method', default='spade', type=str, choices=['spade', 'pgd', 'random']) + parser.add_argument('--opt-level', default='O2', type=str, choices=['O0', 'O1', 'O2'], + help='O0 is FP32 training, O1 is Mixed Precision, and O2 is "Almost FP16" Mixed Precision') + parser.add_argument('--loss-scale', default='1.0', type=str, choices=['1.0', 'dynamic'], + help='If loss_scale is "dynamic", adaptively adjust the loss scale over time') + parser.add_argument('--master-weights', action='store_true', + help='Maintain FP32 master weights to accompany any FP16 model weights, not applicable for O1 opt level') + return parser.parse_args() + + +def main(): + args = get_args() + if args.method == "pgd": + args.out_dir = args.method + "_{}".format(args.epsilon) + else: + args.out_dir = args.method + "_{}_{}".format(args.small_epsilon, args.epsilon) + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir, exist_ok=True) + logfile = os.path.join(args.out_dir, f'output.log') + if os.path.exists(logfile): + os.remove(logfile) + + logging.basicConfig( + format='[%(asctime)s] - %(message)s', + datefmt='%Y/%m/%d %H:%M:%S', + level=logging.INFO, + filename=logfile) + logger.info(args) + + np.random.seed(args.seed) + torch.manual_seed(args.seed) + torch.cuda.manual_seed(args.seed) + + train_loader, test_loader, eps_loader = get_loaders(args.data_dir, args.batch_size) + + epsilon_all = torch.cat(50000*[(args.epsilon/255. / std).view(1,3,1,1)]) + alpha = (args.alpha / 255.) / std + + model = PreActResNet18().cuda() + model.train() + + opt = torch.optim.SGD(model.parameters(), lr=args.lr_max, momentum=args.momentum, weight_decay=args.weight_decay) + amp_args = dict(opt_level=args.opt_level, loss_scale=args.loss_scale, verbosity=False) + if args.opt_level == 'O2': + amp_args['master_weights'] = args.master_weights + model, opt = amp.initialize(model, opt, **amp_args) + criterion = nn.CrossEntropyLoss() + + lr_steps = args.epochs * len(train_loader) + if args.lr_schedule == 'cyclic': + scheduler = torch.optim.lr_scheduler.CyclicLR(opt, base_lr=args.lr_min, max_lr=args.lr_max, + step_size_up=lr_steps / 2, step_size_down=lr_steps / 2) + elif args.lr_schedule == 'multistep': + scheduler = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[lr_steps / 2, lr_steps * 3 / 4], gamma=0.1) + + # compute spade/random score + # and assign epsilon accordingly + if args.method != "pgd": + ins = [] + outs = [] + ids = [] + for i, (img, _, ind) in enumerate(eps_loader): + ins.append(img.numpy().reshape(-1,3072)) + outs.append(model(img.cuda()).detach().clone().cpu().numpy()) + ids.append(ind) + ids = np.concatenate(ids, axis=0) + sort_idx = np.argsort(np.asarray(ids)) + ins = np.concatenate(ins, axis=0)[sort_idx] + outs = np.concatenate(outs, axis=0)[sort_idx] + spade_score = spade(ins, outs) + eps = args.epsilon*np.ones(50000) + idx = (-spade_score).argsort() + if args.method == "random": + np.random.shuffle(idx) + sub_idx = idx[args.topk:] + eps[idx] = args.small_epsilon + eps = torch.from_numpy(eps).float().cuda() + epsilon_all = ((eps/255.).view(-1,1,1,1) / std.view(1,3,1,1)) + + # Training + start_train_time = time.time() + logger.info('Epoch \t Seconds \t LR \t \t Train Loss \t Train Acc') + for epoch in range(args.epochs): + start_epoch_time = time.time() + train_loss = 0 + train_acc = 0 + train_n = 0 + for i, (X, y, idx) in enumerate(train_loader): + X, y, idx = X.cuda(), y.cuda(), idx.cuda() + epsilon = epsilon_all[idx] + + if args.method == "spade": + clean_output = model(X) + clean_index = torch.where(clean_output.max(1)[1]!=y) + epsilon[clean_index] = 0 + + delta = torch.zeros_like(X).cuda() + if args.delta_init == 'random': + delta = 2*epsilon * torch.rand((X.size())).cuda() - epsilon + delta.data = clamp(delta, lower_limit - X, upper_limit - X) + delta.requires_grad = True + for _ in range(args.attack_iters): + output = model(X + delta) + loss = criterion(output, y) + with amp.scale_loss(loss, opt) as scaled_loss: + scaled_loss.backward() + grad = delta.grad.detach() + delta.data = clamp(delta + alpha * torch.sign(grad), -epsilon, epsilon) + delta.data = clamp(delta, lower_limit - X, upper_limit - X) + delta.grad.zero_() + delta = delta.detach() + output = model(X + delta) + loss = criterion(output, y) + opt.zero_grad() + with amp.scale_loss(loss, opt) as scaled_loss: + scaled_loss.backward() + opt.step() + train_loss += loss.item() * y.size(0) + train_acc += (output.max(1)[1] == y).sum().item() + train_n += y.size(0) + scheduler.step() + epoch_time = time.time() + lr = scheduler.get_last_lr()[0] + logger.info('%d \t %.1f \t \t %.4f \t %.4f \t %.4f', + epoch, epoch_time - start_epoch_time, lr, train_loss/train_n, train_acc/train_n) + + train_time = time.time() + torch.save(model.state_dict(), os.path.join(args.out_dir, 'model.pth')) + logger.info('Total train time: %.4f minutes', (train_time - start_train_time)/60) + + # Evaluation + model_test = PreActResNet18().cuda() + model_test.load_state_dict(model.state_dict()) + model_test.float() + model_test.eval() + + pgd_loss, pgd_acc = evaluate_pgd(test_loader, model_test, args.iters, args.restarts, args.eval_epsilon) + test_loss, test_acc = evaluate_standard(test_loader, model_test) + + logger.info('Test Loss \t Test Acc \t PGD Loss \t PGD Acc') + logger.info('%.4f \t \t %.4f \t %.4f \t %.4f', test_loss, test_acc, pgd_loss, pgd_acc) + + +if __name__ == "__main__": + main() diff --git a/adv_train/cifar10/utils.py b/adv_train/cifar10/utils.py new file mode 100644 index 0000000..742f3a6 --- /dev/null +++ b/adv_train/cifar10/utils.py @@ -0,0 +1,140 @@ +import apex.amp as amp +import torch +import torch.nn.functional as F +from torchvision import datasets, transforms +from torch.utils.data import Dataset +from torch.utils.data.sampler import SubsetRandomSampler +import numpy as np + +cifar10_mean = (0.4914, 0.4822, 0.4465) +cifar10_std = (0.2471, 0.2435, 0.2616) + +mu = torch.tensor(cifar10_mean).view(3,1,1).cuda() +std = torch.tensor(cifar10_std).view(3,1,1).cuda() + +upper_limit = ((1 - mu)/ std) +lower_limit = ((0 - mu)/ std) + + +def clamp(X, lower_limit, upper_limit): + return torch.max(torch.min(X, upper_limit), lower_limit) + +class EpsDataset(Dataset): + def __init__(self, datafolder): + self.datafolder = datafolder + + def __len__(self): + return len(self.datafolder) + + def __getitem__(self, idx): + x, y = self.datafolder[idx] + + return x, y, idx + +def get_loaders(dir_, batch_size): + train_transform = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(cifar10_mean, cifar10_std), + ]) + test_transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(cifar10_mean, cifar10_std), + ]) + num_workers = 2 + train_dataset = datasets.CIFAR10( + dir_, train=True, transform=train_transform, download=True) + new_train_dataset = EpsDataset(train_dataset) + test_dataset = datasets.CIFAR10( + dir_, train=False, transform=test_transform, download=True) + eps_loader = torch.utils.data.DataLoader( + dataset=new_train_dataset, + batch_size=batch_size, + shuffle=False, + pin_memory=True, + num_workers=2, + ) + train_loader = torch.utils.data.DataLoader( + dataset=new_train_dataset, + batch_size=batch_size, + shuffle=True, + pin_memory=True, + num_workers=num_workers, + ) + test_loader = torch.utils.data.DataLoader( + dataset=test_dataset, + batch_size=batch_size, + shuffle=False, + pin_memory=True, + num_workers=2, + ) + return train_loader, test_loader, eps_loader + + +def attack_pgd(model, X, y, epsilon, alpha, attack_iters, restarts, opt=None): + max_loss = torch.zeros(y.shape[0]).cuda() + max_delta = torch.zeros_like(X).cuda() + for zz in range(restarts): + delta = torch.zeros_like(X).cuda() + for i in range(len(epsilon)): + delta[:, i, :, :].uniform_(-epsilon[i][0][0].item(), epsilon[i][0][0].item()) + delta.data = clamp(delta, lower_limit - X, upper_limit - X) + delta.requires_grad = True + for _ in range(attack_iters): + output = model(X + delta) + index = torch.where(output.max(1)[1] == y) + if len(index[0]) == 0: + break + loss = F.cross_entropy(output, y) + if opt is not None: + with amp.scale_loss(loss, opt) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + grad = delta.grad.detach() + d = delta[index[0], :, :, :] + g = grad[index[0], :, :, :] + d = clamp(d + alpha * torch.sign(g), -epsilon, epsilon) + d = clamp(d, lower_limit - X[index[0], :, :, :], upper_limit - X[index[0], :, :, :]) + delta.data[index[0], :, :, :] = d + delta.grad.zero_() + all_loss = F.cross_entropy(model(X+delta), y, reduction='none').detach() + max_delta[all_loss >= max_loss] = delta.detach()[all_loss >= max_loss] + max_loss = torch.max(max_loss, all_loss) + return max_delta + + +def evaluate_pgd(test_loader, model, attack_iters, restarts, eps): + epsilon = (eps / 255.) / std + alpha = (2 / 255.) / std + pgd_loss = 0 + pgd_acc = 0 + n = 0 + model.eval() + for i, (X, y) in enumerate(test_loader): + X, y = X.cuda(), y.cuda() + pgd_delta = attack_pgd(model, X, y, epsilon, alpha, attack_iters, restarts) + with torch.no_grad(): + output = model(X + pgd_delta) + loss = F.cross_entropy(output, y) + pgd_loss += loss.item() * y.size(0) + pgd_acc += (output.max(1)[1] == y).sum().item() + n += y.size(0) + return pgd_loss/n, pgd_acc/n + + +def evaluate_standard(test_loader, model): + test_loss = 0 + test_acc = 0 + n = 0 + model.eval() + with torch.no_grad(): + for i, (X, y) in enumerate(test_loader): + X, y = X.cuda(), y.cuda() + output = model(X) + loss = F.cross_entropy(output, y) + test_loss += loss.item() * y.size(0) + test_acc += (output.max(1)[1] == y).sum().item() + n += y.size(0) + return test_loss/n, test_acc/n diff --git a/adv_train/mnist/MNIST_data/t10k-images-idx3-ubyte.gz b/adv_train/mnist/MNIST_data/t10k-images-idx3-ubyte.gz new file mode 100644 index 0000000..5ace8ea Binary files /dev/null and b/adv_train/mnist/MNIST_data/t10k-images-idx3-ubyte.gz differ diff --git a/adv_train/mnist/MNIST_data/t10k-labels-idx1-ubyte.gz b/adv_train/mnist/MNIST_data/t10k-labels-idx1-ubyte.gz new file mode 100644 index 0000000..a7e1415 Binary files /dev/null and b/adv_train/mnist/MNIST_data/t10k-labels-idx1-ubyte.gz differ diff --git a/adv_train/mnist/MNIST_data/train-images-idx3-ubyte.gz b/adv_train/mnist/MNIST_data/train-images-idx3-ubyte.gz new file mode 100644 index 0000000..b50e4b6 Binary files /dev/null and b/adv_train/mnist/MNIST_data/train-images-idx3-ubyte.gz differ diff --git a/adv_train/mnist/MNIST_data/train-labels-idx1-ubyte.gz b/adv_train/mnist/MNIST_data/train-labels-idx1-ubyte.gz new file mode 100644 index 0000000..707a576 Binary files /dev/null and b/adv_train/mnist/MNIST_data/train-labels-idx1-ubyte.gz differ diff --git a/adv_train/mnist/config.json b/adv_train/mnist/config.json new file mode 100644 index 0000000..a4ed331 --- /dev/null +++ b/adv_train/mnist/config.json @@ -0,0 +1,20 @@ +{ + "_comment": "===== TRAINING CONFIGURATION =====", + "random_seed": 4557077, + "max_num_training_steps": 100000, + "num_output_steps": 100, + "num_summary_steps": 100, + "num_checkpoint_steps": 300, + "training_batch_size": 50, + + "_comment": "===== EVAL CONFIGURATION =====", + "num_eval_examples": 10000, + "eval_batch_size": 200, + "eval_on_cpu": true, + + "_comment": "=====ADVERSARIAL EXAMPLES CONFIGURATION=====", + "a": 0.01, + "random_start": true, + "loss_func": "xent", + "store_adv_path": "attack.npy" +} diff --git a/adv_train/mnist/eval.py b/adv_train/mnist/eval.py new file mode 100644 index 0000000..6941e64 --- /dev/null +++ b/adv_train/mnist/eval.py @@ -0,0 +1,166 @@ +""" +The code is adapted from the MadryLab's repo: +https://github.com/MadryLab/mnist_challenge/eval.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import math +import os +import sys +import argparse +import numpy as np + +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data + +from model import Model +from pgd_attack import LinfPGDAttack + + +parser = argparse.ArgumentParser(description='adv_training') +parser.add_argument('--device', type=int, default=0) +parser.add_argument('--cpu', action='store_true') +parser.add_argument('--eps', type=float, default=0.4) +parser.add_argument('--k', type=int, default=100) +parser.add_argument('--restarts', type=int, default=1) +parser.add_argument('--loss', type=str, default="xent", + help="choose loss function, [xent, cw]") +parser.add_argument('--method', type=str, default="spade", + help="choose training method, [spade, random, pgd, clean]") + + +args = parser.parse_args() +print(args) + +os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" +os.environ["CUDA_VISIBLE_DEVICES"]="{}".format(args.device) + +# Global constants +with open('config.json') as config_file: + config = json.load(config_file) +config["epsilon"] = args.eps +config["k"] = args.k +config["loss_func"] = args.loss + +if args.method=="pgd": + config["model_dir"] = "models/pgd_0.4" +elif args.method=="clean": + config["model_dir"] = "models/pgd_0.0" +else: + config["model_dir"] = "models/pgd-{}_0.2_0.4".format(args.method) + +num_eval_examples = config['num_eval_examples'] +eval_batch_size = config['eval_batch_size'] +model_dir = config['model_dir'] +eval_on_cpu = args.cpu + +# Set upd the data, hyperparameters, and the model +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) + +if eval_on_cpu: + with tf.device("/cpu:0"): + model = Model() + attack = LinfPGDAttack(model, + config['k'], + config['a'], + config['random_start'], + config['loss_func']) +else: + model = Model() + attack = LinfPGDAttack(model, + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + +global_step = tf.contrib.framework.get_or_create_global_step() + +# Setting up the Tensorboard and checkpoint outputs +if not os.path.exists(model_dir): + os.makedirs(model_dir) +eval_dir = os.path.join(model_dir, 'eval') +if not os.path.exists(eval_dir): + os.makedirs(eval_dir) + +saver = tf.train.Saver() +summary_writer = tf.summary.FileWriter(eval_dir) + +def evaluate_checkpoint(filename): + with tf.Session() as sess: + # Restore the checkpoint + saver.restore(sess, filename) + + num_batches = int(math.ceil(num_eval_examples / eval_batch_size)) + total_xent_nat = 0. + total_xent_adv = 0. + total_corr_nat = 0 + total_corr_adv = 0 + + for ibatch in range(num_batches): + bstart = ibatch * eval_batch_size + bend = min(bstart + eval_batch_size, num_eval_examples) + + x_batch = mnist.test.images[bstart:bend, :] + y_batch = mnist.test.labels[bstart:bend] + eps_batch = (args.eps * np.ones(bend-bstart)).reshape(-1,1) + + dict_nat = {model.x_input: x_batch, + model.y_input: y_batch} + + best_batch_adv = np.copy(x_batch) + dict_adv = {model.x_input: best_batch_adv, model.y_input: y_batch} + cur_corr, best_loss = sess.run([model.num_correct, model.xent], + feed_dict=dict_adv) + for _ in range(args.restarts): + x_batch_adv = attack.perturb(x_batch, y_batch, sess, eps_batch) + dict_adv = {model.x_input: x_batch_adv, model.y_input: y_batch} + cur_corr, this_loss = sess.run([model.num_correct, model.xent], + feed_dict=dict_adv) + bb = best_loss >= this_loss + bw = best_loss < this_loss + best_batch_adv[bw, :] = x_batch_adv[bw, :] + best_corr, best_loss = sess.run([model.num_correct, model.xent], + feed_dict={model.x_input: best_batch_adv, model.y_input: y_batch}) + + x_batch_adv = np.copy(best_batch_adv) + + dict_adv = {model.x_input: x_batch_adv, + model.y_input: y_batch} + + cur_corr_nat, cur_xent_nat = sess.run( + [model.num_correct,model.xent], + feed_dict = dict_nat) + cur_corr_adv, cur_xent_adv = sess.run( + [model.num_correct,model.xent], + feed_dict = dict_adv) + + total_xent_nat += cur_xent_nat + total_xent_adv += cur_xent_adv + total_corr_nat += cur_corr_nat + total_corr_adv += cur_corr_adv + + avg_xent_nat = total_xent_nat / num_eval_examples + avg_xent_adv = total_xent_adv / num_eval_examples + acc_nat = total_corr_nat / num_eval_examples + acc_adv = total_corr_adv / num_eval_examples + + summary = tf.Summary(value=[ + tf.Summary.Value(tag='xent adv eval', simple_value= avg_xent_adv), + tf.Summary.Value(tag='xent adv', simple_value= avg_xent_adv), + tf.Summary.Value(tag='xent nat', simple_value= avg_xent_nat), + tf.Summary.Value(tag='accuracy adv eval', simple_value= acc_adv), + tf.Summary.Value(tag='accuracy adv', simple_value= acc_adv), + tf.Summary.Value(tag='accuracy nat', simple_value= acc_nat)]) + summary_writer.add_summary(summary, global_step.eval(sess)) + + print('natural: {:.2f}%'.format(100 * acc_nat)) + print('adversarial: {:.2f}%'.format(100 * acc_adv)) + print('avg nat loss: {:.4f}'.format(avg_xent_nat)) + print('avg adv loss: {:.4f}'.format(avg_xent_adv)) + +cur_checkpoint = tf.train.latest_checkpoint(model_dir) +evaluate_checkpoint(cur_checkpoint) diff --git a/adv_train/mnist/fetch_model.py b/adv_train/mnist/fetch_model.py new file mode 100644 index 0000000..afbe7e3 --- /dev/null +++ b/adv_train/mnist/fetch_model.py @@ -0,0 +1,30 @@ +from google_drive_downloader import GoogleDriveDownloader as gdd +import argparse + +parser = argparse.ArgumentParser(description='fetch pretrained model') +parser.add_argument('--method', type=str, default="spade", + help="choose training method, [spade, random, pgd, clean]") + + +args = parser.parse_args() +clean = "1ooDifu31h8zs-2dr9LkBArbRAFpxret-" +pgd = "1kOIbXGO3FCKMUo0kHD4xA9s0TXfocMsq" +spade = "1javi7s8xnVcmaYQ7rB5TAGJewBsZ-xxp" +random = "1FMSRCaMT9KfCS_4ZYquJRvybQZZLqJxX" + +if args.method == "clean": + fid = clean + zip_name = "pgd_0.0.zip" +elif args.method == "pgd": + fid = pgd + zip_name = "pgd_0.4.zip" +elif args.method == "spade": + fid = spade + zip_name = "pgd-spade_0.2_0.4.zip" +elif args.method == "random": + fid = random + zip_name = "pgd-random_0.2_0.4.zip" + +gdd.download_file_from_google_drive(file_id=fid, + dest_path="./models/{}".format(zip_name), + unzip=True) diff --git a/adv_train/mnist/model.py b/adv_train/mnist/model.py new file mode 100644 index 0000000..d29c385 --- /dev/null +++ b/adv_train/mnist/model.py @@ -0,0 +1,79 @@ +""" +The model is adapted from the tensorflow tutorial: +https://www.tensorflow.org/get_started/mnist/pros +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +class Model(object): + def __init__(self): + self.x_input = tf.placeholder(tf.float32, shape = [None, 784]) + self.y_input = tf.placeholder(tf.int64, shape = [None]) + + self.x_image = tf.reshape(self.x_input, [-1, 28, 28, 1]) + + # first convolutional layer + W_conv1 = self._weight_variable([5,5,1,32]) + b_conv1 = self._bias_variable([32]) + + h_conv1 = tf.nn.relu(self._conv2d(self.x_image, W_conv1) + b_conv1) + h_pool1 = self._max_pool_2x2(h_conv1) + + # second convolutional layer + W_conv2 = self._weight_variable([5,5,32,64]) + b_conv2 = self._bias_variable([64]) + + h_conv2 = tf.nn.relu(self._conv2d(h_pool1, W_conv2) + b_conv2) + h_pool2 = self._max_pool_2x2(h_conv2) + + # first fully connected layer + W_fc1 = self._weight_variable([7 * 7 * 64, 1024]) + b_fc1 = self._bias_variable([1024]) + + h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) + h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) + + # output layer + W_fc2 = self._weight_variable([1024,10]) + b_fc2 = self._bias_variable([10]) + + self.pre_softmax = tf.matmul(h_fc1, W_fc2) + b_fc2 + + y_xent = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=self.y_input, logits=self.pre_softmax) + + self.xent = tf.reduce_sum(y_xent) + + self.y_pred = tf.argmax(self.pre_softmax, 1) + + correct_prediction = tf.equal(self.y_pred, self.y_input) + + self.num_correct = tf.reduce_sum(tf.cast(correct_prediction, tf.int64)) + self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + + self.ins = self.x_input + self.outs = self.pre_softmax + + @staticmethod + def _weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + + @staticmethod + def _bias_variable(shape): + initial = tf.constant(0.1, shape = shape) + return tf.Variable(initial) + + @staticmethod + def _conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME') + + @staticmethod + def _max_pool_2x2( x): + return tf.nn.max_pool(x, + ksize = [1,2,2,1], + strides=[1,2,2,1], + padding='SAME') diff --git a/adv_train/mnist/node_spade_score.pkl b/adv_train/mnist/node_spade_score.pkl new file mode 100644 index 0000000..bcccec9 Binary files /dev/null and b/adv_train/mnist/node_spade_score.pkl differ diff --git a/adv_train/mnist/pgd_attack.py b/adv_train/mnist/pgd_attack.py new file mode 100644 index 0000000..315a30b --- /dev/null +++ b/adv_train/mnist/pgd_attack.py @@ -0,0 +1,59 @@ +""" +The code is adapted from the MadryLab's repo: +https://github.com/MadryLab/mnist_challenge/pgd_attack.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +import numpy as np +import sys + + +class LinfPGDAttack: + def __init__(self, model, k, a, random_start, loss_func): + """Attack parameter initialization. The attack performs k steps of + size a, while always staying within epsilon from the initial + point.""" + self.model = model + self.k = k + self.a = a + self.rand = random_start + + if loss_func == 'xent': + loss = model.xent + elif loss_func == 'cw': + label_mask = tf.one_hot(model.y_input, + 10, + on_value=1.0, + off_value=0.0, + dtype=tf.float32) + correct_logit = tf.reduce_sum(label_mask * model.pre_softmax, axis=1) + wrong_logit = tf.reduce_max((1-label_mask) * model.pre_softmax + - 1e4*label_mask, axis=1) + loss = -tf.nn.relu(correct_logit - wrong_logit + 50) + else: + print('Unknown loss function. Defaulting to cross-entropy') + loss = model.xent + + self.grad = tf.gradients(loss, model.x_input)[0] + + def perturb(self, x_nat, y, sess, epsilon): + if self.rand: + x = x_nat + np.random.uniform(-epsilon, epsilon, x_nat.shape) + x = np.clip(x, 0, 1) # ensure valid pixel range + else: + x = np.copy(x_nat) + + for i in range(self.k): + grad = sess.run(self.grad, feed_dict={self.model.x_input: x, + self.model.y_input: y}) + + x += self.a * np.sign(grad) + + x = np.clip(x, x_nat - epsilon, x_nat + epsilon) + x = np.clip(x, 0, 1) # ensure valid pixel range + + return x diff --git a/adv_train/mnist/train.py b/adv_train/mnist/train.py new file mode 100644 index 0000000..26bbe3a --- /dev/null +++ b/adv_train/mnist/train.py @@ -0,0 +1,157 @@ +""" +The code is adapted from the MadryLab's repo: +https://github.com/MadryLab/mnist_challenge/train.py +""" + + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import os +import sys +import math +import pickle +import shutil +import argparse + +import tensorflow as tf +import numpy as np +from numpy.random import permutation +from tensorflow.examples.tutorials.mnist import input_data + +from model import Model +from pgd_attack import LinfPGDAttack + +parser = argparse.ArgumentParser(description='SPADE_adv_training') +parser.add_argument('--device', type=int, default=0) +parser.add_argument('--num_epoch', type=int, default=150) +parser.add_argument('--topk', type=int, default=45000) +parser.add_argument('--k', type=int, default=100) +parser.add_argument('--small_eps', type=float, default=0.2) +parser.add_argument('--eps', type=float, default=0.4) +parser.add_argument('--method', type=str, default="spade", + help="choose training method, [spade, random, pgd]") + + +args = parser.parse_args() +print(args) + +os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" +os.environ["CUDA_VISIBLE_DEVICES"]="{}".format(args.device) +np.random.seed(seed=1234) + +with open('config.json') as config_file: + config = json.load(config_file) + +config["epsilon"] = args.eps +config["k"] = args.k + +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) +x_train = mnist.train.images +y_train = mnist.train.labels + +num_samples = x_train.shape[0] +batch_size = config['training_batch_size'] +num_batches = int(math.ceil(num_samples/batch_size)) + +node_epsilon = args.eps * np.ones(x_train.shape[0]) +if args.method == "pgd": + config["model_dir"] = "models/{}_{}".format(args.method, args.eps) +else: + config["model_dir"] = "models/pgd-{}_{}_{}".format(args.method, args.small_eps, args.eps) + with open("node_spade_score.pkl", 'rb') as fin : + node_score = (pickle.load(fin)).reshape(-1,) + idx = (-node_score).argsort() + if args.method=="random": + np.random.shuffle(idx) + sub_idx = idx[args.topk:] + node_epsilon[sub_idx] = args.small_eps + +x_train = np.concatenate((x_train, node_epsilon.reshape(-1,1)), axis=1) + +# Setting up training parameters +tf.set_random_seed(config['random_seed']) +num_output_steps = config['num_output_steps'] +num_summary_steps = config['num_summary_steps'] +num_checkpoint_steps = config['num_checkpoint_steps'] + +global_step = tf.contrib.framework.get_or_create_global_step() +model = Model() + +# Setting up the optimizer +train_step = tf.train.AdamOptimizer(1e-4).minimize(model.xent, + global_step=global_step) + +# Set up adversary +attack = LinfPGDAttack(model, + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + +# Setting up the Tensorboard and checkpoint outputs +model_dir = config['model_dir'] +if not os.path.exists(model_dir): + os.makedirs(model_dir) + + +saver = tf.train.Saver(max_to_keep=3) +tf.summary.scalar('accuracy adv train', model.accuracy) +tf.summary.scalar('accuracy adv', model.accuracy) +tf.summary.scalar('xent adv train', model.xent / batch_size) +tf.summary.scalar('xent adv', model.xent / batch_size) +tf.summary.image('images adv train', model.x_image) +merged_summaries = tf.summary.merge_all() + +shutil.copy('config.json', model_dir) + +with tf.Session() as sess: + summary_writer = tf.summary.FileWriter(model_dir, sess.graph) + sess.run(tf.global_variables_initializer()) + + # Main training loop + for e in range(args.num_epoch): + perm = permutation(len(x_train)) + x_train = x_train[perm] + y_train = y_train[perm] + for b in range(num_batches): + bstart = b*batch_size + bend = min(bstart+batch_size, num_samples) + + x_eps_batch = x_train[bstart:bend,:] + eps_batch = x_eps_batch[:,-1].reshape(-1,1) + x_batch = x_eps_batch[:,:-1] + y_batch = y_train[bstart:bend] + + # Compute Adversarial Perturbations + x_batch_adv = attack.perturb(x_batch, y_batch, sess, eps_batch) + + nat_dict = {model.x_input: x_batch, + model.y_input: y_batch} + + adv_dict = {model.x_input: x_batch_adv, + model.y_input: y_batch} + + # Output to stdout + if b % num_output_steps == 0: + nat_acc = sess.run(model.accuracy, feed_dict=nat_dict) + adv_acc = sess.run(model.accuracy, feed_dict=adv_dict) + print(f'Epoch {e}, Batch {b}:') + print(' training nat accuracy {:.4}%'.format(nat_acc * 100)) + print(' training adv accuracy {:.4}%'.format(adv_acc * 100)) + + # Tensorboard summaries + if b % num_summary_steps == 0: + summary = sess.run(merged_summaries, feed_dict=adv_dict) + summary_writer.add_summary(summary, global_step.eval(sess)) + + # Write a checkpoint + if b % num_checkpoint_steps == 0: + saver.save(sess, + os.path.join(model_dir, 'checkpoint'), + global_step=global_step) + + # Actual training step + sess.run(train_step, feed_dict=adv_dict) diff --git a/evaluation/CLEVER/COPYRIGHT b/evaluation/CLEVER/COPYRIGHT new file mode 100644 index 0000000..8446fad --- /dev/null +++ b/evaluation/CLEVER/COPYRIGHT @@ -0,0 +1,25 @@ +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Huan Zhang and Lily Weng +Copyright (c) 2016, Nicholas Carlini + +LICENSE + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/evaluation/CLEVER/LICENSE b/evaluation/CLEVER/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/evaluation/CLEVER/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/evaluation/CLEVER/clever.py b/evaluation/CLEVER/clever.py new file mode 100644 index 0000000..0c12168 --- /dev/null +++ b/evaluation/CLEVER/clever.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +clever.py +Compute CLEVER score using collected Lipschitz constants +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" + +import os +import sys +import glob +from functools import partial +from multiprocessing import Pool +import scipy +import scipy.io as sio +from scipy.stats import weibull_min +import scipy.optimize +import numpy as np +import argparse +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +# We observe that the scipy.optimize.fmin optimizer (using Nelder–Mead method) +# sometimes diverges to very large parameters a, b and c. Thus, we add a very +# small regularization to the MLE optimization process to avoid this divergence +def fmin_with_reg(func, x0, args=(), xtol=1e-4, ftol=1e-4, maxiter=None, maxfun=None, + full_output=0, disp=1, retall=0, callback=None, initial_simplex=None, shape_reg = 0.01): + # print('my optimier with shape regularizer = {}'.format(shape_reg)) + def func_with_reg(theta, x): + shape = theta[2] + log_likelyhood = func(theta, x) + reg = shape_reg * shape * shape + # penalize the shape parameter + return log_likelyhood + reg + return scipy.optimize.fmin(func_with_reg, x0, args, xtol, ftol, maxiter, maxfun, + full_output, disp, retall, callback, initial_simplex) + +# fit using weibull_min.fit and run a K-S test +def fit_and_test(rescaled_sample, sample, loc_shift, shape_rescale, optimizer, c_i): + [c, loc, scale] = weibull_min.fit(-rescaled_sample, c_i, optimizer=optimizer) + loc = - loc_shift + loc * shape_rescale + scale *= shape_rescale + ks, pVal = scipy.stats.kstest(-sample, 'weibull_min', args = (c, loc, scale)) + return c, loc, scale, ks, pVal + +def plot_weibull(sample,c,loc,scale,ks,pVal,p,q,figname): + + # compare the sample histogram and fitting result + fig, ax = plt.subplots(1,1) + + x = np.linspace(-1.01*max(sample),-0.99*min(sample),100); + ax.plot(x,weibull_min.pdf(x,c,loc,scale),'r-',label='fitted pdf '+p+'-bnd') + ax.hist(-sample, density=True, bins=20, histtype='stepfilled') + ax.legend(loc='best', frameon=False) + plt.xlabel('-Lips_'+q) + plt.ylabel('pdf') + plt.title('c = {:.2f}, loc = {:.2f}, scale = {:.2f}, ks = {:.2f}, pVal = {:.2f}'.format(c,loc,scale,ks,pVal)) + plt.savefig(figname) + plt.close() + #model = figname.split("_")[1] + #plt.savefig('./debug/'+model+'/'+figname) + #plt.show() # can be used to pause the program + +# We observe than the MLE estimator in scipy sometimes can converge to a bad +# value if the inital shape parameter c is too far from the true value. Thus we +# test a few different initializations and choose the one with best p-value all +# the initializations are tested in parallel; remove some of them to speedup +# computation. +# c_init = [0.01,0.1,0.5,1,5,10,20,50,70,100,200] +c_init = [0.1,1,5,10,20,50,100] + +def get_best_weibull_fit(sample, use_reg = False, shape_reg = 0.01): + + # initialize dictionary to save the fitting results + fitted_paras = {"c":[], "loc":[], "scale": [], "ks": [], "pVal": []} + # reshape the data into a better range + # this helps the MLE solver find the solution easier + loc_shift = np.amax(sample) + dist_range = np.amax(sample) - np.amin(sample) + # if dist_range > 2.5: + shape_rescale = dist_range + # else: + # shape_rescale = 1.0 + print("shape rescale = {}".format(shape_rescale)) + rescaled_sample = np.copy(sample) + rescaled_sample -= loc_shift + rescaled_sample /= shape_rescale + + print("loc_shift = {}".format(loc_shift)) + ##print("rescaled_sample = {}".format(rescaled_sample)) + + # fit weibull distn: sample follows reverse weibull dist, so -sample follows weibull distribution + if use_reg: + results = pool.map(partial(fit_and_test, rescaled_sample, sample, loc_shift, shape_rescale, partial(fmin_with_reg, shape_reg = shape_reg)), c_init) + else: + results = pool.map(partial(fit_and_test, rescaled_sample, sample, loc_shift, shape_rescale, scipy.optimize.fmin), c_init) + + for res, c_i in zip(results, c_init): + c = res[0] + loc = res[1] + scale = res[2] + ks = res[3] + pVal = res[4] + print("[DEBUG][L2] c_init = {:5.5g}, fitted c = {:6.2f}, loc = {:7.2f}, scale = {:7.2f}, ks = {:4.2f}, pVal = {:4.2f}, max = {:7.2f}".format(c_i,c,loc,scale,ks,pVal,loc_shift)) + + ## plot every fitted result + #plot_weibull(sample,c,loc,scale,ks,pVal,p) + + fitted_paras['c'].append(c) + fitted_paras['loc'].append(loc) + fitted_paras['scale'].append(scale) + fitted_paras['ks'].append(ks) + fitted_paras['pVal'].append(pVal) + + + # get the paras of best pVal among c_init + max_pVal = np.nanmax(fitted_paras['pVal']) + if np.isnan(max_pVal) or max_pVal < 0.001: + print("ill-conditioned samples. Using maximum sample value.") + # handle the ill conditioned case + return -1, -1, -max(sample), -1, -1, -1 + + max_pVal_idx = fitted_paras['pVal'].index(max_pVal) + + c_init_best = c_init[max_pVal_idx] + c_best = fitted_paras['c'][max_pVal_idx] + loc_best = fitted_paras['loc'][max_pVal_idx] + scale_best = fitted_paras['scale'][max_pVal_idx] + ks_best = fitted_paras['ks'][max_pVal_idx] + pVal_best = fitted_paras['pVal'][max_pVal_idx] + + return c_init_best, c_best, loc_best, scale_best, ks_best, pVal_best + + +# G_max is the input array of max values +# Return the Weibull position parameter +def get_lipschitz_estimate(G_max, norm = "L2", figname = "", use_reg = False, shape_reg = 0.01): + global plot_res + c_init, c, loc, scale, ks, pVal = get_best_weibull_fit(G_max, use_reg, shape_reg) + + # the norm here is Lipschitz constant norm, not the bound's norm + if norm == "L1": + p = "i"; q = "1" + elif norm == "L2": + p = "2"; q = "2" + elif norm == "Li": + p = "1"; q = "i" + else: + print("Lipschitz norm is not in 1, 2, i!") + + if plot_res is not None: + plot_res.get() + + # plot_weibull(G_max,c,loc,scale,ks,pVal,p,q,figname) + if figname: + figname = figname + '_'+ "L"+ p + ".png" + plot_res = pool.apply_async(plot_weibull, (G_max,c,loc,scale,ks,pVal,p,q,figname)) + + return {'Lips_est':-loc, 'shape':c, 'loc': loc, 'scale': scale, 'ks': ks, 'pVal': pVal} + #return np.max(G_max) + +# file name contains some information, like true_id, true_label and target_label +def parse_filename(filename): + basename = os.path.basename(filename) + name, _ = os.path.splitext(basename) + name_arr = name.split('_') + Nsamp = int(name_arr[0]) + Niters = int(name_arr[1]) + true_id = int(name_arr[2]) + true_label = int(name_arr[3]) + target_label = int(name_arr[4]) + image_info = name_arr[5] + activation = name_arr[6] + order = name_arr[7][-1] + return Nsamp, Niters, true_id, true_label, target_label, image_info, activation, order + +if __name__ == "__main__": + # parse command line parameters + parser = argparse.ArgumentParser(description='Compute CLEVER scores using collected gradient norm data.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('data_folder', help='data folder path') + parser.add_argument('--min', dest='reduce_op', action='store_const', + default=lambda x: sum(x) / len(x) if len(x) > 0 else 0, const=min, + help='report min of all CLEVER scores instead of avg') + parser.add_argument('--user_type', + default="", + help='replace user type with string, used for ImageNet data processing') + parser.add_argument('--use_slope', + action="store_true", + help='report slope estimate. To use this option, collect_gradients.py needs to be run with --compute_slope') + parser.add_argument('--untargeted', + action="store_true", + help='process untargeted attack results (for MNIST and CIFAR)') + parser.add_argument('--num_samples', + type=int, + default=0, + help='the number of samples to use. Default 0 is to use all samples') + parser.add_argument('--num_images', + type=int, + default=0, + help='number of images to use, 0 to use all images') + parser.add_argument('--shape_reg', + default=0.01, + type=float, + help='to avoid the MLE solver in Scipy to diverge, we add a small regularization (default 0.01 is sufficient)') + parser.add_argument('--nthreads', + default=0, + type=int, + help='number of threads (default is len(c_init)+1)') + parser.add_argument('--plot_dir', + default='', + help='output path for weibull fit figures (empty to disable)') + parser.add_argument('--method', + default="mle_reg", + choices=['mle','mle_reg','maxsamp'], + help='Fitting algorithm. Please use mle_reg for best results') + args = vars(parser.parse_args()) + reduce_op = args['reduce_op'] + if args['plot_dir']: + os.system("mkdir -p " + args['plot_dir']) + print(args) + + # create thread pool + if args['nthreads'] == 0: + args['nthreads'] = len(c_init) + 1 + print("using {} threads".format(args['nthreads'])) + pool = Pool(processes = args['nthreads']) + # pool = Pool(1) + # used for asynchronous plotting in background + plot_res = None + + # get a list of all '.mat' files in folder + file_list = glob.glob(args['data_folder'] + '/**/*.mat', recursive = True) + # sort by image ID, then by information (least likely, random, top-2) + file_list = sorted(file_list, key = lambda x: (parse_filename(x)[2], parse_filename(x)[5])) + # get the first num_images files + if args['num_images']: + file_list = file_list[:args['num_images']] + + if args['untargeted']: + bounds = {} + # bounds will be inserted per image + else: + # aggregate information for three different types: least, random and top2 + # each has three bounds: L1, L2, and Linf + bounds = {"least" : [[], [], []], + "random": [[], [], []], + "top2" : [[], [], []]} + + for fname in file_list: + nsamps, niters, true_id, true_label, target_label, img_info, activation, order = parse_filename(fname) + + # keys in mat: + # ['Li_max', 'pred', 'G1_max', 'g_x0', 'path', 'info', 'G2_max', 'true_label', 'args', 'L1_max', 'Gi_max', 'L2_max', 'id', 'target_label'] + mat = sio.loadmat(fname) + print('loading {}'.format(fname)) + + if order == "1" and args['use_slope']: + G1_max = np.squeeze(mat['L1_max']) + G2_max = np.squeeze(mat['L2_max']) + Gi_max = np.squeeze(mat['Li_max']) + elif order == "1": + G1_max = np.squeeze(mat['G1_max']) + G2_max = np.squeeze(mat['G2_max']) + Gi_max = np.squeeze(mat['Gi_max']) + elif order == "2": + """ For Jun 25 experiments: forgot to save g_x0_grad_2_norm, so rerun a 1 sample 1 iterations cases "1_1_*.mat" and load g_x0_grad_2_norm from it + fname_ref = os.path.dirname(fname)+'_1/'+"1_1_"+str(true_id)+"_"+str(true_label)+"_"+str(target_label)+"_"+img_info+"_"+activation+"_order2.mat" + ##fname_ref = 'lipschitz_mat/mnist_normal/'+"1_1_"+str(true_id)+"_"+str(true_label)+"_"+str(target_label)+"_"+img_info+"_"+activation+"_order2.mat" + print("loading {}".format(fname_ref)) + mat_ref = sio.loadmat(fname_ref) + g_x0_grad_2_norm = np.squeeze(mat_ref['g_x0_grad_2_norm']) + print("g_x0_grad_2_norm = {}".format(g_x0_grad_2_norm)) + + #import time + #time.sleep(30) + """ + G2_max = np.abs(np.squeeze(mat['H2_max'])) # forgot to add abs when save in mat file + G1_max = -1*np.empty_like(G2_max) # currently only implemented 2nd order bound for p = 2 + Gi_max = -1*np.empty_like(G2_max) + g_x0_grad_2_norm = np.squeeze(mat['g_x0_grad_2_norm']) + else: + raise RuntimeError('!!! order is {}'.format(order)) + + if args['num_samples'] != 0: + prev_len = len(G1_max) + G1_max = G1_max[:args['num_samples']] + G2_max = G2_max[:args['num_samples']] + Gi_max = Gi_max[:args['num_samples']] + print('Using {} out of {} total samples'.format(len(G1_max), prev_len)) + g_x0 = np.squeeze(mat['g_x0']) + target_label = np.squeeze(mat['target_label']) + true_id = np.squeeze(mat['id']) + true_label = np.squeeze(mat['true_label']) + img_info = mat['info'][0] + if args['user_type'] != "" and img_info == "user": + img_info = args['user_type'] + + # get the filename (.mat) + print('[Filename] {}'.format(fname)) + # get the model name (inception, cifar_2-layer) + possible_names = ["mnist", "cifar", "mobilenet", "inception", "resnet"] + model = "unknown" + for path_seg in args["data_folder"].split("/"): + for n in possible_names: + if n in path_seg: + model = path_seg.replace('_', '-') + break + # model = args["data_folder"].split("/")[1] + + if args['num_samples'] == 0: # default, use all G1_max + figname = 'Fig_'+model+'_'+img_info+'_'+str(true_id)+'_'+str(true_label)+'_'+str(target_label)+'_Nsamp_'+str(len(G1_max)); + elif args['num_samples'] <= len(G1_max) and args['num_samples'] > 0: + figname = 'Fig_'+model+'_'+img_info+'_'+str(true_id)+'_'+str(true_label)+'_'+str(target_label)+'_Nsamp_'+str(args['num_samples']); + else: + print('Warning!! Input arg num_samp = {} exceed len(G1_max) in data_process.py'.format(args['num_samples'])) + continue + + + if args['use_slope']: + figname = figname + '_slope' + + if args['plot_dir']: + figname = os.path.join(args['plot_dir'], figname) + # figname + print('[Figname] {}'.format(figname)) + else: + # disable debugging figure + figname = "" + + + + if args['method'] == "maxsamp": + if order == "1": + Est_G1 = {'Lips_est': max(G1_max), 'shape': -1, 'loc': -1, 'scale': -1, 'ks': -1, 'pVal': -1} + Est_G2 = {'Lips_est': max(G2_max), 'shape': -1, 'loc': -1, 'scale': -1, 'ks': -1, 'pVal': -1} + Est_Gi = {'Lips_est': max(Gi_max), 'shape': -1, 'loc': -1, 'scale': -1, 'ks': -1, 'pVal': -1} + else: # currently only compare bounds in L2 for both order = 1 and order = 2 + Est_G2 = {'Lips_est': max(G2_max), 'shape': -1, 'loc': -1, 'scale': -1, 'ks': -1, 'pVal': -1} + Est_G1 = Est_G2 + Est_Gi = Est_G2 + + elif args['method'] == "mle": + # estimate Lipschitz constant: Est_G1 is a dictionary containing Lips_est and weibull paras + if order == "1": + Est_G1 = get_lipschitz_estimate(G1_max, "L1", figname) + Est_G2 = get_lipschitz_estimate(G2_max, "L2", figname) + Est_Gi = get_lipschitz_estimate(Gi_max, "Li", figname) + else: # currently only compare bounds in L2 for both order = 1 and order = 2 + Est_G2 = get_lipschitz_estimate(G2_max, "L2", figname) + Est_G1 = Est_G2 # haven't implemented + Est_Gi = Est_G2 # haven't implemented + + elif args['method'] == "mle_reg": + + if order == "1": + print('estimating L1...') + Est_G1 = get_lipschitz_estimate(G1_max, "L1", figname, True, args['shape_reg']) + print('estimating L2...') + Est_G2 = get_lipschitz_estimate(G2_max, "L2", figname, True, args['shape_reg']) + print('estimating Li...') + Est_Gi = get_lipschitz_estimate(Gi_max, "Li", figname, True, args['shape_reg']) + else: # currently only compare bounds in L2 for both order = 1 and order = 2 + print('estimating L2...') + Est_G2 = get_lipschitz_estimate(G2_max, "L2", figname, True, args['shape_reg']) + Est_G1 = Est_G2 + Est_Gi = Est_G1 + else: + raise RuntimeError("method not supported") + + # the estimated Lipschitz constant + Lip_G1 = Est_G1['Lips_est'] + Lip_G2 = Est_G2['Lips_est'] + Lip_Gi = Est_Gi['Lips_est'] + + # the estimated shape parameter (c) in Weibull distn + shape_G1 = Est_G1['shape'] + shape_G2 = Est_G2['shape'] + shape_Gi = Est_Gi['shape'] + + # the estimated loc parameters in Weibull distn + loc_G1 = Est_G1['loc'] + loc_G2 = Est_G2['loc'] + loc_Gi = Est_Gi['loc'] + + # the estimated scale parameters in Weibull distn + scale_G1 = Est_G1['scale'] + scale_G2 = Est_G2['scale'] + scale_Gi = Est_Gi['scale'] + + # the computed ks score + ks_G1 = Est_G1['ks'] + ks_G2 = Est_G2['ks'] + ks_Gi = Est_Gi['ks'] + + # the computed pVal + pVal_G1 = Est_G1['pVal'] + pVal_G2 = Est_G2['pVal'] + pVal_Gi = Est_Gi['pVal'] + + + # compute robustness bound + if order == "1": + bnd_L1 = g_x0 / Lip_Gi + bnd_L2 = g_x0 / Lip_G2 + bnd_Li = g_x0 / Lip_G1 + else: + bnd_L2 = (-g_x0_grad_2_norm + np.sqrt(g_x0_grad_2_norm**2+2*g_x0*Lip_G2))/Lip_G2 + bnd_L1 = bnd_L2 # haven't implemented + bnd_Li = bnd_L2 # haven't implemented + # save bound of each image + if args['untargeted']: + true_id = int(true_id) + if true_id not in bounds: + bounds[true_id] = [[], [], []] + bounds[true_id][0].append(bnd_L1) + bounds[true_id][1].append(bnd_L2) + bounds[true_id][2].append(bnd_Li) + else: + bounds[img_info][0].append(bnd_L1) + bounds[img_info][1].append(bnd_L2) + bounds[img_info][2].append(bnd_Li) + + # original data_process mode + #print('[STATS][L1] id = {}, true_label = {}, target_label = {}, info = {}, bnd_L1 = {:.5g}, bnd_L2 = {:.5g}, bnd_Li = {:.5g}'.format(true_id, true_label, target_label, img_info, bnd_L1, bnd_L2, bnd_Li)) + + bndnorm_L1 = "1"; + bndnorm_L2 = "2"; + bndnorm_Li = "i"; + + # if use g_x0 = {:.5g}.format(g_x0), then it will have type error. Not sure why yet. + #print('g_x0 = '+str(g_x0)) + + + if args['method'] == "maxsamp": + if order == "1": + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L1, bnd_L1)) + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L2, bnd_L2)) + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_Li, bnd_Li)) + else: # currently only compare L2 bound + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L2, bnd_L2)) + + elif args['method'] == "mle" or args['method'] == "mle_reg": + if order == "1": + # estimate Lipschitz constant: Est_G1 is a dictionary containing Lips_est and weibull paras + # current debug mode: bound_L1 corresponds to Gi, bound_L2 corresponds to G2, bound_Li corresponds to G1 + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}, ks = {:.5g}, pVal = {:.5g}, shape = {:.5g}, loc = {:.5g}, scale = {:.5g}, g_x0 = {}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L1, bnd_L1, ks_Gi, pVal_Gi, shape_Gi, loc_Gi, scale_Gi, g_x0)) + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}, ks = {:.5g}, pVal = {:.5g}, shape = {:.5g}, loc = {:.5g}, scale = {:.5g}, g_x0 = {}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L2, bnd_L2, ks_G2, pVal_G2, shape_G2, loc_G2, scale_G2, g_x0)) + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}, ks = {:.5g}, pVal = {:.5g}, shape = {:.5g}, loc = {:.5g}, scale = {:.5g}, g_x0 = {}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_Li, bnd_Li, ks_G1, pVal_G1, shape_G1, loc_G1, scale_G1, g_x0)) + else: # currently only compare L2 bound + print('[DEBUG][L1] id = {}, true_label = {}, target_label = {}, info = {}, nsamps = {}, niters = {}, bnd_norm = {}, bnd = {:.5g}, ks = {:.5g}, pVal = {:.5g}, shape = {:.5g}, loc = {:.5g}, scale = {:.5g}, g_x0 = {}'.format(true_id, true_label, target_label, img_info, nsamps, niters, bndnorm_L2, bnd_L2, ks_G2, pVal_G2, shape_G2, loc_G2, scale_G2, g_x0)) + + else: + raise RuntimeError("method not supported") + + sys.stdout.flush() + + if args['untargeted']: + clever_L1s = [] + clever_L2s = [] + clever_Lis = [] + for true_id, true_id_bounds in bounds.items(): + img_clever_L1 = min(true_id_bounds[0]) + img_clever_L2 = min(true_id_bounds[1]) + img_clever_Li = min(true_id_bounds[2]) + n_classes = len(true_id_bounds[0]) + 1 + assert len(true_id_bounds[0]) == len(true_id_bounds[2]) + assert len(true_id_bounds[1]) == len(true_id_bounds[2]) + print('[STATS][L1] image = {:3d}, n_classes = {:3d}, clever_L1 = {:.5g}, clever_L2 = {:.5g}, clever_Li = {:.5g}'.format(true_id, n_classes, img_clever_L1, img_clever_L2, img_clever_Li)) + clever_L1s.append(img_clever_L1) + clever_L2s.append(img_clever_L2) + clever_Lis.append(img_clever_Li) + info = "untargeted" + clever_L1 = reduce_op(clever_L1s) + clever_L2 = reduce_op(clever_L2s) + clever_Li = reduce_op(clever_Lis) + print('[STATS][L0] info = {}, {}_clever_L1 = {:.5g}, {}_clever_L2 = {:.5g}, {}_clever_Li = {:.5g}'.format(info, info, clever_L1, info, clever_L2, info, clever_Li)) + else: + # print min/average bound + for info, info_bounds in bounds.items(): + # reduce each array to a single number (min or avg) + clever_L1 = reduce_op(info_bounds[0]) + clever_L2 = reduce_op(info_bounds[1]) + clever_Li = reduce_op(info_bounds[2]) + if order == "1": + print('[STATS][L0] info = {}, {}_clever_L1 = {:.5g}, {}_clever_L2 = {:.5g}, {}_clever_Li = {:.5g}'.format(info, info, clever_L1, info, clever_L2, info, clever_Li)) + else: # currently only compare L2 bound for both order = 1 and order = 2 + print('[STATS][L0] info = {}, {}_clever_L2 = {:.5g}'.format(info, info, clever_L2)) + + sys.stdout.flush() + + + # shutdown thread pool + pool.close() + pool.join() diff --git a/evaluation/CLEVER/collect_gradients.py b/evaluation/CLEVER/collect_gradients.py new file mode 100644 index 0000000..708947a --- /dev/null +++ b/evaluation/CLEVER/collect_gradients.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +The code is adapted from the Huanzhang's repo: +https://github.com/huanzhang12/CLEVER/blob/master/collect_gradients.py +""" + + +""" +collect_gradients.py + +Front end for collecting maximum gradient norm samples + +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang + +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" + +from __future__ import division + +import numpy as np +import scipy.io as sio +import random +import time +import sys +import os +from functools import partial +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data + +from estimate_gradient_norm import EstimateLipschitz +from utils import generate_data + +if __name__ == "__main__": + + import argparse + parser = argparse.ArgumentParser(description = "Collect gradient norm samples for calculating CLEVER score", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-d", "--dataset", choices=["mnist", "cifar", "imagenet"], default="mnist", help = "choose dataset") + parser.add_argument("-m", "--model_name", default="normal", + help = "Select model. For MNIST and CIFAR, options are: 2-layer (MLP), normal (7-layer CNN), distilled (7-layer CNN with defensive distillation), brelu (7-layer CNN with Bounded ReLU). For ImageNet, available options are: resnet_v2_50, resnet_v2_101, resnet_v2_152, inception_v1, inception_v2, inception_v3, inception_v4, inception_resnet_v2, vgg_16, vgg_19, mobilenet_v1_025, mobilenet_v1_050, mobilenet_v1_100, alexnet, nasnet_large, densenet121_k32, densenet169_k32, densenet161_k48") + parser.add_argument("--activation", type=str, choices=["relu", "tanh", "sigmoid", "elu", "softplus", "hard_sigmoid", "arctan"], help = "activation functions", default="relu") + parser.add_argument("-N", "--Nsamps", type=int, default=1024, help = "number of samples per iterations") + parser.add_argument("-i", "--Niters", type=int, default=500, help = "number of iterations. NITERS maximum gradient norms will be collected. A larger value will give a more accurate estimate") + parser.add_argument("-n", "--numimg", type=int, default=1, help = "number of test images to load from dataset") + parser.add_argument("--ids", default = "", help = "use a filelist of image IDs in CSV file for attack (UNSUPPORTED)") + parser.add_argument("--target_type", type=int, default=0b01111, help = "Binary mask for selecting targeted attack classes. bit0: top-2, bit1: random, bit2: least likely, bit3: use --ids override (UNSUPPORTED), bit4: use all labels (for untargeted)") + parser.add_argument("-f", "--firstimg", type=int, default=0, help = "start from which image in dataset") + parser.add_argument("--transform", default="", help = "input transformation function (defend_reduce, defend_jpeg, defend_png)") + parser.add_argument("--order", type = int, default=1, choices=[1,2], help = "1: first order bound, 2: second order bound for twice differentiable activations") + parser.add_argument("--compute_slope", action='store_true', help = "collect slope estimate") + parser.add_argument("--sample_norm", type=str, choices=["l2", "li", "l1"], help = "norm of sampling ball (l2, l1 or li)", default="l2") + parser.add_argument("--fix_dirty_bug", action='store_true', help = "do not use (UNSUPPORTED)") + parser.add_argument("-b", "--batch_size", type=int, default=0, help = "batch size to run model. 0: use default batch size") + parser.add_argument("--nthreads", type=int, default=0, help = "number of threads for generating random samples in sphere") + parser.add_argument("-s", "--save", default="./lipschitz_mat", help = "results output path") + parser.add_argument("--seed", type=int, default = 1215, help = "random seed") + + # parser.add_argument("--label_path", type=str) + + args = vars(parser.parse_args()) + print(args) + + seed = args['seed'] + Nsamp = args['Nsamps']; + Niters = args['Niters']; + dataset = args['dataset'] + model_name = args['model_name'] + start = args['firstimg'] + numimg = args['numimg'] + save_path = args['save'] + total = 0 + + # label_path = args['label_path'] + + random.seed(seed) + np.random.seed(seed) + + # create output directory + os.system("mkdir -p {}/{}_{}".format(save_path, dataset, model_name)) + + # create a Lipschitz estimator class (initial it early to save multiprocessing memory) + clever_estimator = EstimateLipschitz(sess=None, nthreads=args['nthreads']) + + + import tensorflow as tf + from setup_cifar import CIFAR + from setup_mnist import MNIST + from setup_imagenet import ImageNet + + tf.set_random_seed(seed) + config = tf.ConfigProto() + config.gpu_options.allow_growth=True + with tf.Session(config=config) as sess: + clever_estimator.sess = sess + # returns the input tensor and output prediction vector + img, output = clever_estimator.load_model(dataset, model_name, activation = args['activation'], batch_size = args['batch_size'], compute_slope = args['compute_slope'], order = args['order']) + # load dataset + datasets_loader = {"mnist": MNIST, "cifar": CIFAR, "imagenet": partial(ImageNet, clever_estimator.model.image_size)} + data = datasets_loader[dataset]() + # import the ID lists + + if args['ids']: + import pandas as pd + df = pd.read_csv(args['ids'], sep = ",") + # don't use this + # if args['fix_dirty_bug']: + # df = df[df['new_class'] != df['target']] + ids = list(map(int, df['id'])) + target_type = args['target_type'] + # use the target classes override + if target_type & 0b1000 != 0: + # data.test_labels[] + target_classes = [[np.argmax(data.test_labels[ele])] for ele in df['target']] + # if label_path: + # target_labels = np.loadtxt(label_path) + # target_classes = [[target_labels[ele]] for ele in list(df['target'])] + # else: + # target_classes = [[t] for t in list(df['target'])] + else: + # use generated classes + target_classes = None + else: + ids = None + target_classes = None + target_type = args['target_type'] + + # for prediction + predictor = lambda x: np.squeeze(sess.run(output, feed_dict = {img: x})) + + # gbt custom + if dataset=='mnist' and model_name in ['mnist_0','mnist_03']: + data.test_data = data.test_data+0.5 + data.train_data = data.train_data+0.5 + # generate target images + inputs, targets, true_labels, true_ids, img_info = generate_data(data, samples=numimg, targeted=True, + start=start, predictor=predictor, + random_and_least_likely = True, + ids = ids, target_classes = target_classes, target_type = target_type, + imagenet="imagenet" in dataset, + remove_background_class="imagenet" in dataset and + ("vgg" in model_name or "densenet" in model_name or "alexnet" in model_name)) + + + timestart = time.time() + print("got {} images".format(inputs.shape)) + for i, input_img in enumerate(inputs): + # original_predict = np.squeeze(sess.run(output, feed_dict = {img: [input_img]})) + print("processing image {}".format(i)) + original_predict = predictor([input_img]) + true_label = np.argmax(true_labels[i]) + predicted_label = np.argmax(original_predict) + least_likely_label = np.argmin(original_predict) + original_prob = np.sort(original_predict) + original_class = np.argsort(original_predict) + print("Top-10 classifications:", original_class[-1:-11:-1]) + print("True label:", true_label) + print("Top-10 probabilities/logits:", original_prob[-1:-11:-1]) + print("Most unlikely classifications:", original_class[:10]) + print("Most unlikely probabilities/logits:", original_prob[:10]) + if true_label != predicted_label: + print("[WARNING] This image is classfied wrongly by the classifier! Skipping!") + continue + total += 1 + # set target class + target_label = np.argmax(targets[i]); + print('Target class: ', target_label) + sys.stdout.flush() + + if args['order'] == 1: + [L2_max,L1_max,Li_max,G2_max,G1_max,Gi_max,g_x0,pred] = clever_estimator.estimate(input_img, true_label, target_label, Nsamp, Niters, args['sample_norm'], args['transform'], args['order']) + print("[STATS][L1] total = {}, seq = {}, id = {}, time = {:.3f}, true_class = {}, target_class = {}, info = {}".format(total, i, true_ids[i], time.time() - timestart, true_label, target_label, img_info[i])) + # save to sampling results to matlab ;) + mat_path = "{}/{}_{}/{}_{}_{}_{}_{}_{}_{}_order{}.mat".format(save_path, dataset, model_name, Nsamp, Niters, true_ids[i], true_label, target_label, img_info[i], args['activation'], args['order']) + save_dict = {'L2_max': L2_max, 'L1_max': L1_max, 'Li_max': Li_max, 'G2_max': G2_max, 'G1_max': G1_max, 'Gi_max': Gi_max, 'pred': pred, 'g_x0': g_x0, 'id': true_ids[i], 'true_label': true_label, 'target_label': target_label, 'info':img_info[i], 'args': args, 'path': mat_path} + sio.savemat(mat_path, save_dict) + print('saved to', mat_path) + sys.stdout.flush() + elif args['order'] == 2: + [H2_max,g_x0,g_x0_grad_2_norm,g_x0_grad_1_norm,g_x0_grad_inf_norm,pred] = clever_estimator.estimate(input_img, true_label, target_label, Nsamp, Niters, args['sample_norm'], args['transform'], args['order']) + print("[STATS][L1] total = {}, seq = {}, id = {}, time = {:.3f}, true_class = {}, target_class = {}, info = {}".format(total, i, true_ids[i], time.time() - timestart, true_label, target_label, img_info[i])) + # save to sampling results to matlab ;) + mat_path = "{}/{}_{}/{}_{}_{}_{}_{}_{}_{}_order{}.mat".format(save_path, dataset, model_name, Nsamp, Niters, true_ids[i], true_label, target_label, img_info[i], args['activation'], args['order']) + save_dict = {'H2_max': H2_max, 'pred': pred, 'g_x0': g_x0, 'id': true_ids[i], 'true_label': true_label, 'target_label': target_label, 'info':img_info[i], 'args': args, 'path': mat_path, 'g_x0_grad_2_norm': g_x0_grad_2_norm} + sio.savemat(mat_path, save_dict) + print('saved to', mat_path) + sys.stdout.flush() + diff --git a/evaluation/CLEVER/data/t10k-images-idx3-ubyte.gz b/evaluation/CLEVER/data/t10k-images-idx3-ubyte.gz new file mode 100644 index 0000000..5ace8ea Binary files /dev/null and b/evaluation/CLEVER/data/t10k-images-idx3-ubyte.gz differ diff --git a/evaluation/CLEVER/data/t10k-labels-idx1-ubyte.gz b/evaluation/CLEVER/data/t10k-labels-idx1-ubyte.gz new file mode 100644 index 0000000..a7e1415 Binary files /dev/null and b/evaluation/CLEVER/data/t10k-labels-idx1-ubyte.gz differ diff --git a/evaluation/CLEVER/data/train-images-idx3-ubyte.gz b/evaluation/CLEVER/data/train-images-idx3-ubyte.gz new file mode 100644 index 0000000..b50e4b6 Binary files /dev/null and b/evaluation/CLEVER/data/train-images-idx3-ubyte.gz differ diff --git a/evaluation/CLEVER/data/train-labels-idx1-ubyte.gz b/evaluation/CLEVER/data/train-labels-idx1-ubyte.gz new file mode 100644 index 0000000..707a576 Binary files /dev/null and b/evaluation/CLEVER/data/train-labels-idx1-ubyte.gz differ diff --git a/evaluation/CLEVER/defense.py b/evaluation/CLEVER/defense.py new file mode 100644 index 0000000..25eef20 --- /dev/null +++ b/evaluation/CLEVER/defense.py @@ -0,0 +1,323 @@ +import PIL +import PIL.Image +from io import BytesIO +import numpy as np +import tensorflow as tf +from numba import njit + +def defend_reduce(arr, depth=3): + arr = (arr * 255.0).astype(np.uint8) + shift = 8 - depth + arr = (arr >> shift) << shift + arr = arr.astype(np.float32)/255.0 + return arr + +def defend_jpeg(input_array): + pil_image = PIL.Image.fromarray((input_array*255.0).astype(np.uint8)) + f = BytesIO() + pil_image.save(f, format='jpeg', quality=75) # quality level specified in paper + jpeg_image = np.asarray(PIL.Image.open(f)).astype(np.float32)/255.0 + return jpeg_image + +def defend_png(input_array): + pil_image = PIL.Image.fromarray((np.around(input_array*255.0)).astype(np.uint8)) + f = BytesIO() + pil_image.save(f, format='png') + png_image = np.asarray(PIL.Image.open(f)).astype(np.float32)/255.0 + return png_image + +# for testing implementation +def defend_none(input_array): + # this convertion should be lossless + pil_image = PIL.Image.fromarray(input_array) + ret_image = np.asarray(pil_image) + return ret_image + +# based on https://github.com/scikit-image/scikit-image/blob/master/skimage/restoration/_denoise_cy.pyx + +# super slow since this is implemented in pure python :'( + +@njit +def bregman(image, mask, weight, eps=1e-3, max_iter=100): + rows, cols, dims = image.shape + rows2 = rows + 2 + cols2 = cols + 2 + total = rows * cols * dims + shape_ext = (rows2, cols2, dims) + + u = np.zeros(shape_ext) + dx = np.zeros(shape_ext) + dy = np.zeros(shape_ext) + bx = np.zeros(shape_ext) + by = np.zeros(shape_ext) + + u[1:-1, 1:-1] = image + # reflect image + u[0, 1:-1] = image[1, :] + u[1:-1, 0] = image[:, 1] + u[-1, 1:-1] = image[-2, :] + u[1:-1, -1] = image[:, -2] + + i = 0 + rmse = np.inf + lam = 2 * weight + norm = (weight + 4 * lam) + + while i < max_iter and rmse > eps: + rmse = 0 + + for k in range(dims): + for r in range(1, rows+1): + for c in range(1, cols+1): + uprev = u[r, c, k] + + # forward derivatives + ux = u[r, c+1, k] - uprev + uy = u[r+1, c, k] - uprev + + # Gauss-Seidel method + if mask[r-1, c-1]: + unew = (lam * (u[r+1, c, k] + + u[r-1, c, k] + + u[r, c+1, k] + + u[r, c-1, k] + + dx[r, c-1, k] - + dx[r, c, k] + + dy[r-1, c, k] - + dy[r, c, k] - + bx[r, c-1, k] + + bx[r, c, k] - + by[r-1, c, k] + + by[r, c, k] + ) + weight * image[r-1, c-1, k] + ) / norm + else: + # similar to the update step above, except we take + # lim_{weight->0} of the update step, effectively + # ignoring the l2 loss + unew = (u[r+1, c, k] + + u[r-1, c, k] + + u[r, c+1, k] + + u[r, c-1, k] + + dx[r, c-1, k] - + dx[r, c, k] + + dy[r-1, c, k] - + dy[r, c, k] - + bx[r, c-1, k] + + bx[r, c, k] - + by[r-1, c, k] + + by[r, c, k] + ) / 4.0 + u[r, c, k] = unew + + # update rms error + rmse += (unew - uprev)**2 + + bxx = bx[r, c, k] + byy = by[r, c, k] + + # d_subproblem + s = ux + bxx + if s > 1/lam: + dxx = s - 1/lam + elif s < -1/lam: + dxx = s + 1/lam + else: + dxx = 0 + s = uy + byy + if s > 1/lam: + dyy = s - 1/lam + elif s < -1/lam: + dyy = s + 1/lam + else: + dyy = 0 + + dx[r, c, k] = dxx + dy[r, c, k] = dyy + + bx[r, c, k] += ux - dxx + by[r, c, k] += uy - dyy + + rmse = np.sqrt(rmse / total) + i += 1 + + # return np.squeeze(np.asarray(u[1:-1, 1:-1])) + return u[1:-1, 1:-1] + +@njit +def defend_tv(input_array, keep_prob=0.5, lambda_tv=0.03): + mask = np.random.uniform(0.0, 1.0, size=input_array.shape[:2]) + mask = mask < keep_prob + return bregman(input_array, mask, weight=2.0/lambda_tv) + +def make_defend_quilt(sess): + # setup for quilting + quilt_db = np.load('data/quilt_db.npy') + quilt_db_reshaped = quilt_db.reshape(1000000, -1) + TILE_SIZE = 5 + TILE_OVERLAP = 2 + tile_skip = TILE_SIZE - TILE_OVERLAP + K = 10 + db_tensor = tf.placeholder(tf.float32, quilt_db_reshaped.shape) + query_imgs = tf.placeholder(tf.float32, (TILE_SIZE * TILE_SIZE * 3, None)) + norms = tf.reduce_sum(tf.square(db_tensor), axis=1)[:, tf.newaxis] \ + - 2*tf.matmul(db_tensor, query_imgs) + _, topk_indices = tf.nn.top_k(-tf.transpose(norms), k=K, sorted=False) + def min_error_table(arr, direction): + assert direction in ('horizontal', 'vertical') + y, x = arr.shape + cum = np.zeros_like(arr) + if direction == 'horizontal': + cum[:, -1] = arr[:, -1] + for ix in range(x-2, -1, -1): + for iy in range(y): + m = arr[iy, ix+1] + if iy > 0: + m = min(m, arr[iy-1, ix+1]) + if iy < y - 1: + m = min(m, arr[iy+1, ix+1]) + cum[iy, ix] = arr[iy, ix] + m + elif direction == 'vertical': + cum[-1, :] = arr[-1, :] + for iy in range(y-2, -1, -1): + for ix in range(x): + m = arr[iy+1, ix] + if ix > 0: + m = min(m, arr[iy+1, ix-1]) + if ix < x - 1: + m = min(m, arr[iy+1, ix+1]) + cum[iy, ix] = arr[iy, ix] + m + return cum + def index_exists(arr, index): + if arr.ndim != len(index): + return False + return all(i > 0 for i in index) and all(index[i] < arr.shape[i] for i in range(arr.ndim)) + def assign_block(ix, iy, tile, synth): + posx = tile_skip * ix + posy = tile_skip * iy + + if ix == 0 and iy == 0: + synth[posy:posy+TILE_SIZE, posx:posx+TILE_SIZE, :] = tile + elif iy == 0: + # first row, only have horizontal overlap of the block + tile_left = tile[:, :TILE_OVERLAP, :] + synth_right = synth[:TILE_SIZE, posx:posx+TILE_OVERLAP, :] + errors = np.sum(np.square(tile_left - synth_right), axis=2) + table = min_error_table(errors, direction='vertical') + # copy row by row into synth + xoff = np.argmin(table[0, :]) + synth[posy, posx+xoff:posx+TILE_SIZE] = tile[0, xoff:] + for yoff in range(1, TILE_SIZE): + # explore nearby xoffs + candidates = [(yoff, xoff), (yoff, xoff-1), (yoff, xoff+1)] + index = min((i for i in candidates if index_exists(table, i)), key=lambda i: table[i]) + xoff = index[1] + synth[posy+yoff, posx+xoff:posx+TILE_SIZE] = tile[yoff, xoff:] + elif ix == 0: + # first column, only have vertical overlap of the block + tile_up = tile[:TILE_OVERLAP, :, :] + synth_bottom = synth[posy:posy+TILE_OVERLAP, :TILE_SIZE, :] + errors = np.sum(np.square(tile_up - synth_bottom), axis=2) + table = min_error_table(errors, direction='horizontal') + # copy column by column into synth + yoff = np.argmin(table[:, 0]) + synth[posy+yoff:posy+TILE_SIZE, posx] = tile[yoff:, 0] + for xoff in range(1, TILE_SIZE): + # explore nearby yoffs + candidates = [(yoff, xoff), (yoff-1, xoff), (yoff+1, xoff)] + index = min((i for i in candidates if index_exists(table, i)), key=lambda i: table[i]) + yoff = index[0] + synth[posy+yoff:posy+TILE_SIZE, posx+xoff] = tile[yoff:, xoff] + else: + # glue cuts along diagonal + tile_up = tile[:TILE_OVERLAP, :, :] + synth_bottom = synth[posy:posy+TILE_OVERLAP, :TILE_SIZE, :] + errors_up = np.sum(np.square(tile_up - synth_bottom), axis=2) + table_up = min_error_table(errors_up, direction='horizontal') + tile_left = tile[:, :TILE_OVERLAP, :] + synth_right = synth[:TILE_SIZE, posx:posx+TILE_OVERLAP, :] + errors_left = np.sum(np.square(tile_left - synth_right), axis=2) + table_left = min_error_table(errors_left, direction='vertical') + glue_index = -1 + glue_value = np.inf + for i in range(TILE_OVERLAP): + e = table_up[i, i] + table_left[i, i] + if e < glue_value: + glue_value = e + glue_index = i + # copy left part first, up to the overlap column + xoff = glue_index + synth[posy+glue_index, posx+xoff:posx+TILE_OVERLAP] = tile[glue_index, xoff:TILE_OVERLAP] + for yoff in range(glue_index+1, TILE_SIZE): + # explore nearby xoffs + candidates = [(yoff, xoff), (yoff, xoff-1), (yoff, xoff+1)] + index = min((i for i in candidates if index_exists(table_left, i)), key=lambda i: table_left[i]) + xoff = index[1] + synth[posy+yoff, posx+xoff:posx+TILE_OVERLAP] = tile[yoff, xoff:TILE_OVERLAP] + # copy right part, down to overlap row + yoff = glue_index + synth[posy+yoff:posy+TILE_OVERLAP, posx+glue_index] = tile[yoff:TILE_OVERLAP, glue_index] + for xoff in range(glue_index+1, TILE_SIZE): + # explore nearby yoffs + candidates = [(yoff, xoff), (yoff-1, xoff), (yoff+1, xoff)] + index = min((i for i in candidates if index_exists(table_up, i)), key=lambda i: table_up[i]) + yoff = index[0] + synth[posy+yoff:posy+TILE_OVERLAP, posx+xoff] = tile[yoff:TILE_OVERLAP, xoff] + # copy rest of image + synth[posy+TILE_OVERLAP:posy+TILE_SIZE, posx+TILE_OVERLAP:posx+TILE_SIZE] = tile[TILE_OVERLAP:, TILE_OVERLAP:] + KNN_MAX_BATCH = 1000 + def quilt(arr, graphcut=True): + h, w, c = arr.shape + assert (h - TILE_SIZE) % tile_skip == 0 + assert (w - TILE_SIZE) % tile_skip == 0 + horiz_blocks = (w - TILE_SIZE) // tile_skip + 1 + vert_blocks = (h - TILE_SIZE) // tile_skip + 1 + num_patches = horiz_blocks * vert_blocks + patches = np.zeros((TILE_SIZE * TILE_SIZE * 3, num_patches)) + idx = 0 + for iy in range(vert_blocks): + for ix in range(horiz_blocks): + posx = tile_skip*ix + posy = tile_skip*iy + patches[:, idx] = arr[posy:posy+TILE_SIZE, posx:posx+TILE_SIZE, :].ravel() + idx += 1 + + ind = [] + for chunk in range(num_patches // KNN_MAX_BATCH + (1 if num_patches % KNN_MAX_BATCH != 0 else 0)): + start = KNN_MAX_BATCH * chunk + end = start + KNN_MAX_BATCH + # for some reason, the code below is 10x slower when run in a Jupyter notebook + # not sure why... + indices_ = sess.run(topk_indices, {db_tensor: quilt_db_reshaped, query_imgs: patches[:, start:end]}) + for i in indices_: + ind.append(np.random.choice(i)) + + synth = np.zeros((299, 299, 3)) + + idx = 0 + for iy in range(vert_blocks): + for ix in range(horiz_blocks): + posx = tile_skip*ix + posy = tile_skip*iy + tile = quilt_db[ind[idx]] + if not graphcut: + synth[posy:posy+TILE_SIZE, posx:posx+TILE_SIZE, :] = tile + else: + assign_block(ix, iy, tile, synth) + idx += 1 + return synth + + return quilt + +# x is a square image (3-tensor) +def defend_crop(x, crop_size=90, ensemble_size=30): + x_size = tf.to_float(x.shape[1]) + frac = crop_size/x_size + start_fraction_max = (x_size - crop_size)/x_size + def randomizing_crop(x): + start_x = tf.random_uniform((), 0, start_fraction_max) + start_y = tf.random_uniform((), 0, start_fraction_max) + return tf.image.crop_and_resize([x], boxes=[[start_y, start_x, start_y+frac, start_x+frac]], + box_ind=[0], crop_size=[crop_size, crop_size]) + + return tf.concat([randomizing_crop(x) for _ in range(ensemble_size)], axis=0) diff --git a/evaluation/CLEVER/estimate_gradient_norm.py b/evaluation/CLEVER/estimate_gradient_norm.py new file mode 100644 index 0000000..a407517 --- /dev/null +++ b/evaluation/CLEVER/estimate_gradient_norm.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +The code is adapted from the Huanzhang's repo: +https://github.com/huanzhang12/CLEVER/blob/master/estimate_gradient_norm.py +""" + + + +""" +estimate_gradient_norm.py + +A multithreaded gradient norm sampler + +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang + +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" + +from __future__ import division + +import numpy as np +import random +import ctypes +import time +import sys +import os +import tensorflow as tf + +from multiprocessing import Pool, current_process, cpu_count +from shmemarray import ShmemRawArray, NpShmemArray +from functools import partial +from randsphere import randsphere +from tensorflow.python.ops import gradients_impl + + +class EstimateLipschitz(object): + + def __init__(self, sess, seed = 1215, nthreads = 0): + """ + sess: tensorflow session + Nsamp: number of samples to take per iteration + Niters: number of iterations, each iteration we return a max L + """ + self.sess = sess + self.seed = seed + # create a pool of workers to compute samples in advance + if nthreads == 0: + self.n_processes = max(cpu_count() // 2, 1) + else: + self.n_processes = nthreads + # set up random seed during initialization + def initializer(s): + np.random.seed(s + current_process()._identity[0]) + # using only 1 OpenMP thread + os.environ['OMP_NUM_THREADS'] = "1" + self.pool = Pool(processes = self.n_processes, initializer = initializer, initargs=(self.seed,)) + + def load_model(self, dataset = "mnist", model_name = "2-layer", activation = "relu", model = None, batch_size = 0, compute_slope = False, order = 1): + """ + model: if set to None, then load dataset with model_name. Otherwise use the model directly. + dataset: mnist, cifar and imagenet. recommend to use mnist and cifar as a starting point. + model_name: possible options are 2-layer, distilled, and normal + """ + from setup_cifar import CIFAR, CIFARModel, TwoLayerCIFARModel + from setup_mnist import MNIST, MNISTModel, TwoLayerMNISTModel, mnist_0, mnist_03 + from nlayer_model import NLayerModel + from setup_imagenet import ImageNet, ImageNetModel + + # if set this to true, we will use the logit layer output instead of probability + # the logit layer's gradients are usually larger and more stable + output_logits = True + self.dataset = dataset + self.model_name = model_name + + if model is None: + print('Loading model...') + if dataset == "mnist": + self.batch_size = 1024 + if model_name == "2-layer": + model = TwoLayerMNISTModel("models/mnist_2layer", self.sess, not output_logits) + + elif model_name == "mnist_0": + if activation == "relu": + model = mnist_0("models/mnist-0", self.sess, not output_logits) + + elif model_name == "mnist_03": + if activation == "relu": + model = mnist_03("models/mnist-0.3", self.sess, not output_logits) + + elif model_name == "normal": + if activation == "relu": + model = MNISTModel("models/mnist", self.sess, not output_logits) + else: + print("actviation = {}".format(activation)) + model = MNISTModel("models/mnist_cnn_7layer_"+activation, self.sess, not output_logits, activation= activation) + time.sleep(5) + + elif model_name == "brelu": + model = MNISTModel("models/mnist_brelu", self.sess, not output_logits, use_brelu = True) + elif model_name == "distilled": + model = MNISTModel("models/mnist-distilled-100", self.sess, not output_logits) + else: + # specify model parameters as N,M,opts + model_params = model_name.split(",") + if len(model_params) < 3: + raise(RuntimeError("incorrect model option" + model_name)) + numlayer = int(model_params[0]) + nhidden = int(model_params[1]) + modelfile = "models/mnist_{}layer_relu_{}_{}".format(numlayer, nhidden, model_params[2]) + print("loading", modelfile) + model = NLayerModel([nhidden] * (numlayer - 1), modelfile) + elif dataset == "cifar": + self.batch_size = 1024 + if model_name == "2-layer": + model = TwoLayerCIFARModel("models/cifar_2layer", self.sess, not output_logits) + elif model_name == "normal": + if activation == "relu": + model = CIFARModel("models/cifar", self.sess, not output_logits) + else: + model = CIFARModel("models/cifar_cnn_7layer_"+activation, self.sess, not output_logits, activation = activation) + elif model_name == "brelu": + model = CIFARModel("models/cifar_brelu", self.sess, not output_logits, use_brelu = True) + elif model_name == "distilled": + model = CIFARModel("models/cifar-distilled-100", self.sess, not output_logits) + else: + # specify model parameters as N,M,opts + model_params = model_name.split(",") + if len(model_params) < 3: + raise(RuntimeError("incorrect model option" + model_name)) + numlayer = int(model_params[0]) + nhidden = int(model_params[1]) + modelfile = "models/cifar_{}layer_relu_{}_{}".format(numlayer, nhidden, model_params[2]) + print("loading", modelfile) + model = NLayerModel([nhidden] * (numlayer - 1), modelfile, image_size=32, image_channel=3) + elif dataset == "imagenet": + self.batch_size = 32 + model = ImageNetModel(self.sess, use_softmax = not output_logits, model_name = model_name, create_prediction = False) + else: + raise(RuntimeError("dataset unknown")) + + #print("*** Loaded model successfully") + + self.model = model + self.compute_slope = compute_slope + if batch_size != 0: + self.batch_size = batch_size + + ## placeholders: self.img, self.true_label, self.target_label + # img is the placeholder for image input + self.img = tf.placeholder(shape = [None, model.image_size, model.image_size, model.num_channels], dtype = tf.float32) + # output is the output tensor of the entire network + self.output = model.predict(self.img) + # create the graph to compute gradient + # get the desired true label and target label + self.true_label = tf.placeholder(dtype = tf.int32, shape = []) + self.target_label = tf.placeholder(dtype = tf.int32, shape = []) + true_output = self.output[:, self.true_label] + target_output = self.output[:, self.target_label] + # get the difference + self.objective = true_output - target_output + # get the gradient(deprecated arguments) + self.grad_op = tf.gradients(self.objective, self.img)[0] + # compute gradient norm: (in computation graph, so is faster) + grad_op_rs = tf.reshape(self.grad_op, (tf.shape(self.grad_op)[0], -1)) + self.grad_2_norm_op = tf.norm(grad_op_rs, axis = 1) + self.grad_1_norm_op = tf.norm(grad_op_rs, ord=1, axis = 1) + self.grad_inf_norm_op = tf.norm(grad_op_rs, ord=np.inf, axis = 1) + + ### Lily: added Hessian-vector product calculation here for 2nd order bound: + if order == 2: + ## _hessian_vector_product(ys, xs, v): return a list of tensors containing the product between the Hessian and v + ## ys: a scalar valur or a tensor or a list of tensors to be summed to yield of scalar + ## xs: a list of tensors that we should construct the Hessian over + ## v: a list of tensors with the same shape as xs that we want to multiply by the Hessian + # self.randv: shape = (Nimg,28,28,1) (the v in _hessian_vector_product) + self.randv = tf.placeholder(shape = [None, model.image_size, model.image_size, model.num_channels], dtype = tf.float32) + # hv_op_tmp: shape = (Nimg,28,28,1) for mnist, same as self.img (the xs in _hessian_vector_product) + hv_op_tmp = gradients_impl._hessian_vector_product(self.objective, [self.img], [self.randv])[0] + # hv_op_rs: reshape hv_op_tmp to hv_op_rs whose shape = (Nimg, 784) for mnist + hv_op_rs = tf.reshape(hv_op_tmp, (tf.shape(hv_op_tmp)[0],-1)) + # self.hv_norm_op: norm of hessian vector product, keep shape = (Nimg,1) using keepdims + self.hv_norm_op = tf.norm(hv_op_rs, axis = 1, keepdims=True) + # hv_op_rs_normalize: normalize Hv to Hv/||Hv||, shape = (Nimg, 784) + hv_op_rs_normalize = hv_op_rs/self.hv_norm_op + # self.hv_op: reshape hv_op_rs_normalize to shape = (Nimg,28,28,1) + self.hv_op = tf.reshape(hv_op_rs_normalize, tf.shape(hv_op_tmp)) + + ## reshape randv and compute its norm + # shape: (Nimg, 784) + randv_rs = tf.reshape(self.randv, (tf.shape(self.randv)[0],-1)) + # shape: (Nimg,) + self.randv_norm_op = tf.norm(randv_rs, axis = 1) + ## compute v'Hv: use un-normalized Hv (hv_op_tmp, hv_op_rs) + # element-wise multiplication and then sum over axis = 1 (now shape: (Nimg,)) + self.vhv_op = tf.reduce_sum(tf.multiply(randv_rs,hv_op_rs),axis=1) + ## compute Rayleigh quotient: v'Hv/v'v (estimated largest eigenvalue), shape: (Nimg,) + # note: self.vhv_op and self.randv_norm_op has to be in the same dimension (either (Nimg,) or (Nimg,1)) + self.eig_est = self.vhv_op/tf.square(self.randv_norm_op) + + ## Lily added the tf.while to compute the eigenvalue in computational graph later + # cond for computing largest abs/neg eigen-value + def cond(it, randv, eig_est, eig_est_prev, tfconst): + norm_diff = tf.norm(eig_est-eig_est_prev,axis=0) + return tf.logical_and(it < 500, norm_diff > 0.001) + + # compute largest abs eigenvalue: tfconst = 0 + # compute largest neg eigenvalue: tfconst = 10 + def body(it, randv, eig_est, eig_est_prev, tfconst): + #hv_op_tmp = gradients_impl._hessian_vector_product(self.objective, [self.img], [randv])[0]-10*randv + hv_op_tmp = gradients_impl._hessian_vector_product(self.objective, [self.img], [randv])[0]-tf.multiply(tfconst,randv) + hv_op_rs = tf.reshape(hv_op_tmp, (tf.shape(hv_op_tmp)[0],-1)) + hv_norm_op = tf.norm(hv_op_rs, axis = 1, keepdims=True) + hv_op_rs_normalize = hv_op_rs/hv_norm_op + hv_op = tf.reshape(hv_op_rs_normalize, tf.shape(hv_op_tmp)) + + randv_rs = tf.reshape(randv, (tf.shape(randv)[0],-1)) + randv_norm_op = tf.norm(randv_rs, axis = 1) + vhv_op = tf.reduce_sum(tf.multiply(randv_rs,hv_op_rs),axis=1) + eig_est_prev = eig_est + eig_est = vhv_op/tf.square(randv_norm_op) + + return (it+1, hv_op, eig_est, eig_est_prev, tfconst) + + it = tf.constant(0) + # compute largest abs eigenvalue + result = tf.while_loop(cond, body, [it, self.randv, self.vhv_op, self.eig_est, tf.constant(0.0)]) + # compute largest neg eigenvalue + self.shiftconst = tf.placeholder(shape = (), dtype = tf.float32) + result_1 = tf.while_loop(cond, body, [it, self.randv, self.vhv_op, self.eig_est, self.shiftconst]) + + # computing largest abs eig value and save result + self.it = result[0] + self.while_hv_op = result[1] + self.while_eig = result[2] + + # computing largest neg eig value and save result + self.it_1 = result_1[0] + #self.while_eig_1 = tf.add(result_1[2], tfconst) + self.while_eig_1 = tf.add(result_1[2], result_1[4]) + + show_tensor_op = False + if show_tensor_op: + print("====================") + print("Define hessian_vector_product operator: ") + print("hv_op_tmp = {}".format(hv_op_tmp)) + print("hv_op_rs = {}".format(hv_op_rs)) + print("self.hv_norm_op = {}".format(self.hv_norm_op)) + print("hv_op_rs_normalize = {}".format(hv_op_rs_normalize)) + print("self.hv_op = {}".format(self.hv_op)) + print("self.grad_op = {}".format(self.grad_op)) + print("randv_rs = {}".format(randv_rs)) + print("self.randv_norm_op = {}".format(self.randv_norm_op)) + print("self.vhv_op = {}".format(self.vhv_op)) + print("self.eig_est = {}".format(self.eig_est)) + print("====================") + + return self.img, self.output + + + def _estimate_Lipschitz_multiplerun(self, num, niters, input_image, target_label, true_label, sample_norm = "l2", transform=None, order = 1): + """ + num: number of samples per iteration + niters: number of iterations + input_image: original image (h*w*c) + """ + batch_size = self.batch_size + shape = (batch_size, self.model.image_size, self.model.image_size, self.model.num_channels) + dimension = self.model.image_size * self.model.image_size * self.model.num_channels + + if num < batch_size: + print("Increasing num to", batch_size) + num = batch_size + + """ + 1. Compute input_image related quantities: + """ + # get the original prediction and gradient, gradient norms values on input image: + pred, grad_val, grad_2_norm_val, grad_1_norm_val, grad_inf_norm_val = self.sess.run( + [self.output, self.grad_op, self.grad_2_norm_op, self.grad_1_norm_op, self.grad_inf_norm_op], + feed_dict = {self.img: [input_image], self.true_label: true_label, self.target_label: target_label}) + pred = np.squeeze(pred) + # print(pred) + # print(grad_val) + + # class c and class j in Hein's paper. c is original class + c = true_label + j = target_label + # get g_x0 = f_c(x_0) - f_j(x_0) + g_x0 = pred[c] - pred[j] + # grad_z_norm should be scalar + g_x0_grad_2_norm = np.squeeze(grad_2_norm_val) + g_x0_grad_1_norm = np.squeeze(grad_1_norm_val) + g_x0_grad_inf_norm = np.squeeze(grad_inf_norm_val) + + print("** Evaluating g_x0, grad_2_norm_val on the input image x0: ") + print("shape of input_image = {}".format(input_image.shape)) + print("g_x0 = {:.3f}, grad_2_norm_val = {:3f}, grad_1_norm_val = {:.3f}, grad_inf_norm_val = {:3f}".format(g_x0, g_x0_grad_2_norm, g_x0_grad_1_norm, g_x0_grad_inf_norm)) + + ##### Lily ##### + if order == 2: # evaluate the hv and hv norm on input_image + # set randv as a random matrix with the same shape as input_image + print("** Evaluating hv and hv_norm on the input image x0:") + randv = np.random.randn(*input_image.shape) + hv, hv_norm = self.sess.run([self.hv_op, self.hv_norm_op], + feed_dict = {self.img: [input_image], self.randv:[randv], self.true_label: true_label, self.target_label: target_label}) + print("hv shape = {}, hv_norm = {}".format(hv.shape, hv_norm)) + + + """ + 2. Prepare for sampling: + """ + def div_work_to_cores(njobs, nprocs): + process_item_list = [] + while njobs > 0: + process_item_list.append(int(np.ceil(njobs / float(nprocs)))) + njobs -= process_item_list[-1] + nprocs -= 1 + return process_item_list + # n is the dimension + + if self.dataset == "imagenet": + # for imagenet, generate random samples for this batch only + # array in shared memory storing results of all threads + total_item_size = batch_size + else: + # for cifar and mnist, generate random samples for this entire iteration + total_item_size = num + # divide the jobs evenly to all available threads + process_item_list = div_work_to_cores(total_item_size, self.n_processes) + self.n_processes = len(process_item_list) + # select random sample generation function + if sample_norm == "l2": + # the scaling constant in [a,b]: scale the L2 norm of each sample (has originally norm ~1) + a = 0; b = 3; + elif sample_norm == "li": + # for Linf we don't need the scaling + a = 0.1; b = 0.1; + elif sample_norm == "l1": + # TODO: make the sample ball radius adjustable + a = 0; b = 30; + else: + raise RuntimeError("Unknown sample_norm " + sample_norm) + print('Using sphere', sample_norm) + + ## create necessary shared array structures (saved in /dev/shm) and will be used (and written) in randsphere.py: + # result_arr, scale, input_example, all_inputs + # note: need to use scale[:] = ... not scale = ..., o.w. the contents will not be saved to the shared array + # inputs_0 is the image x_0 + inputs_0 = np.array(input_image) + tag_prefix = str(os.getpid()) + "_" + result_arr = NpShmemArray(np.float32, (total_item_size, dimension), tag_prefix + "randsphere") + # we have an extra batch_size to avoid overflow + scale = NpShmemArray(np.float32, (num+batch_size), tag_prefix + "scale") + scale[:] = (b-a)*np.random.rand(num+batch_size)+a; + input_example = NpShmemArray(np.float32, inputs_0.shape, tag_prefix + "input_example") + # this is a read-only array + input_example[:] = inputs_0 + # all_inputs is a shared memeory array and will be written in the randsphere to save the samples + # all_inputs holds the perturbations for one batch or all samples + all_inputs = NpShmemArray(np.float32, (total_item_size,) + inputs_0.shape, tag_prefix + "all_inputs") + # holds the results copied from all_inputs + clipped_all_inputs = np.empty(dtype=np.float32, shape = (total_item_size,) + inputs_0.shape) + # prepare the argument list + offset_list = [0] + for item in process_item_list[:-1]: + offset_list.append(offset_list[-1] + item) + print(self.n_processes, "threads launched with parameter", process_item_list, offset_list) + + ## create multiple process to generate samples + # randsphere: generate samples (see randsphere.py); partial is a function similar to lambda, now worker_func is a function of idx only + worker_func = partial(randsphere, n = dimension, input_shape = inputs_0.shape, total_size = total_item_size, scale_size = num+batch_size, tag_prefix = tag_prefix, r = 1.0, norm = sample_norm, transform = transform) + worker_args = list(zip(process_item_list, offset_list, [0] * self.n_processes)) + # sample_results is an object to monitor if the process has ended (meaning finish generating samples in randsphere.py) + # this line of code will initiate the worker_func to start working (like initiate the job) + sample_results = self.pool.map_async(worker_func, worker_args) + + # num: # of samples to be run, \leq samples.shape[0] + + # number of iterations + Niters = niters; + + if order == 1: + # store the max L in each iteration + L2_max = np.zeros(Niters) + L1_max = np.zeros(Niters) + Li_max = np.zeros(Niters) + # store the max G in each iteration + G2_max = np.zeros(Niters) + G1_max = np.zeros(Niters) + Gi_max = np.zeros(Niters) + # store computed Lispschitz constants in each iteration + L2 = np.zeros(num) + L1 = np.zeros(num) + Li = np.zeros(num) + # store computed gradient norm in each iteration + G2 = np.zeros(num) + G1 = np.zeros(num) + Gi = np.zeros(num) + elif order == 2: + # store the max H in each iteration + H2_max = np.zeros(Niters) + # store computed 2 norm of H in each iteration + H2 = np.zeros(num) + H2_neg = np.zeros(num) + + # how many batches we have + Nbatches = num // batch_size + + # timer + search_begin_time = time.time() + + """ + 3. Start performing sampling: + """ + ## Start + # multiple runs: generating the samples + ## use worker_func to generate x samples, and then use sess.run to evaluate the gradient norm operator + for iters in range(Niters): + iter_begin_time = time.time() + + # shuffled index + # idx_shuffle = np.random.permutation(num); + + # the scaling constant in [a,b]: scale the L2 norm of each sample (has originally norm ~1) + scale[:] = (b-a)*np.random.rand(num+batch_size)+a; + + # number of L's we have computed + L_counter = 0 + G_counter = 0 + H_counter = 0 + + overhead_time = 0.0 + overhead_start = time.time() + # for cifar and mnist, generate all the random input samples (x in the paper) at once + # for imagenet, generate one batch of input samples (x in the paper) for each iteration + if self.dataset != "imagenet": + # get samples for this iteration: make sure randsphere finished computing samples and stored in all_inputs + # if the samples have not yet done generating, then this line will block the codes until the processes are done, then it will return + sample_results.get() + # copy the results to a buffer and do clipping + np.clip(all_inputs, -0.5, 0.5, out = clipped_all_inputs) + # create multiple process again to generate samples for next batch (initiate a new job) because in below we will need to do sess.run in GPU which might be slow. So we want to generate samples on CPU while running sess.run on GPU to save time + sample_results = self.pool.map_async(worker_func, worker_args) + overhead_time += time.time() - overhead_start + + ## generate input samples "batch_inputs" and compute corresponding gradient norms samples "perturbed_grad_x_norm" + for i in range(Nbatches): + overhead_start = time.time() + # for imagenet, generate random samples for this batch only + if self.dataset == "imagenet": + # get samples for this batch + sample_results.get() + # copy the results to a buffer and do clipping + np.clip(all_inputs, -0.5, 0.5, out = clipped_all_inputs) + # create multiple threads to generate samples for next batch + worker_args = zip(process_item_list, offset_list, [(i + 1) * batch_size] * self.n_processes) + sample_results = self.pool.map_async(worker_func, worker_args) + + if self.dataset == "imagenet": + # we generate samples for each batch at a time + batch_inputs = clipped_all_inputs + else: + # we generate samples for all batches + batch_inputs = clipped_all_inputs[i * batch_size: (i + 1) * batch_size] + # print(result_arr.shape, result_arr) + # print('------------------------') + # print(batch_inputs.shape, batch_inputs.reshape(result_arr.shape)) + # print('------------------------') + overhead_time += time.time() - overhead_start + + if order == 1: + # run inference and get the gradient + perturbed_predicts, perturbed_grad_2_norm, perturbed_grad_1_norm, perturbed_grad_inf_norm = self.sess.run( + [self.output, self.grad_2_norm_op, self.grad_1_norm_op, self.grad_inf_norm_op], + feed_dict = {self.img: batch_inputs, self.target_label: target_label, self.true_label: true_label}) + + if self.compute_slope: + s = batch_inputs.reshape(batch_size, -1) + # compute distance between consecutive samples: not use sequential samples + s12_2_norm = np.linalg.norm(s[0:batch_size-1:2] - s[1:batch_size:2], axis = 1) + s12_1_norm = np.linalg.norm(s[0:batch_size-1:2] - s[1:batch_size:2], ord=1, axis = 1) + s12_i_norm = np.linalg.norm(s[0:batch_size-1:2] - s[1:batch_size:2], ord=np.inf, axis = 1) + # compute function value differences: not use sequential samples + g_x1 = perturbed_predicts[0:batch_size-1:2, c] - perturbed_predicts[0:batch_size-1:2, j] + g_x2 = perturbed_predicts[1:batch_size:2, c] - perturbed_predicts[1:batch_size:2, j] + # estimated Lipschitz constants for this batch + # for slope estimate, we need the DUAL norm + batch_L2 = np.abs(g_x1 - g_x2) / s12_2_norm + batch_L1 = np.abs(g_x1 - g_x2) / s12_i_norm + batch_Li = np.abs(g_x1 - g_x2) / s12_1_norm + L2[L_counter : L_counter + batch_size//2] = batch_L2 + L1[L_counter : L_counter + batch_size//2] = batch_L1 + Li[L_counter : L_counter + batch_size//2] = batch_Li + + G2[G_counter : G_counter + batch_size] = perturbed_grad_2_norm + G1[G_counter : G_counter + batch_size] = perturbed_grad_1_norm + Gi[G_counter : G_counter + batch_size] = perturbed_grad_inf_norm + L_counter += (batch_size//2) + G_counter += batch_size + + elif order == 2: + ##### Lily ##### + randv_batch = np.random.randn(*batch_inputs.shape) + perturbed_hv, perturbed_hv_norm = self.sess.run([self.hv_op, self.hv_norm_op], + feed_dict = {self.img: batch_inputs, self.randv: randv_batch, + self.true_label: true_label, self.target_label: target_label}) + + show_tensor_dim = False + if show_tensor_dim: + print("====================") + print("** Evaluating perturbed_hv and perturbed_hv_norm in batch {}: ".format(iters)) + print("pertubed_hv_prod shape = {}".format(perturbed_hv.shape)) + print("randv_batch shape = {}".format(randv_batch.shape)) + print("perturbed_hv_norm = {}".format(perturbed_hv_norm[:,0])) # size: (Nimg, 1) + print("perturbed_hv_norm shape = {}".format(perturbed_hv_norm.shape)) + #print("perturbed_grad_2_norm= {}".format(perturbed_grad_2_norm)) + #print("perturbed_grad_2_norm shape = {}".format(perturbed_grad_2_norm.shape)) + + + pt_hvs = [] + pt_hvs.append(perturbed_hv+0*randv_batch) + + #print("************** Using tf.while_loop:********************") + # compute max eigenvalue + temp_hv, temp_eig, niter_eig = self.sess.run([self.while_hv_op, self.while_eig, self.it], feed_dict = {self.img: batch_inputs, self.randv: randv_batch, self.true_label: true_label, self.target_label: target_label}) + ##print("converge in {} steps, temp_eig = {}".format(niter_eig, temp_eig)) + + # if max eigenvalue is positive, compute the max neg eigenvalue by using the shiftconst + if max(temp_eig) > 0: + shiftconst = max(temp_eig) + temp_eig_1, niter_eig_1 = self.sess.run([self.while_eig_1, self.it_1], feed_dict = {self.img: batch_inputs, self.randv: randv_batch, self.true_label: true_label, self.target_label: target_label, self.shiftconst: shiftconst}) + ##print("converge in {} steps, temp_eig_1 = {}".format(niter_eig_1, temp_eig_1)) + else: + temp_eig_1 = temp_eig + niter_eig_1 = -1 + + print("temp_eig (abs) converge in {} steps, temp_eig_1 (neg) converge in {} steps".format(niter_eig, niter_eig_1)) + + ## use outer while_loop + #max_eig_iters = 10 + #print_flag = True + #final_est_eig_1 = self._compute_max_abseig(pt_hvs, batch_inputs, true_label, target_label, max_eig_iters, print_flag) + #print("************** Using outer while_loop:********************") + #print("outer loop final_est_eig_1 = {}".format(final_est_eig_1)) + + ## use tf while_loop + final_est_eig = temp_eig + final_est_eig_neg = temp_eig_1 + + H2[H_counter : H_counter + batch_size] = final_est_eig + H2_neg[H_counter : H_counter + batch_size] = final_est_eig_neg + H_counter += batch_size + + if order == 1: + # at the end of each iteration: get the per-iteration max gradient norm + if self.compute_slope: + L2_max[iters] = np.max(L2) + L1_max[iters] = np.max(L1) + Li_max[iters] = np.max(Li) + G2_max[iters] = np.max(G2) + G1_max[iters] = np.max(G1) + Gi_max[iters] = np.max(Gi) + + + if self.compute_slope: + print('[STATS][L2] loop = {}, time = {:.5g}, iter_time = {:.5g}, overhead = {:.5g}, L2 = {:.5g}, L1 = {:.5g}, Linf = {:.5g}, G2 = {:.5g}, G1 = {:.5g}, Ginf = {:.5g}'.format(iters, time.time() - search_begin_time, time.time() - iter_begin_time, overhead_time, L2_max[iters], L1_max[iters], Li_max[iters], G2_max[iters], G1_max[iters], Gi_max[iters])) + else: + print('[STATS][L2] loop = {}, time = {:.5g}, iter_time = {:.5g}, overhead = {:.5g}, G2 = {:.5g}, G1 = {:.5g}, Ginf = {:.5g}'.format(iters, time.time() - search_begin_time, time.time() - iter_begin_time, overhead_time, G2_max[iters], G1_max[iters], Gi_max[iters])) + sys.stdout.flush() + # reset per iteration L and G by filling 0 + if self.compute_slope: + L2.fill(0) + L1.fill(0) + Li.fill(0) + G2.fill(0) + G1.fill(0) + Gi.fill(0) + elif order == 2: + ## consider -lambda_min + idx = H2 > 0 + H2[idx] = H2_neg[idx] + idx_max = np.argmax(abs(H2)) + H2_max[iters] = H2[idx_max] + + print('[STATS][L2] loop = {}, time = {:.5g}, iter_time = {:.5g}, overhead = {:.5g}, H2 = {:.5g}'.format(iters, time.time() - search_begin_time, time.time() - iter_begin_time, overhead_time, H2_max[iters])) + + if order == 1: + print('[STATS][L1] g_x0 = {:.5g}, L2_max = {:.5g}, L1_max = {:.5g}, Linf_max = {:.5g}, G2_max = {:.5g}, G1_max = {:.5g}, Ginf_max = {:.5g}'.format( + g_x0, np.max(L2_max), np.max(L1_max), np.max(Li_max), np.max(G2_max), np.max(G1_max), np.max(Gi_max))) + # when compute the bound we need the DUAL norm + if self.compute_slope: + print('[STATS][L1] bnd_L2_max = {:.5g}, bnd_L1_max = {:.5g}, bnd_Linf_max = {:.5g}, bnd_G2_max = {:.5g}, bnd_G1_max = {:.5g}, bnd_Ginf_max = {:.5g}'.format(g_x0/np.max(L2_max), g_x0/np.max(Li_max), g_x0/np.max(L1_max), g_x0/np.max(G2_max), g_x0/np.max(Gi_max), g_x0/np.max(G1_max))) + else: + print('[STATS][L1] bnd_G2_max = {:.5g}, bnd_G1_max = {:.5g}, bnd_Ginf_max = {:.5g}'.format(g_x0/np.max(G2_max), g_x0/np.max(Gi_max), g_x0/np.max(G1_max))) + + sys.stdout.flush() + + # discard the last batch of samples + sample_results.get() + return [L2_max,L1_max,Li_max,G2_max,G1_max,Gi_max,g_x0,pred] + + elif order == 2: + # find positive eig value and substitute with its corresponding negative eig value, then we only need to sort once + + #print("H2_max = {}".format(H2_max)) + # find max abs(H2_max) + H2_max_val = max(abs(H2_max)) + + print('[STATS][L1] g_x0 = {:.5g}, g_x0_grad_2_norm = {:.5g}, g_x0_grad_1_norm = {:.5g}, g_x0_grad_inf_norm = {:.5g}, H2_max = {:.5g}'.format(g_x0, g_x0_grad_2_norm, g_x0_grad_1_norm, g_x0_grad_inf_norm, H2_max_val)) + + bnd = (-g_x0_grad_2_norm + np.sqrt(g_x0_grad_2_norm**2+2*g_x0*H2_max_val))/H2_max_val + print('[STATS][L1] bnd_H2_max = {:.5g}'.format(bnd)) + sys.stdout.flush() + + sample_results.get() + return [H2_max, g_x0, g_x0_grad_2_norm, g_x0_grad_1_norm, g_x0_grad_inf_norm, pred] + + + def _compute_max_abseig(self, pt_hvs, batch_inputs, true_label, target_label, max_eig_iters, print_flag): + + ## compute hv and est_eig: + i = 0 + cond = False + + pt_eigs = [] + + print("pt_hvs[0] shape = {}".format(pt_hvs[0].shape)) + + # perform power iteration loop outside tensorflow + while (i 0: + cond_element = abs(tmp_est_eig-pt_eigs[i-1]) < 1e-3 + if print_flag: + print("cond = {}".format(cond_element)) + cond = cond_element.all() + i+=1 + + if i == max_eig_iters: + print("==== Reach max iterations!!! ====") + + return pt_eigs[-1] + + + def __del__(self): + # terminate the pool + self.pool.terminate() + + def estimate(self, x_0, true_label, target_label, Nsamp, Niters, sample_norm, transform, order): + result = self._estimate_Lipschitz_multiplerun(Nsamp,Niters,x_0,target_label,true_label,sample_norm, transform, order) + return result + + diff --git a/evaluation/CLEVER/labels/imagenet_val_to_carlini.py b/evaluation/CLEVER/labels/imagenet_val_to_carlini.py new file mode 100644 index 0000000..b9a7753 --- /dev/null +++ b/evaluation/CLEVER/labels/imagenet_val_to_carlini.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import os +import sys +import glob + +f = open('label2num.txt') + +mapping = {} +for line in f: + l = line.strip().split(':') + mapping[l[1]] = l[0] + +print("Total {} classes loaded".format(len(mapping))) + +os.system("mkdir -p imgs") + +file_list = glob.glob('val/**/*.JPEG', recursive=True) +print("Total {} files found".format(len(file_list))) + +cur = 1 +total = len(file_list) + +for img_path in file_list: + s = img_path.split('/')[1] + n = os.path.splitext(os.path.basename(img_path))[0].split('_')[2] + label = mapping[s] + os.system("cp {} imgs/{}.{}.jpg".format(img_path, label, n)) + if cur % 1000 == 0: + print("{}/{} finished ".format(cur, total)) + cur += 1 + +print() + diff --git a/evaluation/CLEVER/labels/label2num.txt b/evaluation/CLEVER/labels/label2num.txt new file mode 100644 index 0000000..e12899c --- /dev/null +++ b/evaluation/CLEVER/labels/label2num.txt @@ -0,0 +1,1000 @@ +1:n01440764:tench, Tinca tinca +2:n01443537:goldfish, Carassius auratus +3:n01484850:great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias +4:n01491361:tiger shark, Galeocerdo cuvieri +5:n01494475:hammerhead, hammerhead shark +6:n01496331:electric ray, crampfish, numbfish, torpedo +7:n01498041:stingray +8:n01514668:cock +9:n01514859:hen +10:n01518878:ostrich, Struthio camelus +11:n01530575:brambling, Fringilla montifringilla +12:n01531178:goldfinch, Carduelis carduelis +13:n01532829:house finch, linnet, Carpodacus mexicanus +14:n01534433:junco, snowbird +15:n01537544:indigo bunting, indigo finch, indigo bird, Passerina cyanea +16:n01558993:robin, American robin, Turdus migratorius +17:n01560419:bulbul +18:n01580077:jay +19:n01582220:magpie +20:n01592084:chickadee +21:n01601694:water ouzel, dipper +22:n01608432:kite +23:n01614925:bald eagle, American eagle, Haliaeetus leucocephalus +24:n01616318:vulture +25:n01622779:great grey owl, great gray owl, Strix nebulosa +26:n01629819:European fire salamander, Salamandra salamandra +27:n01630670:common newt, Triturus vulgaris +28:n01631663:eft +29:n01632458:spotted salamander, Ambystoma maculatum +30:n01632777:axolotl, mud puppy, Ambystoma mexicanum +31:n01641577:bullfrog, Rana catesbeiana +32:n01644373:tree frog, tree-frog +33:n01644900:tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +34:n01664065:loggerhead, loggerhead turtle, Caretta caretta +35:n01665541:leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +36:n01667114:mud turtle +37:n01667778:terrapin +38:n01669191:box turtle, box tortoise +39:n01675722:banded gecko +40:n01677366:common iguana, iguana, Iguana iguana +41:n01682714:American chameleon, anole, Anolis carolinensis +42:n01685808:whiptail, whiptail lizard +43:n01687978:agama +44:n01688243:frilled lizard, Chlamydosaurus kingi +45:n01689811:alligator lizard +46:n01692333:Gila monster, Heloderma suspectum +47:n01693334:green lizard, Lacerta viridis +48:n01694178:African chameleon, Chamaeleo chamaeleon +49:n01695060:Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +50:n01697457:African crocodile, Nile crocodile, Crocodylus niloticus +51:n01698640:American alligator, Alligator mississipiensis +52:n01704323:triceratops +53:n01728572:thunder snake, worm snake, Carphophis amoenus +54:n01728920:ringneck snake, ring-necked snake, ring snake +55:n01729322:hognose snake, puff adder, sand viper +56:n01729977:green snake, grass snake +57:n01734418:king snake, kingsnake +58:n01735189:garter snake, grass snake +59:n01737021:water snake +60:n01739381:vine snake +61:n01740131:night snake, Hypsiglena torquata +62:n01742172:boa constrictor, Constrictor constrictor +63:n01744401:rock python, rock snake, Python sebae +64:n01748264:Indian cobra, Naja naja +65:n01749939:green mamba +66:n01751748:sea snake +67:n01753488:horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +68:n01755581:diamondback, diamondback rattlesnake, Crotalus adamanteus +69:n01756291:sidewinder, horned rattlesnake, Crotalus cerastes +70:n01768244:trilobite +71:n01770081:harvestman, daddy longlegs, Phalangium opilio +72:n01770393:scorpion +73:n01773157:black and gold garden spider, Argiope aurantia +74:n01773549:barn spider, Araneus cavaticus +75:n01773797:garden spider, Aranea diademata +76:n01774384:black widow, Latrodectus mactans +77:n01774750:tarantula +78:n01775062:wolf spider, hunting spider +79:n01776313:tick +80:n01784675:centipede +81:n01795545:black grouse +82:n01796340:ptarmigan +83:n01797886:ruffed grouse, partridge, Bonasa umbellus +84:n01798484:prairie chicken, prairie grouse, prairie fowl +85:n01806143:peacock +86:n01806567:quail +87:n01807496:partridge +88:n01817953:African grey, African gray, Psittacus erithacus +89:n01818515:macaw +90:n01819313:sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +91:n01820546:lorikeet +92:n01824575:coucal +93:n01828970:bee eater +94:n01829413:hornbill +95:n01833805:hummingbird +96:n01843065:jacamar +97:n01843383:toucan +98:n01847000:drake +99:n01855032:red-breasted merganser, Mergus serrator +100:n01855672:goose +101:n01860187:black swan, Cygnus atratus +102:n01871265:tusker +103:n01872401:echidna, spiny anteater, anteater +104:n01873310:platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus +105:n01877812:wallaby, brush kangaroo +106:n01882714:koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +107:n01883070:wombat +108:n01910747:jellyfish +109:n01914609:sea anemone, anemone +110:n01917289:brain coral +111:n01924916:flatworm, platyhelminth +112:n01930112:nematode, nematode worm, roundworm +113:n01943899:conch +114:n01944390:snail +115:n01945685:slug +116:n01950731:sea slug, nudibranch +117:n01955084:chiton, coat-of-mail shell, sea cradle, polyplacophore +118:n01968897:chambered nautilus, pearly nautilus, nautilus +119:n01978287:Dungeness crab, Cancer magister +120:n01978455:rock crab, Cancer irroratus +121:n01980166:fiddler crab +122:n01981276:king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica +123:n01983481:American lobster, Northern lobster, Maine lobster, Homarus americanus +124:n01984695:spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +125:n01985128:crayfish, crawfish, crawdad, crawdaddy +126:n01986214:hermit crab +127:n01990800:isopod +128:n02002556:white stork, Ciconia ciconia +129:n02002724:black stork, Ciconia nigra +130:n02006656:spoonbill +131:n02007558:flamingo +132:n02009229:little blue heron, Egretta caerulea +133:n02009912:American egret, great white heron, Egretta albus +134:n02011460:bittern +518:n02012849:crane +136:n02013706:limpkin, Aramus pictus +137:n02017213:European gallinule, Porphyrio porphyrio +138:n02018207:American coot, marsh hen, mud hen, water hen, Fulica americana +139:n02018795:bustard +140:n02025239:ruddy turnstone, Arenaria interpres +141:n02027492:red-backed sandpiper, dunlin, Erolia alpina +142:n02028035:redshank, Tringa totanus +143:n02033041:dowitcher +144:n02037110:oystercatcher, oyster catcher +145:n02051845:pelican +146:n02056570:king penguin, Aptenodytes patagonica +147:n02058221:albatross, mollymawk +148:n02066245:grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus +149:n02071294:killer whale, killer, orca, grampus, sea wolf, Orcinus orca +150:n02074367:dugong, Dugong dugon +151:n02077923:sea lion +152:n02085620:Chihuahua +153:n02085782:Japanese spaniel +154:n02085936:Maltese dog, Maltese terrier, Maltese +155:n02086079:Pekinese, Pekingese, Peke +156:n02086240:Shih-Tzu +157:n02086646:Blenheim spaniel +158:n02086910:papillon +159:n02087046:toy terrier +160:n02087394:Rhodesian ridgeback +161:n02088094:Afghan hound, Afghan +162:n02088238:basset, basset hound +163:n02088364:beagle +164:n02088466:bloodhound, sleuthhound +165:n02088632:bluetick +166:n02089078:black-and-tan coonhound +167:n02089867:Walker hound, Walker foxhound +168:n02089973:English foxhound +169:n02090379:redbone +170:n02090622:borzoi, Russian wolfhound +171:n02090721:Irish wolfhound +172:n02091032:Italian greyhound +173:n02091134:whippet +174:n02091244:Ibizan hound, Ibizan Podenco +175:n02091467:Norwegian elkhound, elkhound +176:n02091635:otterhound, otter hound +177:n02091831:Saluki, gazelle hound +178:n02092002:Scottish deerhound, deerhound +179:n02092339:Weimaraner +180:n02093256:Staffordshire bullterrier, Staffordshire bull terrier +181:n02093428:American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier +182:n02093647:Bedlington terrier +183:n02093754:Border terrier +184:n02093859:Kerry blue terrier +185:n02093991:Irish terrier +186:n02094114:Norfolk terrier +187:n02094258:Norwich terrier +188:n02094433:Yorkshire terrier +189:n02095314:wire-haired fox terrier +190:n02095570:Lakeland terrier +191:n02095889:Sealyham terrier, Sealyham +192:n02096051:Airedale, Airedale terrier +193:n02096177:cairn, cairn terrier +194:n02096294:Australian terrier +195:n02096437:Dandie Dinmont, Dandie Dinmont terrier +196:n02096585:Boston bull, Boston terrier +197:n02097047:miniature schnauzer +198:n02097130:giant schnauzer +199:n02097209:standard schnauzer +200:n02097298:Scotch terrier, Scottish terrier, Scottie +201:n02097474:Tibetan terrier, chrysanthemum dog +202:n02097658:silky terrier, Sydney silky +203:n02098105:soft-coated wheaten terrier +204:n02098286:West Highland white terrier +205:n02098413:Lhasa, Lhasa apso +206:n02099267:flat-coated retriever +207:n02099429:curly-coated retriever +208:n02099601:golden retriever +209:n02099712:Labrador retriever +210:n02099849:Chesapeake Bay retriever +211:n02100236:German short-haired pointer +212:n02100583:vizsla, Hungarian pointer +213:n02100735:English setter +214:n02100877:Irish setter, red setter +215:n02101006:Gordon setter +216:n02101388:Brittany spaniel +217:n02101556:clumber, clumber spaniel +218:n02102040:English springer, English springer spaniel +219:n02102177:Welsh springer spaniel +220:n02102318:cocker spaniel, English cocker spaniel, cocker +221:n02102480:Sussex spaniel +222:n02102973:Irish water spaniel +223:n02104029:kuvasz +224:n02104365:schipperke +225:n02105056:groenendael +226:n02105162:malinois +227:n02105251:briard +228:n02105412:kelpie +229:n02105505:komondor +230:n02105641:Old English sheepdog, bobtail +231:n02105855:Shetland sheepdog, Shetland sheep dog, Shetland +232:n02106030:collie +233:n02106166:Border collie +234:n02106382:Bouvier des Flandres, Bouviers des Flandres +235:n02106550:Rottweiler +236:n02106662:German shepherd, German shepherd dog, German police dog, alsatian +237:n02107142:Doberman, Doberman pinscher +238:n02107312:miniature pinscher +239:n02107574:Greater Swiss Mountain dog +240:n02107683:Bernese mountain dog +241:n02107908:Appenzeller +242:n02108000:EntleBucher +243:n02108089:boxer +244:n02108422:bull mastiff +245:n02108551:Tibetan mastiff +246:n02108915:French bulldog +247:n02109047:Great Dane +248:n02109525:Saint Bernard, St Bernard +249:n02109961:Eskimo dog, husky +250:n02110063:malamute, malemute, Alaskan malamute +251:n02110185:Siberian husky +252:n02110341:dalmatian, coach dog, carriage dog +253:n02110627:affenpinscher, monkey pinscher, monkey dog +254:n02110806:basenji +255:n02110958:pug, pug-dog +256:n02111129:Leonberg +257:n02111277:Newfoundland, Newfoundland dog +258:n02111500:Great Pyrenees +259:n02111889:Samoyed, Samoyede +260:n02112018:Pomeranian +261:n02112137:chow, chow chow +262:n02112350:keeshond +263:n02112706:Brabancon griffon +264:n02113023:Pembroke, Pembroke Welsh corgi +265:n02113186:Cardigan, Cardigan Welsh corgi +266:n02113624:toy poodle +267:n02113712:miniature poodle +268:n02113799:standard poodle +269:n02113978:Mexican hairless +270:n02114367:timber wolf, grey wolf, gray wolf, Canis lupus +271:n02114548:white wolf, Arctic wolf, Canis lupus tundrarum +272:n02114712:red wolf, maned wolf, Canis rufus, Canis niger +273:n02114855:coyote, prairie wolf, brush wolf, Canis latrans +274:n02115641:dingo, warrigal, warragal, Canis dingo +275:n02115913:dhole, Cuon alpinus +276:n02116738:African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +277:n02117135:hyena, hyaena +278:n02119022:red fox, Vulpes vulpes +279:n02119789:kit fox, Vulpes macrotis +280:n02120079:Arctic fox, white fox, Alopex lagopus +281:n02120505:grey fox, gray fox, Urocyon cinereoargenteus +282:n02123045:tabby, tabby cat +283:n02123159:tiger cat +284:n02123394:Persian cat +285:n02123597:Siamese cat, Siamese +286:n02124075:Egyptian cat +287:n02125311:cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +288:n02127052:lynx, catamount +289:n02128385:leopard, Panthera pardus +290:n02128757:snow leopard, ounce, Panthera uncia +291:n02128925:jaguar, panther, Panthera onca, Felis onca +292:n02129165:lion, king of beasts, Panthera leo +293:n02129604:tiger, Panthera tigris +294:n02130308:cheetah, chetah, Acinonyx jubatus +295:n02132136:brown bear, bruin, Ursus arctos +296:n02133161:American black bear, black bear, Ursus americanus, Euarctos americanus +297:n02134084:ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +298:n02134418:sloth bear, Melursus ursinus, Ursus ursinus +299:n02137549:mongoose +300:n02138441:meerkat, mierkat +301:n02165105:tiger beetle +302:n02165456:ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +303:n02167151:ground beetle, carabid beetle +304:n02168699:long-horned beetle, longicorn, longicorn beetle +305:n02169497:leaf beetle, chrysomelid +306:n02172182:dung beetle +307:n02174001:rhinoceros beetle +308:n02177972:weevil +309:n02190166:fly +310:n02206856:bee +311:n02219486:ant, emmet, pismire +312:n02226429:grasshopper, hopper +313:n02229544:cricket +314:n02231487:walking stick, walkingstick, stick insect +315:n02233338:cockroach, roach +316:n02236044:mantis, mantid +317:n02256656:cicada, cicala +318:n02259212:leafhopper +319:n02264363:lacewing, lacewing fly +320:n02268443:dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk +321:n02268853:damselfly +322:n02276258:admiral +323:n02277742:ringlet, ringlet butterfly +324:n02279972:monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +325:n02280649:cabbage butterfly +326:n02281406:sulphur butterfly, sulfur butterfly +327:n02281787:lycaenid, lycaenid butterfly +328:n02317335:starfish, sea star +329:n02319095:sea urchin +330:n02321529:sea cucumber, holothurian +331:n02325366:wood rabbit, cottontail, cottontail rabbit +332:n02326432:hare +333:n02328150:Angora, Angora rabbit +334:n02342885:hamster +335:n02346627:porcupine, hedgehog +336:n02356798:fox squirrel, eastern fox squirrel, Sciurus niger +337:n02361337:marmot +338:n02363005:beaver +339:n02364673:guinea pig, Cavia cobaya +340:n02389026:sorrel +341:n02391049:zebra +342:n02395406:hog, pig, grunter, squealer, Sus scrofa +343:n02396427:wild boar, boar, Sus scrofa +344:n02397096:warthog +345:n02398521:hippopotamus, hippo, river horse, Hippopotamus amphibius +346:n02403003:ox +347:n02408429:water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +348:n02410509:bison +349:n02412080:ram, tup +350:n02415577:bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis +351:n02417914:ibex, Capra ibex +352:n02422106:hartebeest +353:n02422699:impala, Aepyceros melampus +354:n02423022:gazelle +355:n02437312:Arabian camel, dromedary, Camelus dromedarius +356:n02437616:llama +357:n02441942:weasel +358:n02442845:mink +359:n02443114:polecat, fitch, foulmart, foumart, Mustela putorius +360:n02443484:black-footed ferret, ferret, Mustela nigripes +361:n02444819:otter +362:n02445715:skunk, polecat, wood pussy +363:n02447366:badger +364:n02454379:armadillo +365:n02457408:three-toed sloth, ai, Bradypus tridactylus +366:n02480495:orangutan, orang, orangutang, Pongo pygmaeus +367:n02480855:gorilla, Gorilla gorilla +368:n02481823:chimpanzee, chimp, Pan troglodytes +369:n02483362:gibbon, Hylobates lar +370:n02483708:siamang, Hylobates syndactylus, Symphalangus syndactylus +371:n02484975:guenon, guenon monkey +372:n02486261:patas, hussar monkey, Erythrocebus patas +373:n02486410:baboon +374:n02487347:macaque +375:n02488291:langur +376:n02488702:colobus, colobus monkey +377:n02489166:proboscis monkey, Nasalis larvatus +378:n02490219:marmoset +379:n02492035:capuchin, ringtail, Cebus capucinus +380:n02492660:howler monkey, howler +381:n02493509:titi, titi monkey +382:n02493793:spider monkey, Ateles geoffroyi +383:n02494079:squirrel monkey, Saimiri sciureus +384:n02497673:Madagascar cat, ring-tailed lemur, Lemur catta +385:n02500267:indri, indris, Indri indri, Indri brevicaudatus +386:n02504013:Indian elephant, Elephas maximus +387:n02504458:African elephant, Loxodonta africana +388:n02509815:lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +389:n02510455:giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +390:n02514041:barracouta, snoek +391:n02526121:eel +392:n02536864:coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +393:n02606052:rock beauty, Holocanthus tricolor +394:n02607072:anemone fish +395:n02640242:sturgeon +396:n02641379:gar, garfish, garpike, billfish, Lepisosteus osseus +397:n02643566:lionfish +398:n02655020:puffer, pufferfish, blowfish, globefish +399:n02666196:abacus +400:n02667093:abaya +401:n02669723:academic gown, academic robe, judge's robe +402:n02672831:accordion, piano accordion, squeeze box +403:n02676566:acoustic guitar +404:n02687172:aircraft carrier, carrier, flattop, attack aircraft carrier +405:n02690373:airliner +406:n02692877:airship, dirigible +407:n02699494:altar +408:n02701002:ambulance +409:n02704792:amphibian, amphibious vehicle +410:n02708093:analog clock +411:n02727426:apiary, bee house +412:n02730930:apron +413:n02747177:ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin +414:n02749479:assault rifle, assault gun +415:n02769748:backpack, back pack, knapsack, packsack, rucksack, haversack +416:n02776631:bakery, bakeshop, bakehouse +417:n02777292:balance beam, beam +418:n02782093:balloon +419:n02783161:ballpoint, ballpoint pen, ballpen, Biro +420:n02786058:Band Aid +421:n02787622:banjo +422:n02788148:bannister, banister, balustrade, balusters, handrail +423:n02790996:barbell +424:n02791124:barber chair +425:n02791270:barbershop +426:n02793495:barn +427:n02794156:barometer +428:n02795169:barrel, cask +429:n02797295:barrow, garden cart, lawn cart, wheelbarrow +430:n02799071:baseball +431:n02802426:basketball +432:n02804414:bassinet +433:n02804610:bassoon +434:n02807133:bathing cap, swimming cap +435:n02808304:bath towel +436:n02808440:bathtub, bathing tub, bath, tub +437:n02814533:beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon +438:n02814860:beacon, lighthouse, beacon light, pharos +439:n02815834:beaker +440:n02817516:bearskin, busby, shako +441:n02823428:beer bottle +442:n02823750:beer glass +443:n02825657:bell cote, bell cot +444:n02834397:bib +445:n02835271:bicycle-built-for-two, tandem bicycle, tandem +446:n02837789:bikini, two-piece +447:n02840245:binder, ring-binder +448:n02841315:binoculars, field glasses, opera glasses +449:n02843684:birdhouse +450:n02859443:boathouse +451:n02860847:bobsled, bobsleigh, bob +452:n02865351:bolo tie, bolo, bola tie, bola +453:n02869837:bonnet, poke bonnet +454:n02870880:bookcase +455:n02871525:bookshop, bookstore, bookstall +456:n02877765:bottlecap +457:n02879718:bow +458:n02883205:bow tie, bow-tie, bowtie +459:n02892201:brass, memorial tablet, plaque +460:n02892767:brassiere, bra, bandeau +461:n02894605:breakwater, groin, groyne, mole, bulwark, seawall, jetty +462:n02895154:breastplate, aegis, egis +463:n02906734:broom +464:n02909870:bucket, pail +465:n02910353:buckle +466:n02916936:bulletproof vest +467:n02917067:bullet train, bullet +468:n02927161:butcher shop, meat market +469:n02930766:cab, hack, taxi, taxicab +470:n02939185:caldron, cauldron +471:n02948072:candle, taper, wax light +472:n02950826:cannon +473:n02951358:canoe +474:n02951585:can opener, tin opener +475:n02963159:cardigan +476:n02965783:car mirror +477:n02966193:carousel, carrousel, merry-go-round, roundabout, whirligig +478:n02966687:carpenter's kit, tool kit +479:n02971356:carton +480:n02974003:car wheel +481:n02977058:cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM +482:n02978881:cassette +483:n02979186:cassette player +484:n02980441:castle +485:n02981792:catamaran +486:n02988304:CD player +487:n02992211:cello, violoncello +488:n02992529:cellular telephone, cellular phone, cellphone, cell, mobile phone +489:n02999410:chain +490:n03000134:chainlink fence +491:n03000247:chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour +492:n03000684:chain saw, chainsaw +493:n03014705:chest +494:n03016953:chiffonier, commode +495:n03017168:chime, bell, gong +496:n03018349:china cabinet, china closet +497:n03026506:Christmas stocking +498:n03028079:church, church building +499:n03032252:cinema, movie theater, movie theatre, movie house, picture palace +500:n03041632:cleaver, meat cleaver, chopper +501:n03042490:cliff dwelling +502:n03045698:cloak +503:n03047690:clog, geta, patten, sabot +504:n03062245:cocktail shaker +505:n03063599:coffee mug +506:n03063689:coffeepot +507:n03065424:coil, spiral, volute, whorl, helix +508:n03075370:combination lock +509:n03085013:computer keyboard, keypad +510:n03089624:confectionery, confectionary, candy store +511:n03095699:container ship, containership, container vessel +512:n03100240:convertible +513:n03109150:corkscrew, bottle screw +514:n03110669:cornet, horn, trumpet, trump +515:n03124043:cowboy boot +516:n03124170:cowboy hat, ten-gallon hat +517:n03125729:cradle +518:n03126707:crane +519:n03127747:crash helmet +520:n03127925:crate +521:n03131574:crib, cot +522:n03133878:Crock Pot +523:n03134739:croquet ball +524:n03141823:crutch +525:n03146219:cuirass +526:n03160309:dam, dike, dyke +527:n03179701:desk +528:n03180011:desktop computer +529:n03187595:dial telephone, dial phone +530:n03188531:diaper, nappy, napkin +531:n03196217:digital clock +532:n03197337:digital watch +533:n03201208:dining table, board +534:n03207743:dishrag, dishcloth +535:n03207941:dishwasher, dish washer, dishwashing machine +536:n03208938:disk brake, disc brake +537:n03216828:dock, dockage, docking facility +538:n03218198:dogsled, dog sled, dog sleigh +539:n03220513:dome +540:n03223299:doormat, welcome mat +541:n03240683:drilling platform, offshore rig +542:n03249569:drum, membranophone, tympan +543:n03250847:drumstick +544:n03255030:dumbbell +545:n03259280:Dutch oven +546:n03271574:electric fan, blower +547:n03272010:electric guitar +548:n03272562:electric locomotive +549:n03290653:entertainment center +550:n03291819:envelope +551:n03297495:espresso maker +552:n03314780:face powder +553:n03325584:feather boa, boa +554:n03337140:file, file cabinet, filing cabinet +555:n03344393:fireboat +556:n03345487:fire engine, fire truck +557:n03347037:fire screen, fireguard +558:n03355925:flagpole, flagstaff +559:n03372029:flute, transverse flute +560:n03376595:folding chair +561:n03379051:football helmet +562:n03384352:forklift +563:n03388043:fountain +564:n03388183:fountain pen +565:n03388549:four-poster +566:n03393912:freight car +567:n03394916:French horn, horn +568:n03400231:frying pan, frypan, skillet +569:n03404251:fur coat +570:n03417042:garbage truck, dustcart +571:n03424325:gasmask, respirator, gas helmet +572:n03425413:gas pump, gasoline pump, petrol pump, island dispenser +573:n03443371:goblet +574:n03444034:go-kart +575:n03445777:golf ball +576:n03445924:golfcart, golf cart +577:n03447447:gondola +578:n03447721:gong, tam-tam +579:n03450230:gown +580:n03452741:grand piano, grand +581:n03457902:greenhouse, nursery, glasshouse +582:n03459775:grille, radiator grille +583:n03461385:grocery store, grocery, food market, market +584:n03467068:guillotine +585:n03476684:hair slide +586:n03476991:hair spray +587:n03478589:half track +588:n03481172:hammer +589:n03482405:hamper +590:n03483316:hand blower, blow dryer, blow drier, hair dryer, hair drier +591:n03485407:hand-held computer, hand-held microcomputer +592:n03485794:handkerchief, hankie, hanky, hankey +593:n03492542:hard disc, hard disk, fixed disk +594:n03494278:harmonica, mouth organ, harp, mouth harp +595:n03495258:harp +596:n03496892:harvester, reaper +597:n03498962:hatchet +598:n03527444:holster +599:n03529860:home theater, home theatre +600:n03530642:honeycomb +601:n03532672:hook, claw +602:n03534580:hoopskirt, crinoline +603:n03535780:horizontal bar, high bar +604:n03538406:horse cart, horse-cart +605:n03544143:hourglass +606:n03584254:iPod +607:n03584829:iron, smoothing iron +608:n03590841:jack-o'-lantern +609:n03594734:jean, blue jean, denim +610:n03594945:jeep, landrover +611:n03595614:jersey, T-shirt, tee shirt +612:n03598930:jigsaw puzzle +613:n03599486:jinrikisha, ricksha, rickshaw +614:n03602883:joystick +615:n03617480:kimono +616:n03623198:knee pad +617:n03627232:knot +618:n03630383:lab coat, laboratory coat +619:n03633091:ladle +620:n03637318:lampshade, lamp shade +621:n03642806:laptop, laptop computer +622:n03649909:lawn mower, mower +623:n03657121:lens cap, lens cover +624:n03658185:letter opener, paper knife, paperknife +625:n03661043:library +626:n03662601:lifeboat +627:n03666591:lighter, light, igniter, ignitor +628:n03670208:limousine, limo +629:n03673027:liner, ocean liner +630:n03676483:lipstick, lip rouge +631:n03680355:Loafer +632:n03690938:lotion +633:n03691459:loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +634:n03692522:loupe, jeweler's loupe +635:n03697007:lumbermill, sawmill +636:n03706229:magnetic compass +637:n03709823:mailbag, postbag +638:n03710193:mailbox, letter box +639:n03710637:maillot +640:n03710721:maillot, tank suit +641:n03717622:manhole cover +642:n03720891:maraca +643:n03721384:marimba, xylophone +644:n03724870:mask +645:n03729826:matchstick +646:n03733131:maypole +647:n03733281:maze, labyrinth +648:n03733805:measuring cup +649:n03742115:medicine chest, medicine cabinet +650:n03743016:megalith, megalithic structure +651:n03759954:microphone, mike +652:n03761084:microwave, microwave oven +653:n03763968:military uniform +654:n03764736:milk can +655:n03769881:minibus +656:n03770439:miniskirt, mini +657:n03770679:minivan +658:n03773504:missile +659:n03775071:mitten +660:n03775546:mixing bowl +661:n03776460:mobile home, manufactured home +662:n03777568:Model T +663:n03777754:modem +664:n03781244:monastery +665:n03782006:monitor +666:n03785016:moped +667:n03786901:mortar +668:n03787032:mortarboard +669:n03788195:mosque +670:n03788365:mosquito net +671:n03791053:motor scooter, scooter +672:n03792782:mountain bike, all-terrain bike, off-roader +673:n03792972:mountain tent +674:n03793489:mouse, computer mouse +675:n03794056:mousetrap +676:n03796401:moving van +677:n03803284:muzzle +678:n03804744:nail +679:n03814639:neck brace +680:n03814906:necklace +681:n03825788:nipple +682:n03832673:notebook, notebook computer +683:n03837869:obelisk +684:n03838899:oboe, hautboy, hautbois +685:n03840681:ocarina, sweet potato +686:n03841143:odometer, hodometer, mileometer, milometer +687:n03843555:oil filter +688:n03854065:organ, pipe organ +689:n03857828:oscilloscope, scope, cathode-ray oscilloscope, CRO +690:n03866082:overskirt +691:n03868242:oxcart +692:n03868863:oxygen mask +693:n03871628:packet +694:n03873416:paddle, boat paddle +695:n03874293:paddlewheel, paddle wheel +696:n03874599:padlock +697:n03876231:paintbrush +698:n03877472:pajama, pyjama, pj's, jammies +699:n03877845:palace +700:n03884397:panpipe, pandean pipe, syrinx +701:n03887697:paper towel +702:n03888257:parachute, chute +703:n03888605:parallel bars, bars +704:n03891251:park bench +705:n03891332:parking meter +706:n03895866:passenger car, coach, carriage +707:n03899768:patio, terrace +708:n03902125:pay-phone, pay-station +709:n03903868:pedestal, plinth, footstall +710:n03908618:pencil box, pencil case +711:n03908714:pencil sharpener +712:n03916031:perfume, essence +713:n03920288:Petri dish +714:n03924679:photocopier +715:n03929660:pick, plectrum, plectron +716:n03929855:pickelhaube +717:n03930313:picket fence, paling +718:n03930630:pickup, pickup truck +719:n03933933:pier +720:n03935335:piggy bank, penny bank +721:n03937543:pill bottle +722:n03938244:pillow +723:n03942813:ping-pong ball +724:n03944341:pinwheel +725:n03947888:pirate, pirate ship +726:n03950228:pitcher, ewer +727:n03954731:plane, carpenter's plane, woodworking plane +728:n03956157:planetarium +729:n03958227:plastic bag +730:n03961711:plate rack +731:n03967562:plow, plough +732:n03970156:plunger, plumber's helper +733:n03976467:Polaroid camera, Polaroid Land camera +734:n03976657:pole +735:n03977966:police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +736:n03980874:poncho +737:n03982430:pool table, billiard table, snooker table +738:n03983396:pop bottle, soda bottle +739:n03991062:pot, flowerpot +740:n03992509:potter's wheel +741:n03995372:power drill +742:n03998194:prayer rug, prayer mat +743:n04004767:printer +744:n04005630:prison, prison house +745:n04008634:projectile, missile +746:n04009552:projector +747:n04019541:puck, hockey puck +748:n04023962:punching bag, punch bag, punching ball, punchball +749:n04026417:purse +750:n04033901:quill, quill pen +751:n04033995:quilt, comforter, comfort, puff +752:n04037443:racer, race car, racing car +753:n04039381:racket, racquet +754:n04040759:radiator +755:n04041544:radio, wireless +756:n04044716:radio telescope, radio reflector +757:n04049303:rain barrel +758:n04065272:recreational vehicle, RV, R.V. +759:n04067472:reel +760:n04069434:reflex camera +761:n04070727:refrigerator, icebox +762:n04074963:remote control, remote +763:n04081281:restaurant, eating house, eating place, eatery +764:n04086273:revolver, six-gun, six-shooter +765:n04090263:rifle +766:n04099969:rocking chair, rocker +767:n04111531:rotisserie +768:n04116512:rubber eraser, rubber, pencil eraser +769:n04118538:rugby ball +770:n04118776:rule, ruler +771:n04120489:running shoe +772:n04125021:safe +773:n04127249:safety pin +774:n04131690:saltshaker, salt shaker +775:n04133789:sandal +776:n04136333:sarong +777:n04141076:sax, saxophone +778:n04141327:scabbard +779:n04141975:scale, weighing machine +780:n04146614:school bus +781:n04147183:schooner +782:n04149813:scoreboard +783:n04152593:screen, CRT screen +784:n04153751:screw +785:n04154565:screwdriver +786:n04162706:seat belt, seatbelt +787:n04179913:sewing machine +788:n04192698:shield, buckler +789:n04200800:shoe shop, shoe-shop, shoe store +790:n04201297:shoji +791:n04204238:shopping basket +792:n04204347:shopping cart +793:n04208210:shovel +794:n04209133:shower cap +795:n04209239:shower curtain +796:n04228054:ski +797:n04229816:ski mask +798:n04235860:sleeping bag +799:n04238763:slide rule, slipstick +800:n04239074:sliding door +801:n04243546:slot, one-armed bandit +802:n04251144:snorkel +803:n04252077:snowmobile +804:n04252225:snowplow, snowplough +805:n04254120:soap dispenser +806:n04254680:soccer ball +807:n04254777:sock +808:n04258138:solar dish, solar collector, solar furnace +809:n04259630:sombrero +810:n04263257:soup bowl +811:n04264628:space bar +812:n04265275:space heater +813:n04266014:space shuttle +814:n04270147:spatula +815:n04273569:speedboat +816:n04275548:spider web, spider's web +817:n04277352:spindle +818:n04285008:sports car, sport car +819:n04286575:spotlight, spot +820:n04296562:stage +821:n04310018:steam locomotive +822:n04311004:steel arch bridge +823:n04311174:steel drum +824:n04317175:stethoscope +825:n04325704:stole +826:n04326547:stone wall +827:n04328186:stopwatch, stop watch +828:n04330267:stove +829:n04332243:strainer +830:n04335435:streetcar, tram, tramcar, trolley, trolley car +831:n04336792:stretcher +832:n04344873:studio couch, day bed +833:n04346328:stupa, tope +834:n04347754:submarine, pigboat, sub, U-boat +835:n04350905:suit, suit of clothes +836:n04355338:sundial +837:n04355933:sunglass +838:n04356056:sunglasses, dark glasses, shades +839:n04357314:sunscreen, sunblock, sun blocker +840:n04366367:suspension bridge +841:n04367480:swab, swob, mop +842:n04370456:sweatshirt +843:n04371430:swimming trunks, bathing trunks +844:n04371774:swing +845:n04372370:switch, electric switch, electrical switch +846:n04376876:syringe +847:n04380533:table lamp +848:n04389033:tank, army tank, armored combat vehicle, armoured combat vehicle +849:n04392985:tape player +850:n04398044:teapot +851:n04399382:teddy, teddy bear +852:n04404412:television, television system +853:n04409515:tennis ball +854:n04417672:thatch, thatched roof +855:n04418357:theater curtain, theatre curtain +856:n04423845:thimble +857:n04428191:thresher, thrasher, threshing machine +858:n04429376:throne +859:n04435653:tile roof +860:n04442312:toaster +861:n04443257:tobacco shop, tobacconist shop, tobacconist +862:n04447861:toilet seat +863:n04456115:torch +864:n04458633:totem pole +865:n04461696:tow truck, tow car, wrecker +866:n04462240:toyshop +867:n04465501:tractor +868:n04467665:trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +869:n04476259:tray +870:n04479046:trench coat +871:n04482393:tricycle, trike, velocipede +872:n04483307:trimaran +873:n04485082:tripod +874:n04486054:triumphal arch +875:n04487081:trolleybus, trolley coach, trackless trolley +876:n04487394:trombone +877:n04493381:tub, vat +878:n04501370:turnstile +879:n04505470:typewriter keyboard +880:n04507155:umbrella +881:n04509417:unicycle, monocycle +882:n04515003:upright, upright piano +883:n04517823:vacuum, vacuum cleaner +884:n04522168:vase +885:n04523525:vault +886:n04525038:velvet +887:n04525305:vending machine +888:n04532106:vestment +889:n04532670:viaduct +890:n04536866:violin, fiddle +891:n04540053:volleyball +892:n04542943:waffle iron +893:n04548280:wall clock +894:n04548362:wallet, billfold, notecase, pocketbook +895:n04550184:wardrobe, closet, press +896:n04552348:warplane, military plane +897:n04553703:washbasin, handbasin, washbowl, lavabo, wash-hand basin +898:n04554684:washer, automatic washer, washing machine +899:n04557648:water bottle +900:n04560804:water jug +901:n04562935:water tower +902:n04579145:whiskey jug +903:n04579432:whistle +904:n04584207:wig +905:n04589890:window screen +906:n04590129:window shade +907:n04591157:Windsor tie +908:n04591713:wine bottle +909:n04592741:wing +910:n04596742:wok +911:n04597913:wooden spoon +912:n04599235:wool, woolen, woollen +913:n04604644:worm fence, snake fence, snake-rail fence, Virginia fence +914:n04606251:wreck +915:n04612504:yawl +916:n04613696:yurt +917:n06359193:web site, website, internet site, site +918:n06596364:comic book +919:n06785654:crossword puzzle, crossword +920:n06794110:street sign +921:n06874185:traffic light, traffic signal, stoplight +922:n07248320:book jacket, dust cover, dust jacket, dust wrapper +923:n07565083:menu +924:n07579787:plate +925:n07583066:guacamole +926:n07584110:consomme +927:n07590611:hot pot, hotpot +928:n07613480:trifle +929:n07614500:ice cream, icecream +930:n07615774:ice lolly, lolly, lollipop, popsicle +931:n07684084:French loaf +932:n07693725:bagel, beigel +933:n07695742:pretzel +934:n07697313:cheeseburger +935:n07697537:hotdog, hot dog, red hot +936:n07711569:mashed potato +937:n07714571:head cabbage +938:n07714990:broccoli +939:n07715103:cauliflower +940:n07716358:zucchini, courgette +941:n07716906:spaghetti squash +942:n07717410:acorn squash +943:n07717556:butternut squash +944:n07718472:cucumber, cuke +945:n07718747:artichoke, globe artichoke +946:n07720875:bell pepper +947:n07730033:cardoon +948:n07734744:mushroom +949:n07742313:Granny Smith +950:n07745940:strawberry +951:n07747607:orange +952:n07749582:lemon +953:n07753113:fig +954:n07753275:pineapple, ananas +955:n07753592:banana +956:n07754684:jackfruit, jak, jack +957:n07760859:custard apple +958:n07768694:pomegranate +959:n07802026:hay +960:n07831146:carbonara +961:n07836838:chocolate sauce, chocolate syrup +962:n07860988:dough +963:n07871810:meat loaf, meatloaf +964:n07873807:pizza, pizza pie +965:n07875152:potpie +966:n07880968:burrito +967:n07892512:red wine +968:n07920052:espresso +969:n07930864:cup +970:n07932039:eggnog +971:n09193705:alp +972:n09229709:bubble +973:n09246464:cliff, drop, drop-off +974:n09256479:coral reef +975:n09288635:geyser +976:n09332890:lakeside, lakeshore +977:n09399592:promontory, headland, head, foreland +978:n09421951:sandbar, sand bar +979:n09428293:seashore, coast, seacoast, sea-coast +980:n09468604:valley, vale +981:n09472597:volcano +982:n09835506:ballplayer, baseball player +983:n10148035:groom, bridegroom +984:n10565667:scuba diver +985:n11879895:rapeseed +986:n11939491:daisy +987:n12057211:yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum +988:n12144580:corn +989:n12267677:acorn +990:n12620546:hip, rose hip, rosehip +991:n12768682:buckeye, horse chestnut, conker +992:n12985857:coral fungus +993:n12998815:agaric +994:n13037406:gyromitra +995:n13040303:stinkhorn, carrion fungus +996:n13044778:earthstar +997:n13052670:hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +998:n13054560:bolete +999:n13133613:ear, spike, capitulum +1000:n15075141:toilet tissue, toilet paper, bathroom tissue diff --git a/evaluation/CLEVER/labels/labels.py b/evaluation/CLEVER/labels/labels.py new file mode 100644 index 0000000..5543811 --- /dev/null +++ b/evaluation/CLEVER/labels/labels.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +new_label = "labels.txt" +imagenet_map = "synset_words.txt" +label2num = "label2num.txt" + +num_dict = {} + +out_file = open(label2num, 'w') + +with open(new_label, 'r') as f: + for line in f: + if line: + num = int(line.split(':')[0]) + lab = line.split(':')[1].strip() + num_dict[lab] = num + +print(len(num_dict)) + +with open(imagenet_map, 'r') as f: + for line in f: + if line: + nn = line[:9] + lab = line[10:].strip() + print(nn, lab) + num = num_dict[lab] + out_file.write("{}:{}:{}\n".format(num, nn, lab)) + diff --git a/evaluation/CLEVER/labels/labels.txt b/evaluation/CLEVER/labels/labels.txt new file mode 100644 index 0000000..d74ff55 --- /dev/null +++ b/evaluation/CLEVER/labels/labels.txt @@ -0,0 +1,1001 @@ +0:background +1:tench, Tinca tinca +2:goldfish, Carassius auratus +3:great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias +4:tiger shark, Galeocerdo cuvieri +5:hammerhead, hammerhead shark +6:electric ray, crampfish, numbfish, torpedo +7:stingray +8:cock +9:hen +10:ostrich, Struthio camelus +11:brambling, Fringilla montifringilla +12:goldfinch, Carduelis carduelis +13:house finch, linnet, Carpodacus mexicanus +14:junco, snowbird +15:indigo bunting, indigo finch, indigo bird, Passerina cyanea +16:robin, American robin, Turdus migratorius +17:bulbul +18:jay +19:magpie +20:chickadee +21:water ouzel, dipper +22:kite +23:bald eagle, American eagle, Haliaeetus leucocephalus +24:vulture +25:great grey owl, great gray owl, Strix nebulosa +26:European fire salamander, Salamandra salamandra +27:common newt, Triturus vulgaris +28:eft +29:spotted salamander, Ambystoma maculatum +30:axolotl, mud puppy, Ambystoma mexicanum +31:bullfrog, Rana catesbeiana +32:tree frog, tree-frog +33:tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +34:loggerhead, loggerhead turtle, Caretta caretta +35:leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +36:mud turtle +37:terrapin +38:box turtle, box tortoise +39:banded gecko +40:common iguana, iguana, Iguana iguana +41:American chameleon, anole, Anolis carolinensis +42:whiptail, whiptail lizard +43:agama +44:frilled lizard, Chlamydosaurus kingi +45:alligator lizard +46:Gila monster, Heloderma suspectum +47:green lizard, Lacerta viridis +48:African chameleon, Chamaeleo chamaeleon +49:Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +50:African crocodile, Nile crocodile, Crocodylus niloticus +51:American alligator, Alligator mississipiensis +52:triceratops +53:thunder snake, worm snake, Carphophis amoenus +54:ringneck snake, ring-necked snake, ring snake +55:hognose snake, puff adder, sand viper +56:green snake, grass snake +57:king snake, kingsnake +58:garter snake, grass snake +59:water snake +60:vine snake +61:night snake, Hypsiglena torquata +62:boa constrictor, Constrictor constrictor +63:rock python, rock snake, Python sebae +64:Indian cobra, Naja naja +65:green mamba +66:sea snake +67:horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +68:diamondback, diamondback rattlesnake, Crotalus adamanteus +69:sidewinder, horned rattlesnake, Crotalus cerastes +70:trilobite +71:harvestman, daddy longlegs, Phalangium opilio +72:scorpion +73:black and gold garden spider, Argiope aurantia +74:barn spider, Araneus cavaticus +75:garden spider, Aranea diademata +76:black widow, Latrodectus mactans +77:tarantula +78:wolf spider, hunting spider +79:tick +80:centipede +81:black grouse +82:ptarmigan +83:ruffed grouse, partridge, Bonasa umbellus +84:prairie chicken, prairie grouse, prairie fowl +85:peacock +86:quail +87:partridge +88:African grey, African gray, Psittacus erithacus +89:macaw +90:sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +91:lorikeet +92:coucal +93:bee eater +94:hornbill +95:hummingbird +96:jacamar +97:toucan +98:drake +99:red-breasted merganser, Mergus serrator +100:goose +101:black swan, Cygnus atratus +102:tusker +103:echidna, spiny anteater, anteater +104:platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus +105:wallaby, brush kangaroo +106:koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +107:wombat +108:jellyfish +109:sea anemone, anemone +110:brain coral +111:flatworm, platyhelminth +112:nematode, nematode worm, roundworm +113:conch +114:snail +115:slug +116:sea slug, nudibranch +117:chiton, coat-of-mail shell, sea cradle, polyplacophore +118:chambered nautilus, pearly nautilus, nautilus +119:Dungeness crab, Cancer magister +120:rock crab, Cancer irroratus +121:fiddler crab +122:king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica +123:American lobster, Northern lobster, Maine lobster, Homarus americanus +124:spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +125:crayfish, crawfish, crawdad, crawdaddy +126:hermit crab +127:isopod +128:white stork, Ciconia ciconia +129:black stork, Ciconia nigra +130:spoonbill +131:flamingo +132:little blue heron, Egretta caerulea +133:American egret, great white heron, Egretta albus +134:bittern +135:crane +136:limpkin, Aramus pictus +137:European gallinule, Porphyrio porphyrio +138:American coot, marsh hen, mud hen, water hen, Fulica americana +139:bustard +140:ruddy turnstone, Arenaria interpres +141:red-backed sandpiper, dunlin, Erolia alpina +142:redshank, Tringa totanus +143:dowitcher +144:oystercatcher, oyster catcher +145:pelican +146:king penguin, Aptenodytes patagonica +147:albatross, mollymawk +148:grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus +149:killer whale, killer, orca, grampus, sea wolf, Orcinus orca +150:dugong, Dugong dugon +151:sea lion +152:Chihuahua +153:Japanese spaniel +154:Maltese dog, Maltese terrier, Maltese +155:Pekinese, Pekingese, Peke +156:Shih-Tzu +157:Blenheim spaniel +158:papillon +159:toy terrier +160:Rhodesian ridgeback +161:Afghan hound, Afghan +162:basset, basset hound +163:beagle +164:bloodhound, sleuthhound +165:bluetick +166:black-and-tan coonhound +167:Walker hound, Walker foxhound +168:English foxhound +169:redbone +170:borzoi, Russian wolfhound +171:Irish wolfhound +172:Italian greyhound +173:whippet +174:Ibizan hound, Ibizan Podenco +175:Norwegian elkhound, elkhound +176:otterhound, otter hound +177:Saluki, gazelle hound +178:Scottish deerhound, deerhound +179:Weimaraner +180:Staffordshire bullterrier, Staffordshire bull terrier +181:American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier +182:Bedlington terrier +183:Border terrier +184:Kerry blue terrier +185:Irish terrier +186:Norfolk terrier +187:Norwich terrier +188:Yorkshire terrier +189:wire-haired fox terrier +190:Lakeland terrier +191:Sealyham terrier, Sealyham +192:Airedale, Airedale terrier +193:cairn, cairn terrier +194:Australian terrier +195:Dandie Dinmont, Dandie Dinmont terrier +196:Boston bull, Boston terrier +197:miniature schnauzer +198:giant schnauzer +199:standard schnauzer +200:Scotch terrier, Scottish terrier, Scottie +201:Tibetan terrier, chrysanthemum dog +202:silky terrier, Sydney silky +203:soft-coated wheaten terrier +204:West Highland white terrier +205:Lhasa, Lhasa apso +206:flat-coated retriever +207:curly-coated retriever +208:golden retriever +209:Labrador retriever +210:Chesapeake Bay retriever +211:German short-haired pointer +212:vizsla, Hungarian pointer +213:English setter +214:Irish setter, red setter +215:Gordon setter +216:Brittany spaniel +217:clumber, clumber spaniel +218:English springer, English springer spaniel +219:Welsh springer spaniel +220:cocker spaniel, English cocker spaniel, cocker +221:Sussex spaniel +222:Irish water spaniel +223:kuvasz +224:schipperke +225:groenendael +226:malinois +227:briard +228:kelpie +229:komondor +230:Old English sheepdog, bobtail +231:Shetland sheepdog, Shetland sheep dog, Shetland +232:collie +233:Border collie +234:Bouvier des Flandres, Bouviers des Flandres +235:Rottweiler +236:German shepherd, German shepherd dog, German police dog, alsatian +237:Doberman, Doberman pinscher +238:miniature pinscher +239:Greater Swiss Mountain dog +240:Bernese mountain dog +241:Appenzeller +242:EntleBucher +243:boxer +244:bull mastiff +245:Tibetan mastiff +246:French bulldog +247:Great Dane +248:Saint Bernard, St Bernard +249:Eskimo dog, husky +250:malamute, malemute, Alaskan malamute +251:Siberian husky +252:dalmatian, coach dog, carriage dog +253:affenpinscher, monkey pinscher, monkey dog +254:basenji +255:pug, pug-dog +256:Leonberg +257:Newfoundland, Newfoundland dog +258:Great Pyrenees +259:Samoyed, Samoyede +260:Pomeranian +261:chow, chow chow +262:keeshond +263:Brabancon griffon +264:Pembroke, Pembroke Welsh corgi +265:Cardigan, Cardigan Welsh corgi +266:toy poodle +267:miniature poodle +268:standard poodle +269:Mexican hairless +270:timber wolf, grey wolf, gray wolf, Canis lupus +271:white wolf, Arctic wolf, Canis lupus tundrarum +272:red wolf, maned wolf, Canis rufus, Canis niger +273:coyote, prairie wolf, brush wolf, Canis latrans +274:dingo, warrigal, warragal, Canis dingo +275:dhole, Cuon alpinus +276:African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +277:hyena, hyaena +278:red fox, Vulpes vulpes +279:kit fox, Vulpes macrotis +280:Arctic fox, white fox, Alopex lagopus +281:grey fox, gray fox, Urocyon cinereoargenteus +282:tabby, tabby cat +283:tiger cat +284:Persian cat +285:Siamese cat, Siamese +286:Egyptian cat +287:cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +288:lynx, catamount +289:leopard, Panthera pardus +290:snow leopard, ounce, Panthera uncia +291:jaguar, panther, Panthera onca, Felis onca +292:lion, king of beasts, Panthera leo +293:tiger, Panthera tigris +294:cheetah, chetah, Acinonyx jubatus +295:brown bear, bruin, Ursus arctos +296:American black bear, black bear, Ursus americanus, Euarctos americanus +297:ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +298:sloth bear, Melursus ursinus, Ursus ursinus +299:mongoose +300:meerkat, mierkat +301:tiger beetle +302:ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +303:ground beetle, carabid beetle +304:long-horned beetle, longicorn, longicorn beetle +305:leaf beetle, chrysomelid +306:dung beetle +307:rhinoceros beetle +308:weevil +309:fly +310:bee +311:ant, emmet, pismire +312:grasshopper, hopper +313:cricket +314:walking stick, walkingstick, stick insect +315:cockroach, roach +316:mantis, mantid +317:cicada, cicala +318:leafhopper +319:lacewing, lacewing fly +320:dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk +321:damselfly +322:admiral +323:ringlet, ringlet butterfly +324:monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +325:cabbage butterfly +326:sulphur butterfly, sulfur butterfly +327:lycaenid, lycaenid butterfly +328:starfish, sea star +329:sea urchin +330:sea cucumber, holothurian +331:wood rabbit, cottontail, cottontail rabbit +332:hare +333:Angora, Angora rabbit +334:hamster +335:porcupine, hedgehog +336:fox squirrel, eastern fox squirrel, Sciurus niger +337:marmot +338:beaver +339:guinea pig, Cavia cobaya +340:sorrel +341:zebra +342:hog, pig, grunter, squealer, Sus scrofa +343:wild boar, boar, Sus scrofa +344:warthog +345:hippopotamus, hippo, river horse, Hippopotamus amphibius +346:ox +347:water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +348:bison +349:ram, tup +350:bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis +351:ibex, Capra ibex +352:hartebeest +353:impala, Aepyceros melampus +354:gazelle +355:Arabian camel, dromedary, Camelus dromedarius +356:llama +357:weasel +358:mink +359:polecat, fitch, foulmart, foumart, Mustela putorius +360:black-footed ferret, ferret, Mustela nigripes +361:otter +362:skunk, polecat, wood pussy +363:badger +364:armadillo +365:three-toed sloth, ai, Bradypus tridactylus +366:orangutan, orang, orangutang, Pongo pygmaeus +367:gorilla, Gorilla gorilla +368:chimpanzee, chimp, Pan troglodytes +369:gibbon, Hylobates lar +370:siamang, Hylobates syndactylus, Symphalangus syndactylus +371:guenon, guenon monkey +372:patas, hussar monkey, Erythrocebus patas +373:baboon +374:macaque +375:langur +376:colobus, colobus monkey +377:proboscis monkey, Nasalis larvatus +378:marmoset +379:capuchin, ringtail, Cebus capucinus +380:howler monkey, howler +381:titi, titi monkey +382:spider monkey, Ateles geoffroyi +383:squirrel monkey, Saimiri sciureus +384:Madagascar cat, ring-tailed lemur, Lemur catta +385:indri, indris, Indri indri, Indri brevicaudatus +386:Indian elephant, Elephas maximus +387:African elephant, Loxodonta africana +388:lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +389:giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +390:barracouta, snoek +391:eel +392:coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +393:rock beauty, Holocanthus tricolor +394:anemone fish +395:sturgeon +396:gar, garfish, garpike, billfish, Lepisosteus osseus +397:lionfish +398:puffer, pufferfish, blowfish, globefish +399:abacus +400:abaya +401:academic gown, academic robe, judge's robe +402:accordion, piano accordion, squeeze box +403:acoustic guitar +404:aircraft carrier, carrier, flattop, attack aircraft carrier +405:airliner +406:airship, dirigible +407:altar +408:ambulance +409:amphibian, amphibious vehicle +410:analog clock +411:apiary, bee house +412:apron +413:ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin +414:assault rifle, assault gun +415:backpack, back pack, knapsack, packsack, rucksack, haversack +416:bakery, bakeshop, bakehouse +417:balance beam, beam +418:balloon +419:ballpoint, ballpoint pen, ballpen, Biro +420:Band Aid +421:banjo +422:bannister, banister, balustrade, balusters, handrail +423:barbell +424:barber chair +425:barbershop +426:barn +427:barometer +428:barrel, cask +429:barrow, garden cart, lawn cart, wheelbarrow +430:baseball +431:basketball +432:bassinet +433:bassoon +434:bathing cap, swimming cap +435:bath towel +436:bathtub, bathing tub, bath, tub +437:beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon +438:beacon, lighthouse, beacon light, pharos +439:beaker +440:bearskin, busby, shako +441:beer bottle +442:beer glass +443:bell cote, bell cot +444:bib +445:bicycle-built-for-two, tandem bicycle, tandem +446:bikini, two-piece +447:binder, ring-binder +448:binoculars, field glasses, opera glasses +449:birdhouse +450:boathouse +451:bobsled, bobsleigh, bob +452:bolo tie, bolo, bola tie, bola +453:bonnet, poke bonnet +454:bookcase +455:bookshop, bookstore, bookstall +456:bottlecap +457:bow +458:bow tie, bow-tie, bowtie +459:brass, memorial tablet, plaque +460:brassiere, bra, bandeau +461:breakwater, groin, groyne, mole, bulwark, seawall, jetty +462:breastplate, aegis, egis +463:broom +464:bucket, pail +465:buckle +466:bulletproof vest +467:bullet train, bullet +468:butcher shop, meat market +469:cab, hack, taxi, taxicab +470:caldron, cauldron +471:candle, taper, wax light +472:cannon +473:canoe +474:can opener, tin opener +475:cardigan +476:car mirror +477:carousel, carrousel, merry-go-round, roundabout, whirligig +478:carpenter's kit, tool kit +479:carton +480:car wheel +481:cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM +482:cassette +483:cassette player +484:castle +485:catamaran +486:CD player +487:cello, violoncello +488:cellular telephone, cellular phone, cellphone, cell, mobile phone +489:chain +490:chainlink fence +491:chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour +492:chain saw, chainsaw +493:chest +494:chiffonier, commode +495:chime, bell, gong +496:china cabinet, china closet +497:Christmas stocking +498:church, church building +499:cinema, movie theater, movie theatre, movie house, picture palace +500:cleaver, meat cleaver, chopper +501:cliff dwelling +502:cloak +503:clog, geta, patten, sabot +504:cocktail shaker +505:coffee mug +506:coffeepot +507:coil, spiral, volute, whorl, helix +508:combination lock +509:computer keyboard, keypad +510:confectionery, confectionary, candy store +511:container ship, containership, container vessel +512:convertible +513:corkscrew, bottle screw +514:cornet, horn, trumpet, trump +515:cowboy boot +516:cowboy hat, ten-gallon hat +517:cradle +518:crane +519:crash helmet +520:crate +521:crib, cot +522:Crock Pot +523:croquet ball +524:crutch +525:cuirass +526:dam, dike, dyke +527:desk +528:desktop computer +529:dial telephone, dial phone +530:diaper, nappy, napkin +531:digital clock +532:digital watch +533:dining table, board +534:dishrag, dishcloth +535:dishwasher, dish washer, dishwashing machine +536:disk brake, disc brake +537:dock, dockage, docking facility +538:dogsled, dog sled, dog sleigh +539:dome +540:doormat, welcome mat +541:drilling platform, offshore rig +542:drum, membranophone, tympan +543:drumstick +544:dumbbell +545:Dutch oven +546:electric fan, blower +547:electric guitar +548:electric locomotive +549:entertainment center +550:envelope +551:espresso maker +552:face powder +553:feather boa, boa +554:file, file cabinet, filing cabinet +555:fireboat +556:fire engine, fire truck +557:fire screen, fireguard +558:flagpole, flagstaff +559:flute, transverse flute +560:folding chair +561:football helmet +562:forklift +563:fountain +564:fountain pen +565:four-poster +566:freight car +567:French horn, horn +568:frying pan, frypan, skillet +569:fur coat +570:garbage truck, dustcart +571:gasmask, respirator, gas helmet +572:gas pump, gasoline pump, petrol pump, island dispenser +573:goblet +574:go-kart +575:golf ball +576:golfcart, golf cart +577:gondola +578:gong, tam-tam +579:gown +580:grand piano, grand +581:greenhouse, nursery, glasshouse +582:grille, radiator grille +583:grocery store, grocery, food market, market +584:guillotine +585:hair slide +586:hair spray +587:half track +588:hammer +589:hamper +590:hand blower, blow dryer, blow drier, hair dryer, hair drier +591:hand-held computer, hand-held microcomputer +592:handkerchief, hankie, hanky, hankey +593:hard disc, hard disk, fixed disk +594:harmonica, mouth organ, harp, mouth harp +595:harp +596:harvester, reaper +597:hatchet +598:holster +599:home theater, home theatre +600:honeycomb +601:hook, claw +602:hoopskirt, crinoline +603:horizontal bar, high bar +604:horse cart, horse-cart +605:hourglass +606:iPod +607:iron, smoothing iron +608:jack-o'-lantern +609:jean, blue jean, denim +610:jeep, landrover +611:jersey, T-shirt, tee shirt +612:jigsaw puzzle +613:jinrikisha, ricksha, rickshaw +614:joystick +615:kimono +616:knee pad +617:knot +618:lab coat, laboratory coat +619:ladle +620:lampshade, lamp shade +621:laptop, laptop computer +622:lawn mower, mower +623:lens cap, lens cover +624:letter opener, paper knife, paperknife +625:library +626:lifeboat +627:lighter, light, igniter, ignitor +628:limousine, limo +629:liner, ocean liner +630:lipstick, lip rouge +631:Loafer +632:lotion +633:loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +634:loupe, jeweler's loupe +635:lumbermill, sawmill +636:magnetic compass +637:mailbag, postbag +638:mailbox, letter box +639:maillot +640:maillot, tank suit +641:manhole cover +642:maraca +643:marimba, xylophone +644:mask +645:matchstick +646:maypole +647:maze, labyrinth +648:measuring cup +649:medicine chest, medicine cabinet +650:megalith, megalithic structure +651:microphone, mike +652:microwave, microwave oven +653:military uniform +654:milk can +655:minibus +656:miniskirt, mini +657:minivan +658:missile +659:mitten +660:mixing bowl +661:mobile home, manufactured home +662:Model T +663:modem +664:monastery +665:monitor +666:moped +667:mortar +668:mortarboard +669:mosque +670:mosquito net +671:motor scooter, scooter +672:mountain bike, all-terrain bike, off-roader +673:mountain tent +674:mouse, computer mouse +675:mousetrap +676:moving van +677:muzzle +678:nail +679:neck brace +680:necklace +681:nipple +682:notebook, notebook computer +683:obelisk +684:oboe, hautboy, hautbois +685:ocarina, sweet potato +686:odometer, hodometer, mileometer, milometer +687:oil filter +688:organ, pipe organ +689:oscilloscope, scope, cathode-ray oscilloscope, CRO +690:overskirt +691:oxcart +692:oxygen mask +693:packet +694:paddle, boat paddle +695:paddlewheel, paddle wheel +696:padlock +697:paintbrush +698:pajama, pyjama, pj's, jammies +699:palace +700:panpipe, pandean pipe, syrinx +701:paper towel +702:parachute, chute +703:parallel bars, bars +704:park bench +705:parking meter +706:passenger car, coach, carriage +707:patio, terrace +708:pay-phone, pay-station +709:pedestal, plinth, footstall +710:pencil box, pencil case +711:pencil sharpener +712:perfume, essence +713:Petri dish +714:photocopier +715:pick, plectrum, plectron +716:pickelhaube +717:picket fence, paling +718:pickup, pickup truck +719:pier +720:piggy bank, penny bank +721:pill bottle +722:pillow +723:ping-pong ball +724:pinwheel +725:pirate, pirate ship +726:pitcher, ewer +727:plane, carpenter's plane, woodworking plane +728:planetarium +729:plastic bag +730:plate rack +731:plow, plough +732:plunger, plumber's helper +733:Polaroid camera, Polaroid Land camera +734:pole +735:police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +736:poncho +737:pool table, billiard table, snooker table +738:pop bottle, soda bottle +739:pot, flowerpot +740:potter's wheel +741:power drill +742:prayer rug, prayer mat +743:printer +744:prison, prison house +745:projectile, missile +746:projector +747:puck, hockey puck +748:punching bag, punch bag, punching ball, punchball +749:purse +750:quill, quill pen +751:quilt, comforter, comfort, puff +752:racer, race car, racing car +753:racket, racquet +754:radiator +755:radio, wireless +756:radio telescope, radio reflector +757:rain barrel +758:recreational vehicle, RV, R.V. +759:reel +760:reflex camera +761:refrigerator, icebox +762:remote control, remote +763:restaurant, eating house, eating place, eatery +764:revolver, six-gun, six-shooter +765:rifle +766:rocking chair, rocker +767:rotisserie +768:rubber eraser, rubber, pencil eraser +769:rugby ball +770:rule, ruler +771:running shoe +772:safe +773:safety pin +774:saltshaker, salt shaker +775:sandal +776:sarong +777:sax, saxophone +778:scabbard +779:scale, weighing machine +780:school bus +781:schooner +782:scoreboard +783:screen, CRT screen +784:screw +785:screwdriver +786:seat belt, seatbelt +787:sewing machine +788:shield, buckler +789:shoe shop, shoe-shop, shoe store +790:shoji +791:shopping basket +792:shopping cart +793:shovel +794:shower cap +795:shower curtain +796:ski +797:ski mask +798:sleeping bag +799:slide rule, slipstick +800:sliding door +801:slot, one-armed bandit +802:snorkel +803:snowmobile +804:snowplow, snowplough +805:soap dispenser +806:soccer ball +807:sock +808:solar dish, solar collector, solar furnace +809:sombrero +810:soup bowl +811:space bar +812:space heater +813:space shuttle +814:spatula +815:speedboat +816:spider web, spider's web +817:spindle +818:sports car, sport car +819:spotlight, spot +820:stage +821:steam locomotive +822:steel arch bridge +823:steel drum +824:stethoscope +825:stole +826:stone wall +827:stopwatch, stop watch +828:stove +829:strainer +830:streetcar, tram, tramcar, trolley, trolley car +831:stretcher +832:studio couch, day bed +833:stupa, tope +834:submarine, pigboat, sub, U-boat +835:suit, suit of clothes +836:sundial +837:sunglass +838:sunglasses, dark glasses, shades +839:sunscreen, sunblock, sun blocker +840:suspension bridge +841:swab, swob, mop +842:sweatshirt +843:swimming trunks, bathing trunks +844:swing +845:switch, electric switch, electrical switch +846:syringe +847:table lamp +848:tank, army tank, armored combat vehicle, armoured combat vehicle +849:tape player +850:teapot +851:teddy, teddy bear +852:television, television system +853:tennis ball +854:thatch, thatched roof +855:theater curtain, theatre curtain +856:thimble +857:thresher, thrasher, threshing machine +858:throne +859:tile roof +860:toaster +861:tobacco shop, tobacconist shop, tobacconist +862:toilet seat +863:torch +864:totem pole +865:tow truck, tow car, wrecker +866:toyshop +867:tractor +868:trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +869:tray +870:trench coat +871:tricycle, trike, velocipede +872:trimaran +873:tripod +874:triumphal arch +875:trolleybus, trolley coach, trackless trolley +876:trombone +877:tub, vat +878:turnstile +879:typewriter keyboard +880:umbrella +881:unicycle, monocycle +882:upright, upright piano +883:vacuum, vacuum cleaner +884:vase +885:vault +886:velvet +887:vending machine +888:vestment +889:viaduct +890:violin, fiddle +891:volleyball +892:waffle iron +893:wall clock +894:wallet, billfold, notecase, pocketbook +895:wardrobe, closet, press +896:warplane, military plane +897:washbasin, handbasin, washbowl, lavabo, wash-hand basin +898:washer, automatic washer, washing machine +899:water bottle +900:water jug +901:water tower +902:whiskey jug +903:whistle +904:wig +905:window screen +906:window shade +907:Windsor tie +908:wine bottle +909:wing +910:wok +911:wooden spoon +912:wool, woolen, woollen +913:worm fence, snake fence, snake-rail fence, Virginia fence +914:wreck +915:yawl +916:yurt +917:web site, website, internet site, site +918:comic book +919:crossword puzzle, crossword +920:street sign +921:traffic light, traffic signal, stoplight +922:book jacket, dust cover, dust jacket, dust wrapper +923:menu +924:plate +925:guacamole +926:consomme +927:hot pot, hotpot +928:trifle +929:ice cream, icecream +930:ice lolly, lolly, lollipop, popsicle +931:French loaf +932:bagel, beigel +933:pretzel +934:cheeseburger +935:hotdog, hot dog, red hot +936:mashed potato +937:head cabbage +938:broccoli +939:cauliflower +940:zucchini, courgette +941:spaghetti squash +942:acorn squash +943:butternut squash +944:cucumber, cuke +945:artichoke, globe artichoke +946:bell pepper +947:cardoon +948:mushroom +949:Granny Smith +950:strawberry +951:orange +952:lemon +953:fig +954:pineapple, ananas +955:banana +956:jackfruit, jak, jack +957:custard apple +958:pomegranate +959:hay +960:carbonara +961:chocolate sauce, chocolate syrup +962:dough +963:meat loaf, meatloaf +964:pizza, pizza pie +965:potpie +966:burrito +967:red wine +968:espresso +969:cup +970:eggnog +971:alp +972:bubble +973:cliff, drop, drop-off +974:coral reef +975:geyser +976:lakeside, lakeshore +977:promontory, headland, head, foreland +978:sandbar, sand bar +979:seashore, coast, seacoast, sea-coast +980:valley, vale +981:volcano +982:ballplayer, baseball player +983:groom, bridegroom +984:scuba diver +985:rapeseed +986:daisy +987:yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum +988:corn +989:acorn +990:hip, rose hip, rosehip +991:buckeye, horse chestnut, conker +992:coral fungus +993:agaric +994:gyromitra +995:stinkhorn, carrion fungus +996:earthstar +997:hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +998:bolete +999:ear, spike, capitulum +1000:toilet tissue, toilet paper, bathroom tissue diff --git a/evaluation/CLEVER/labels/synset_words.txt b/evaluation/CLEVER/labels/synset_words.txt new file mode 100644 index 0000000..a9e8c7f --- /dev/null +++ b/evaluation/CLEVER/labels/synset_words.txt @@ -0,0 +1,1000 @@ +n01440764 tench, Tinca tinca +n01443537 goldfish, Carassius auratus +n01484850 great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias +n01491361 tiger shark, Galeocerdo cuvieri +n01494475 hammerhead, hammerhead shark +n01496331 electric ray, crampfish, numbfish, torpedo +n01498041 stingray +n01514668 cock +n01514859 hen +n01518878 ostrich, Struthio camelus +n01530575 brambling, Fringilla montifringilla +n01531178 goldfinch, Carduelis carduelis +n01532829 house finch, linnet, Carpodacus mexicanus +n01534433 junco, snowbird +n01537544 indigo bunting, indigo finch, indigo bird, Passerina cyanea +n01558993 robin, American robin, Turdus migratorius +n01560419 bulbul +n01580077 jay +n01582220 magpie +n01592084 chickadee +n01601694 water ouzel, dipper +n01608432 kite +n01614925 bald eagle, American eagle, Haliaeetus leucocephalus +n01616318 vulture +n01622779 great grey owl, great gray owl, Strix nebulosa +n01629819 European fire salamander, Salamandra salamandra +n01630670 common newt, Triturus vulgaris +n01631663 eft +n01632458 spotted salamander, Ambystoma maculatum +n01632777 axolotl, mud puppy, Ambystoma mexicanum +n01641577 bullfrog, Rana catesbeiana +n01644373 tree frog, tree-frog +n01644900 tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +n01664065 loggerhead, loggerhead turtle, Caretta caretta +n01665541 leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +n01667114 mud turtle +n01667778 terrapin +n01669191 box turtle, box tortoise +n01675722 banded gecko +n01677366 common iguana, iguana, Iguana iguana +n01682714 American chameleon, anole, Anolis carolinensis +n01685808 whiptail, whiptail lizard +n01687978 agama +n01688243 frilled lizard, Chlamydosaurus kingi +n01689811 alligator lizard +n01692333 Gila monster, Heloderma suspectum +n01693334 green lizard, Lacerta viridis +n01694178 African chameleon, Chamaeleo chamaeleon +n01695060 Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +n01697457 African crocodile, Nile crocodile, Crocodylus niloticus +n01698640 American alligator, Alligator mississipiensis +n01704323 triceratops +n01728572 thunder snake, worm snake, Carphophis amoenus +n01728920 ringneck snake, ring-necked snake, ring snake +n01729322 hognose snake, puff adder, sand viper +n01729977 green snake, grass snake +n01734418 king snake, kingsnake +n01735189 garter snake, grass snake +n01737021 water snake +n01739381 vine snake +n01740131 night snake, Hypsiglena torquata +n01742172 boa constrictor, Constrictor constrictor +n01744401 rock python, rock snake, Python sebae +n01748264 Indian cobra, Naja naja +n01749939 green mamba +n01751748 sea snake +n01753488 horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +n01755581 diamondback, diamondback rattlesnake, Crotalus adamanteus +n01756291 sidewinder, horned rattlesnake, Crotalus cerastes +n01768244 trilobite +n01770081 harvestman, daddy longlegs, Phalangium opilio +n01770393 scorpion +n01773157 black and gold garden spider, Argiope aurantia +n01773549 barn spider, Araneus cavaticus +n01773797 garden spider, Aranea diademata +n01774384 black widow, Latrodectus mactans +n01774750 tarantula +n01775062 wolf spider, hunting spider +n01776313 tick +n01784675 centipede +n01795545 black grouse +n01796340 ptarmigan +n01797886 ruffed grouse, partridge, Bonasa umbellus +n01798484 prairie chicken, prairie grouse, prairie fowl +n01806143 peacock +n01806567 quail +n01807496 partridge +n01817953 African grey, African gray, Psittacus erithacus +n01818515 macaw +n01819313 sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +n01820546 lorikeet +n01824575 coucal +n01828970 bee eater +n01829413 hornbill +n01833805 hummingbird +n01843065 jacamar +n01843383 toucan +n01847000 drake +n01855032 red-breasted merganser, Mergus serrator +n01855672 goose +n01860187 black swan, Cygnus atratus +n01871265 tusker +n01872401 echidna, spiny anteater, anteater +n01873310 platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus +n01877812 wallaby, brush kangaroo +n01882714 koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +n01883070 wombat +n01910747 jellyfish +n01914609 sea anemone, anemone +n01917289 brain coral +n01924916 flatworm, platyhelminth +n01930112 nematode, nematode worm, roundworm +n01943899 conch +n01944390 snail +n01945685 slug +n01950731 sea slug, nudibranch +n01955084 chiton, coat-of-mail shell, sea cradle, polyplacophore +n01968897 chambered nautilus, pearly nautilus, nautilus +n01978287 Dungeness crab, Cancer magister +n01978455 rock crab, Cancer irroratus +n01980166 fiddler crab +n01981276 king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica +n01983481 American lobster, Northern lobster, Maine lobster, Homarus americanus +n01984695 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +n01985128 crayfish, crawfish, crawdad, crawdaddy +n01986214 hermit crab +n01990800 isopod +n02002556 white stork, Ciconia ciconia +n02002724 black stork, Ciconia nigra +n02006656 spoonbill +n02007558 flamingo +n02009229 little blue heron, Egretta caerulea +n02009912 American egret, great white heron, Egretta albus +n02011460 bittern +n02012849 crane +n02013706 limpkin, Aramus pictus +n02017213 European gallinule, Porphyrio porphyrio +n02018207 American coot, marsh hen, mud hen, water hen, Fulica americana +n02018795 bustard +n02025239 ruddy turnstone, Arenaria interpres +n02027492 red-backed sandpiper, dunlin, Erolia alpina +n02028035 redshank, Tringa totanus +n02033041 dowitcher +n02037110 oystercatcher, oyster catcher +n02051845 pelican +n02056570 king penguin, Aptenodytes patagonica +n02058221 albatross, mollymawk +n02066245 grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus +n02071294 killer whale, killer, orca, grampus, sea wolf, Orcinus orca +n02074367 dugong, Dugong dugon +n02077923 sea lion +n02085620 Chihuahua +n02085782 Japanese spaniel +n02085936 Maltese dog, Maltese terrier, Maltese +n02086079 Pekinese, Pekingese, Peke +n02086240 Shih-Tzu +n02086646 Blenheim spaniel +n02086910 papillon +n02087046 toy terrier +n02087394 Rhodesian ridgeback +n02088094 Afghan hound, Afghan +n02088238 basset, basset hound +n02088364 beagle +n02088466 bloodhound, sleuthhound +n02088632 bluetick +n02089078 black-and-tan coonhound +n02089867 Walker hound, Walker foxhound +n02089973 English foxhound +n02090379 redbone +n02090622 borzoi, Russian wolfhound +n02090721 Irish wolfhound +n02091032 Italian greyhound +n02091134 whippet +n02091244 Ibizan hound, Ibizan Podenco +n02091467 Norwegian elkhound, elkhound +n02091635 otterhound, otter hound +n02091831 Saluki, gazelle hound +n02092002 Scottish deerhound, deerhound +n02092339 Weimaraner +n02093256 Staffordshire bullterrier, Staffordshire bull terrier +n02093428 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier +n02093647 Bedlington terrier +n02093754 Border terrier +n02093859 Kerry blue terrier +n02093991 Irish terrier +n02094114 Norfolk terrier +n02094258 Norwich terrier +n02094433 Yorkshire terrier +n02095314 wire-haired fox terrier +n02095570 Lakeland terrier +n02095889 Sealyham terrier, Sealyham +n02096051 Airedale, Airedale terrier +n02096177 cairn, cairn terrier +n02096294 Australian terrier +n02096437 Dandie Dinmont, Dandie Dinmont terrier +n02096585 Boston bull, Boston terrier +n02097047 miniature schnauzer +n02097130 giant schnauzer +n02097209 standard schnauzer +n02097298 Scotch terrier, Scottish terrier, Scottie +n02097474 Tibetan terrier, chrysanthemum dog +n02097658 silky terrier, Sydney silky +n02098105 soft-coated wheaten terrier +n02098286 West Highland white terrier +n02098413 Lhasa, Lhasa apso +n02099267 flat-coated retriever +n02099429 curly-coated retriever +n02099601 golden retriever +n02099712 Labrador retriever +n02099849 Chesapeake Bay retriever +n02100236 German short-haired pointer +n02100583 vizsla, Hungarian pointer +n02100735 English setter +n02100877 Irish setter, red setter +n02101006 Gordon setter +n02101388 Brittany spaniel +n02101556 clumber, clumber spaniel +n02102040 English springer, English springer spaniel +n02102177 Welsh springer spaniel +n02102318 cocker spaniel, English cocker spaniel, cocker +n02102480 Sussex spaniel +n02102973 Irish water spaniel +n02104029 kuvasz +n02104365 schipperke +n02105056 groenendael +n02105162 malinois +n02105251 briard +n02105412 kelpie +n02105505 komondor +n02105641 Old English sheepdog, bobtail +n02105855 Shetland sheepdog, Shetland sheep dog, Shetland +n02106030 collie +n02106166 Border collie +n02106382 Bouvier des Flandres, Bouviers des Flandres +n02106550 Rottweiler +n02106662 German shepherd, German shepherd dog, German police dog, alsatian +n02107142 Doberman, Doberman pinscher +n02107312 miniature pinscher +n02107574 Greater Swiss Mountain dog +n02107683 Bernese mountain dog +n02107908 Appenzeller +n02108000 EntleBucher +n02108089 boxer +n02108422 bull mastiff +n02108551 Tibetan mastiff +n02108915 French bulldog +n02109047 Great Dane +n02109525 Saint Bernard, St Bernard +n02109961 Eskimo dog, husky +n02110063 malamute, malemute, Alaskan malamute +n02110185 Siberian husky +n02110341 dalmatian, coach dog, carriage dog +n02110627 affenpinscher, monkey pinscher, monkey dog +n02110806 basenji +n02110958 pug, pug-dog +n02111129 Leonberg +n02111277 Newfoundland, Newfoundland dog +n02111500 Great Pyrenees +n02111889 Samoyed, Samoyede +n02112018 Pomeranian +n02112137 chow, chow chow +n02112350 keeshond +n02112706 Brabancon griffon +n02113023 Pembroke, Pembroke Welsh corgi +n02113186 Cardigan, Cardigan Welsh corgi +n02113624 toy poodle +n02113712 miniature poodle +n02113799 standard poodle +n02113978 Mexican hairless +n02114367 timber wolf, grey wolf, gray wolf, Canis lupus +n02114548 white wolf, Arctic wolf, Canis lupus tundrarum +n02114712 red wolf, maned wolf, Canis rufus, Canis niger +n02114855 coyote, prairie wolf, brush wolf, Canis latrans +n02115641 dingo, warrigal, warragal, Canis dingo +n02115913 dhole, Cuon alpinus +n02116738 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +n02117135 hyena, hyaena +n02119022 red fox, Vulpes vulpes +n02119789 kit fox, Vulpes macrotis +n02120079 Arctic fox, white fox, Alopex lagopus +n02120505 grey fox, gray fox, Urocyon cinereoargenteus +n02123045 tabby, tabby cat +n02123159 tiger cat +n02123394 Persian cat +n02123597 Siamese cat, Siamese +n02124075 Egyptian cat +n02125311 cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +n02127052 lynx, catamount +n02128385 leopard, Panthera pardus +n02128757 snow leopard, ounce, Panthera uncia +n02128925 jaguar, panther, Panthera onca, Felis onca +n02129165 lion, king of beasts, Panthera leo +n02129604 tiger, Panthera tigris +n02130308 cheetah, chetah, Acinonyx jubatus +n02132136 brown bear, bruin, Ursus arctos +n02133161 American black bear, black bear, Ursus americanus, Euarctos americanus +n02134084 ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +n02134418 sloth bear, Melursus ursinus, Ursus ursinus +n02137549 mongoose +n02138441 meerkat, mierkat +n02165105 tiger beetle +n02165456 ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +n02167151 ground beetle, carabid beetle +n02168699 long-horned beetle, longicorn, longicorn beetle +n02169497 leaf beetle, chrysomelid +n02172182 dung beetle +n02174001 rhinoceros beetle +n02177972 weevil +n02190166 fly +n02206856 bee +n02219486 ant, emmet, pismire +n02226429 grasshopper, hopper +n02229544 cricket +n02231487 walking stick, walkingstick, stick insect +n02233338 cockroach, roach +n02236044 mantis, mantid +n02256656 cicada, cicala +n02259212 leafhopper +n02264363 lacewing, lacewing fly +n02268443 dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk +n02268853 damselfly +n02276258 admiral +n02277742 ringlet, ringlet butterfly +n02279972 monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +n02280649 cabbage butterfly +n02281406 sulphur butterfly, sulfur butterfly +n02281787 lycaenid, lycaenid butterfly +n02317335 starfish, sea star +n02319095 sea urchin +n02321529 sea cucumber, holothurian +n02325366 wood rabbit, cottontail, cottontail rabbit +n02326432 hare +n02328150 Angora, Angora rabbit +n02342885 hamster +n02346627 porcupine, hedgehog +n02356798 fox squirrel, eastern fox squirrel, Sciurus niger +n02361337 marmot +n02363005 beaver +n02364673 guinea pig, Cavia cobaya +n02389026 sorrel +n02391049 zebra +n02395406 hog, pig, grunter, squealer, Sus scrofa +n02396427 wild boar, boar, Sus scrofa +n02397096 warthog +n02398521 hippopotamus, hippo, river horse, Hippopotamus amphibius +n02403003 ox +n02408429 water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +n02410509 bison +n02412080 ram, tup +n02415577 bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis +n02417914 ibex, Capra ibex +n02422106 hartebeest +n02422699 impala, Aepyceros melampus +n02423022 gazelle +n02437312 Arabian camel, dromedary, Camelus dromedarius +n02437616 llama +n02441942 weasel +n02442845 mink +n02443114 polecat, fitch, foulmart, foumart, Mustela putorius +n02443484 black-footed ferret, ferret, Mustela nigripes +n02444819 otter +n02445715 skunk, polecat, wood pussy +n02447366 badger +n02454379 armadillo +n02457408 three-toed sloth, ai, Bradypus tridactylus +n02480495 orangutan, orang, orangutang, Pongo pygmaeus +n02480855 gorilla, Gorilla gorilla +n02481823 chimpanzee, chimp, Pan troglodytes +n02483362 gibbon, Hylobates lar +n02483708 siamang, Hylobates syndactylus, Symphalangus syndactylus +n02484975 guenon, guenon monkey +n02486261 patas, hussar monkey, Erythrocebus patas +n02486410 baboon +n02487347 macaque +n02488291 langur +n02488702 colobus, colobus monkey +n02489166 proboscis monkey, Nasalis larvatus +n02490219 marmoset +n02492035 capuchin, ringtail, Cebus capucinus +n02492660 howler monkey, howler +n02493509 titi, titi monkey +n02493793 spider monkey, Ateles geoffroyi +n02494079 squirrel monkey, Saimiri sciureus +n02497673 Madagascar cat, ring-tailed lemur, Lemur catta +n02500267 indri, indris, Indri indri, Indri brevicaudatus +n02504013 Indian elephant, Elephas maximus +n02504458 African elephant, Loxodonta africana +n02509815 lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +n02510455 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +n02514041 barracouta, snoek +n02526121 eel +n02536864 coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +n02606052 rock beauty, Holocanthus tricolor +n02607072 anemone fish +n02640242 sturgeon +n02641379 gar, garfish, garpike, billfish, Lepisosteus osseus +n02643566 lionfish +n02655020 puffer, pufferfish, blowfish, globefish +n02666196 abacus +n02667093 abaya +n02669723 academic gown, academic robe, judge's robe +n02672831 accordion, piano accordion, squeeze box +n02676566 acoustic guitar +n02687172 aircraft carrier, carrier, flattop, attack aircraft carrier +n02690373 airliner +n02692877 airship, dirigible +n02699494 altar +n02701002 ambulance +n02704792 amphibian, amphibious vehicle +n02708093 analog clock +n02727426 apiary, bee house +n02730930 apron +n02747177 ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin +n02749479 assault rifle, assault gun +n02769748 backpack, back pack, knapsack, packsack, rucksack, haversack +n02776631 bakery, bakeshop, bakehouse +n02777292 balance beam, beam +n02782093 balloon +n02783161 ballpoint, ballpoint pen, ballpen, Biro +n02786058 Band Aid +n02787622 banjo +n02788148 bannister, banister, balustrade, balusters, handrail +n02790996 barbell +n02791124 barber chair +n02791270 barbershop +n02793495 barn +n02794156 barometer +n02795169 barrel, cask +n02797295 barrow, garden cart, lawn cart, wheelbarrow +n02799071 baseball +n02802426 basketball +n02804414 bassinet +n02804610 bassoon +n02807133 bathing cap, swimming cap +n02808304 bath towel +n02808440 bathtub, bathing tub, bath, tub +n02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon +n02814860 beacon, lighthouse, beacon light, pharos +n02815834 beaker +n02817516 bearskin, busby, shako +n02823428 beer bottle +n02823750 beer glass +n02825657 bell cote, bell cot +n02834397 bib +n02835271 bicycle-built-for-two, tandem bicycle, tandem +n02837789 bikini, two-piece +n02840245 binder, ring-binder +n02841315 binoculars, field glasses, opera glasses +n02843684 birdhouse +n02859443 boathouse +n02860847 bobsled, bobsleigh, bob +n02865351 bolo tie, bolo, bola tie, bola +n02869837 bonnet, poke bonnet +n02870880 bookcase +n02871525 bookshop, bookstore, bookstall +n02877765 bottlecap +n02879718 bow +n02883205 bow tie, bow-tie, bowtie +n02892201 brass, memorial tablet, plaque +n02892767 brassiere, bra, bandeau +n02894605 breakwater, groin, groyne, mole, bulwark, seawall, jetty +n02895154 breastplate, aegis, egis +n02906734 broom +n02909870 bucket, pail +n02910353 buckle +n02916936 bulletproof vest +n02917067 bullet train, bullet +n02927161 butcher shop, meat market +n02930766 cab, hack, taxi, taxicab +n02939185 caldron, cauldron +n02948072 candle, taper, wax light +n02950826 cannon +n02951358 canoe +n02951585 can opener, tin opener +n02963159 cardigan +n02965783 car mirror +n02966193 carousel, carrousel, merry-go-round, roundabout, whirligig +n02966687 carpenter's kit, tool kit +n02971356 carton +n02974003 car wheel +n02977058 cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM +n02978881 cassette +n02979186 cassette player +n02980441 castle +n02981792 catamaran +n02988304 CD player +n02992211 cello, violoncello +n02992529 cellular telephone, cellular phone, cellphone, cell, mobile phone +n02999410 chain +n03000134 chainlink fence +n03000247 chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour +n03000684 chain saw, chainsaw +n03014705 chest +n03016953 chiffonier, commode +n03017168 chime, bell, gong +n03018349 china cabinet, china closet +n03026506 Christmas stocking +n03028079 church, church building +n03032252 cinema, movie theater, movie theatre, movie house, picture palace +n03041632 cleaver, meat cleaver, chopper +n03042490 cliff dwelling +n03045698 cloak +n03047690 clog, geta, patten, sabot +n03062245 cocktail shaker +n03063599 coffee mug +n03063689 coffeepot +n03065424 coil, spiral, volute, whorl, helix +n03075370 combination lock +n03085013 computer keyboard, keypad +n03089624 confectionery, confectionary, candy store +n03095699 container ship, containership, container vessel +n03100240 convertible +n03109150 corkscrew, bottle screw +n03110669 cornet, horn, trumpet, trump +n03124043 cowboy boot +n03124170 cowboy hat, ten-gallon hat +n03125729 cradle +n03126707 crane +n03127747 crash helmet +n03127925 crate +n03131574 crib, cot +n03133878 Crock Pot +n03134739 croquet ball +n03141823 crutch +n03146219 cuirass +n03160309 dam, dike, dyke +n03179701 desk +n03180011 desktop computer +n03187595 dial telephone, dial phone +n03188531 diaper, nappy, napkin +n03196217 digital clock +n03197337 digital watch +n03201208 dining table, board +n03207743 dishrag, dishcloth +n03207941 dishwasher, dish washer, dishwashing machine +n03208938 disk brake, disc brake +n03216828 dock, dockage, docking facility +n03218198 dogsled, dog sled, dog sleigh +n03220513 dome +n03223299 doormat, welcome mat +n03240683 drilling platform, offshore rig +n03249569 drum, membranophone, tympan +n03250847 drumstick +n03255030 dumbbell +n03259280 Dutch oven +n03271574 electric fan, blower +n03272010 electric guitar +n03272562 electric locomotive +n03290653 entertainment center +n03291819 envelope +n03297495 espresso maker +n03314780 face powder +n03325584 feather boa, boa +n03337140 file, file cabinet, filing cabinet +n03344393 fireboat +n03345487 fire engine, fire truck +n03347037 fire screen, fireguard +n03355925 flagpole, flagstaff +n03372029 flute, transverse flute +n03376595 folding chair +n03379051 football helmet +n03384352 forklift +n03388043 fountain +n03388183 fountain pen +n03388549 four-poster +n03393912 freight car +n03394916 French horn, horn +n03400231 frying pan, frypan, skillet +n03404251 fur coat +n03417042 garbage truck, dustcart +n03424325 gasmask, respirator, gas helmet +n03425413 gas pump, gasoline pump, petrol pump, island dispenser +n03443371 goblet +n03444034 go-kart +n03445777 golf ball +n03445924 golfcart, golf cart +n03447447 gondola +n03447721 gong, tam-tam +n03450230 gown +n03452741 grand piano, grand +n03457902 greenhouse, nursery, glasshouse +n03459775 grille, radiator grille +n03461385 grocery store, grocery, food market, market +n03467068 guillotine +n03476684 hair slide +n03476991 hair spray +n03478589 half track +n03481172 hammer +n03482405 hamper +n03483316 hand blower, blow dryer, blow drier, hair dryer, hair drier +n03485407 hand-held computer, hand-held microcomputer +n03485794 handkerchief, hankie, hanky, hankey +n03492542 hard disc, hard disk, fixed disk +n03494278 harmonica, mouth organ, harp, mouth harp +n03495258 harp +n03496892 harvester, reaper +n03498962 hatchet +n03527444 holster +n03529860 home theater, home theatre +n03530642 honeycomb +n03532672 hook, claw +n03534580 hoopskirt, crinoline +n03535780 horizontal bar, high bar +n03538406 horse cart, horse-cart +n03544143 hourglass +n03584254 iPod +n03584829 iron, smoothing iron +n03590841 jack-o'-lantern +n03594734 jean, blue jean, denim +n03594945 jeep, landrover +n03595614 jersey, T-shirt, tee shirt +n03598930 jigsaw puzzle +n03599486 jinrikisha, ricksha, rickshaw +n03602883 joystick +n03617480 kimono +n03623198 knee pad +n03627232 knot +n03630383 lab coat, laboratory coat +n03633091 ladle +n03637318 lampshade, lamp shade +n03642806 laptop, laptop computer +n03649909 lawn mower, mower +n03657121 lens cap, lens cover +n03658185 letter opener, paper knife, paperknife +n03661043 library +n03662601 lifeboat +n03666591 lighter, light, igniter, ignitor +n03670208 limousine, limo +n03673027 liner, ocean liner +n03676483 lipstick, lip rouge +n03680355 Loafer +n03690938 lotion +n03691459 loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +n03692522 loupe, jeweler's loupe +n03697007 lumbermill, sawmill +n03706229 magnetic compass +n03709823 mailbag, postbag +n03710193 mailbox, letter box +n03710637 maillot +n03710721 maillot, tank suit +n03717622 manhole cover +n03720891 maraca +n03721384 marimba, xylophone +n03724870 mask +n03729826 matchstick +n03733131 maypole +n03733281 maze, labyrinth +n03733805 measuring cup +n03742115 medicine chest, medicine cabinet +n03743016 megalith, megalithic structure +n03759954 microphone, mike +n03761084 microwave, microwave oven +n03763968 military uniform +n03764736 milk can +n03769881 minibus +n03770439 miniskirt, mini +n03770679 minivan +n03773504 missile +n03775071 mitten +n03775546 mixing bowl +n03776460 mobile home, manufactured home +n03777568 Model T +n03777754 modem +n03781244 monastery +n03782006 monitor +n03785016 moped +n03786901 mortar +n03787032 mortarboard +n03788195 mosque +n03788365 mosquito net +n03791053 motor scooter, scooter +n03792782 mountain bike, all-terrain bike, off-roader +n03792972 mountain tent +n03793489 mouse, computer mouse +n03794056 mousetrap +n03796401 moving van +n03803284 muzzle +n03804744 nail +n03814639 neck brace +n03814906 necklace +n03825788 nipple +n03832673 notebook, notebook computer +n03837869 obelisk +n03838899 oboe, hautboy, hautbois +n03840681 ocarina, sweet potato +n03841143 odometer, hodometer, mileometer, milometer +n03843555 oil filter +n03854065 organ, pipe organ +n03857828 oscilloscope, scope, cathode-ray oscilloscope, CRO +n03866082 overskirt +n03868242 oxcart +n03868863 oxygen mask +n03871628 packet +n03873416 paddle, boat paddle +n03874293 paddlewheel, paddle wheel +n03874599 padlock +n03876231 paintbrush +n03877472 pajama, pyjama, pj's, jammies +n03877845 palace +n03884397 panpipe, pandean pipe, syrinx +n03887697 paper towel +n03888257 parachute, chute +n03888605 parallel bars, bars +n03891251 park bench +n03891332 parking meter +n03895866 passenger car, coach, carriage +n03899768 patio, terrace +n03902125 pay-phone, pay-station +n03903868 pedestal, plinth, footstall +n03908618 pencil box, pencil case +n03908714 pencil sharpener +n03916031 perfume, essence +n03920288 Petri dish +n03924679 photocopier +n03929660 pick, plectrum, plectron +n03929855 pickelhaube +n03930313 picket fence, paling +n03930630 pickup, pickup truck +n03933933 pier +n03935335 piggy bank, penny bank +n03937543 pill bottle +n03938244 pillow +n03942813 ping-pong ball +n03944341 pinwheel +n03947888 pirate, pirate ship +n03950228 pitcher, ewer +n03954731 plane, carpenter's plane, woodworking plane +n03956157 planetarium +n03958227 plastic bag +n03961711 plate rack +n03967562 plow, plough +n03970156 plunger, plumber's helper +n03976467 Polaroid camera, Polaroid Land camera +n03976657 pole +n03977966 police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +n03980874 poncho +n03982430 pool table, billiard table, snooker table +n03983396 pop bottle, soda bottle +n03991062 pot, flowerpot +n03992509 potter's wheel +n03995372 power drill +n03998194 prayer rug, prayer mat +n04004767 printer +n04005630 prison, prison house +n04008634 projectile, missile +n04009552 projector +n04019541 puck, hockey puck +n04023962 punching bag, punch bag, punching ball, punchball +n04026417 purse +n04033901 quill, quill pen +n04033995 quilt, comforter, comfort, puff +n04037443 racer, race car, racing car +n04039381 racket, racquet +n04040759 radiator +n04041544 radio, wireless +n04044716 radio telescope, radio reflector +n04049303 rain barrel +n04065272 recreational vehicle, RV, R.V. +n04067472 reel +n04069434 reflex camera +n04070727 refrigerator, icebox +n04074963 remote control, remote +n04081281 restaurant, eating house, eating place, eatery +n04086273 revolver, six-gun, six-shooter +n04090263 rifle +n04099969 rocking chair, rocker +n04111531 rotisserie +n04116512 rubber eraser, rubber, pencil eraser +n04118538 rugby ball +n04118776 rule, ruler +n04120489 running shoe +n04125021 safe +n04127249 safety pin +n04131690 saltshaker, salt shaker +n04133789 sandal +n04136333 sarong +n04141076 sax, saxophone +n04141327 scabbard +n04141975 scale, weighing machine +n04146614 school bus +n04147183 schooner +n04149813 scoreboard +n04152593 screen, CRT screen +n04153751 screw +n04154565 screwdriver +n04162706 seat belt, seatbelt +n04179913 sewing machine +n04192698 shield, buckler +n04200800 shoe shop, shoe-shop, shoe store +n04201297 shoji +n04204238 shopping basket +n04204347 shopping cart +n04208210 shovel +n04209133 shower cap +n04209239 shower curtain +n04228054 ski +n04229816 ski mask +n04235860 sleeping bag +n04238763 slide rule, slipstick +n04239074 sliding door +n04243546 slot, one-armed bandit +n04251144 snorkel +n04252077 snowmobile +n04252225 snowplow, snowplough +n04254120 soap dispenser +n04254680 soccer ball +n04254777 sock +n04258138 solar dish, solar collector, solar furnace +n04259630 sombrero +n04263257 soup bowl +n04264628 space bar +n04265275 space heater +n04266014 space shuttle +n04270147 spatula +n04273569 speedboat +n04275548 spider web, spider's web +n04277352 spindle +n04285008 sports car, sport car +n04286575 spotlight, spot +n04296562 stage +n04310018 steam locomotive +n04311004 steel arch bridge +n04311174 steel drum +n04317175 stethoscope +n04325704 stole +n04326547 stone wall +n04328186 stopwatch, stop watch +n04330267 stove +n04332243 strainer +n04335435 streetcar, tram, tramcar, trolley, trolley car +n04336792 stretcher +n04344873 studio couch, day bed +n04346328 stupa, tope +n04347754 submarine, pigboat, sub, U-boat +n04350905 suit, suit of clothes +n04355338 sundial +n04355933 sunglass +n04356056 sunglasses, dark glasses, shades +n04357314 sunscreen, sunblock, sun blocker +n04366367 suspension bridge +n04367480 swab, swob, mop +n04370456 sweatshirt +n04371430 swimming trunks, bathing trunks +n04371774 swing +n04372370 switch, electric switch, electrical switch +n04376876 syringe +n04380533 table lamp +n04389033 tank, army tank, armored combat vehicle, armoured combat vehicle +n04392985 tape player +n04398044 teapot +n04399382 teddy, teddy bear +n04404412 television, television system +n04409515 tennis ball +n04417672 thatch, thatched roof +n04418357 theater curtain, theatre curtain +n04423845 thimble +n04428191 thresher, thrasher, threshing machine +n04429376 throne +n04435653 tile roof +n04442312 toaster +n04443257 tobacco shop, tobacconist shop, tobacconist +n04447861 toilet seat +n04456115 torch +n04458633 totem pole +n04461696 tow truck, tow car, wrecker +n04462240 toyshop +n04465501 tractor +n04467665 trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +n04476259 tray +n04479046 trench coat +n04482393 tricycle, trike, velocipede +n04483307 trimaran +n04485082 tripod +n04486054 triumphal arch +n04487081 trolleybus, trolley coach, trackless trolley +n04487394 trombone +n04493381 tub, vat +n04501370 turnstile +n04505470 typewriter keyboard +n04507155 umbrella +n04509417 unicycle, monocycle +n04515003 upright, upright piano +n04517823 vacuum, vacuum cleaner +n04522168 vase +n04523525 vault +n04525038 velvet +n04525305 vending machine +n04532106 vestment +n04532670 viaduct +n04536866 violin, fiddle +n04540053 volleyball +n04542943 waffle iron +n04548280 wall clock +n04548362 wallet, billfold, notecase, pocketbook +n04550184 wardrobe, closet, press +n04552348 warplane, military plane +n04553703 washbasin, handbasin, washbowl, lavabo, wash-hand basin +n04554684 washer, automatic washer, washing machine +n04557648 water bottle +n04560804 water jug +n04562935 water tower +n04579145 whiskey jug +n04579432 whistle +n04584207 wig +n04589890 window screen +n04590129 window shade +n04591157 Windsor tie +n04591713 wine bottle +n04592741 wing +n04596742 wok +n04597913 wooden spoon +n04599235 wool, woolen, woollen +n04604644 worm fence, snake fence, snake-rail fence, Virginia fence +n04606251 wreck +n04612504 yawl +n04613696 yurt +n06359193 web site, website, internet site, site +n06596364 comic book +n06785654 crossword puzzle, crossword +n06794110 street sign +n06874185 traffic light, traffic signal, stoplight +n07248320 book jacket, dust cover, dust jacket, dust wrapper +n07565083 menu +n07579787 plate +n07583066 guacamole +n07584110 consomme +n07590611 hot pot, hotpot +n07613480 trifle +n07614500 ice cream, icecream +n07615774 ice lolly, lolly, lollipop, popsicle +n07684084 French loaf +n07693725 bagel, beigel +n07695742 pretzel +n07697313 cheeseburger +n07697537 hotdog, hot dog, red hot +n07711569 mashed potato +n07714571 head cabbage +n07714990 broccoli +n07715103 cauliflower +n07716358 zucchini, courgette +n07716906 spaghetti squash +n07717410 acorn squash +n07717556 butternut squash +n07718472 cucumber, cuke +n07718747 artichoke, globe artichoke +n07720875 bell pepper +n07730033 cardoon +n07734744 mushroom +n07742313 Granny Smith +n07745940 strawberry +n07747607 orange +n07749582 lemon +n07753113 fig +n07753275 pineapple, ananas +n07753592 banana +n07754684 jackfruit, jak, jack +n07760859 custard apple +n07768694 pomegranate +n07802026 hay +n07831146 carbonara +n07836838 chocolate sauce, chocolate syrup +n07860988 dough +n07871810 meat loaf, meatloaf +n07873807 pizza, pizza pie +n07875152 potpie +n07880968 burrito +n07892512 red wine +n07920052 espresso +n07930864 cup +n07932039 eggnog +n09193705 alp +n09229709 bubble +n09246464 cliff, drop, drop-off +n09256479 coral reef +n09288635 geyser +n09332890 lakeside, lakeshore +n09399592 promontory, headland, head, foreland +n09421951 sandbar, sand bar +n09428293 seashore, coast, seacoast, sea-coast +n09468604 valley, vale +n09472597 volcano +n09835506 ballplayer, baseball player +n10148035 groom, bridegroom +n10565667 scuba diver +n11879895 rapeseed +n11939491 daisy +n12057211 yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum +n12144580 corn +n12267677 acorn +n12620546 hip, rose hip, rosehip +n12768682 buckeye, horse chestnut, conker +n12985857 coral fungus +n12998815 agaric +n13037406 gyromitra +n13040303 stinkhorn, carrion fungus +n13044778 earthstar +n13052670 hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +n13054560 bolete +n13133613 ear, spike, capitulum +n15075141 toilet tissue, toilet paper, bathroom tissue diff --git a/evaluation/CLEVER/nlayer_model.py b/evaluation/CLEVER/nlayer_model.py new file mode 100644 index 0000000..6da963e --- /dev/null +++ b/evaluation/CLEVER/nlayer_model.py @@ -0,0 +1,105 @@ +## setup_mnist.py -- mnist data and model loading code +## +## Copyright (C) 2017-2018, IBM Corp. +## Copyright (C) 2017, Huan Zhang . +## +## This program is licenced under the Apache 2.0 licence, +## contained in the LICENCE file in this directory. + +import numpy as np +import os +import pickle +import gzip +import argparse +import urllib.request + +from tensorflow.contrib.keras.api.keras.models import Sequential +from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout, Activation, Flatten +from tensorflow.contrib.keras.api.keras.layers import Conv2D, MaxPooling2D +from tensorflow.contrib.keras.api.keras.models import load_model +from tensorflow.contrib.keras.api.keras import backend as K + + +class NLayerModel: + def __init__(self, params, restore = None, session=None, use_log=False, image_size=28, image_channel=1): + + self.image_size = image_size + self.num_channels = image_channel + self.num_labels = 10 + + model = Sequential() + model.add(Flatten(input_shape=(image_size, image_size, image_channel))) + # list of all hidden units weights + self.U = [] + for param in params: + # add each dense layer, and save a reference to list U + self.U.append(Dense(param)) + model.add(self.U[-1]) + # ReLU activation + model.add(Activation('relu')) + self.W = Dense(10) + model.add(self.W) + # output log probability, used for black-box attack + if use_log: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.layer_outputs = layer_outputs + self.model = model + + def predict(self, data): + return self.model(data) + + +if __name__ == "__main__": + import scipy.io as sio + parser = argparse.ArgumentParser(description='save n-layer MNIST and CIFAR weights') + parser.add_argument('--model', + default="mnist", + choices=["mnist", "cifar"], + help='model name') + parser.add_argument('--modelfile', + default="", + help='override the model filename, use user specied one') + parser.add_argument('layer_parameters', + nargs='+', + help='number of hidden units per layer') + args = parser.parse_args() + nlayers = len(args.layer_parameters) + 1 + + import tensorflow as tf + with tf.Session() as sess: + # if a model file is not specified, use a manual override + if not args.modelfile: + args.modelfile = "models/"+args.model+"_"+str(nlayers)+"layer_relu" + if args.model == "mnist": + model = NLayerModel(args.layer_parameters, args.modelfile, sess) + #model = NLayerModel(args.layer_parameters, "models/mnist_"+str(nlayers)+"layer_relu") + elif args.model == "cifar": + model = NLayerModel(args.layer_parameters, args.modelfile, sess, image_size=32, image_channel=3) + else: + raise(RuntimeError("Unknow model")) + + + [W, bias_W] = model.W.get_weights() + save_dict = {'W': W, 'bias_W': bias_W} + print("Output layer shape:", W.shape) + U = model.U + for i, Ui in enumerate(U): + # save hidden layer weights, layer by layer + [weight_Ui, bias_Ui] = Ui.get_weights() + print("Hidden layer {} shape: {}".format(i, weight_Ui.shape)) + save_dict['U'+str(i+1)] = weight_Ui + save_dict['bias_U'+str(i+1)] = bias_Ui + + save_name = args.model + "_" + str(nlayers) + "layers.mat" + print('saving to {} with matrices {}'.format(save_name, save_dict.keys())) + # results saved to mnist.mat or cifar.mat + sio.savemat(save_name, save_dict) + diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_cnn.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_cnn.csv new file mode 100644 index 0000000..0e49f59 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_cnn.csv @@ -0,0 +1,101 @@ +id,target +6159 +6279 +4663 +3394 +4292 +2819 +6701 +4278 +7516 +7363 +4425 +6104 +9179 +540 +1378 +3001 +4912 +4532 +2484 +8839 +7876 +2947 +5604 +8623 +1412 +3904 +8215 +4379 +4773 +3058 +493 +9592 +2541 +3632 +8772 +1301 +990 +4207 +2238 +6522 +1313 +1614 +2439 +2450 +8744 +104 +513 +5323 +7792 +940 +7483 +7715 +987 +3389 +4543 +490 +9763 +915 +1564 +8663 +4855 +4848 +3162 +1500 +5439 +8155 +2220 +8579 +7008 +9824 +4314 +3583 +8564 +1631 +6058 +6585 +3661 +1229 +3175 +9662 +8880 +2345 +9974 +626 +6048 +6789 +9618 +7936 +4466 +1270 +1048 +1810 +4736 +5918 +4371 +726 +6301 +3039 +6555 +4688 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_dd.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_dd.csv new file mode 100644 index 0000000..00508e6 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_dd.csv @@ -0,0 +1,101 @@ +id,target +8744 +540 +4425 +9592 +6159 +4379 +4532 +9179 +8772 +4207 +7516 +7234 +4371 +4278 +4773 +3001 +7363 +293 +7282 +2819 +6104 +871 +1313 +1500 +7715 +8215 +6701 +6080 +4848 +3661 +940 +1158 +231 +2484 +1301 +4999 +513 +987 +4663 +973 +4912 +5918 +490 +4314 +6814 +493 +4713 +3372 +7156 +390 +1190 +5604 +7876 +4284 +3058 +8623 +5439 +9824 +6672 +7792 +2645 +3904 +8579 +4292 +9974 +2220 +1378 +4890 +7451 +1810 +604 +3691 +6788 +9662 +7113 +9909 +1631 +5801 +5472 +3394 +9763 +5875 +2439 +5887 +9351 +5741 +2932 +7013 +5905 +5321 +6585 +9645 +2693 +626 +2947 +5480 +9865 +9634 +7484 +3632 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_mlp.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_mlp.csv new file mode 100644 index 0000000..f363e31 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_cifar_mlp.csv @@ -0,0 +1,101 @@ +id,target +1300 +7999 +9930 +8272 +9466 +6252 +9369 +1770 +5183 +9294 +5717 +4903 +8086 +5912 +9097 +8609 +6812 +7626 +7258 +4282 +5725 +1657 +3393 +8734 +1656 +3551 +3183 +104 +1376 +2202 +3312 +7222 +9610 +1033 +9504 +9609 +2315 +7945 +9408 +5052 +7205 +575 +5839 +2786 +4244 +3597 +9732 +1766 +7406 +1306 +7802 +3885 +817 +7437 +2912 +3095 +3456 +9306 +8104 +3387 +783 +178 +4661 +6051 +1759 +7982 +3967 +1438 +4614 +9003 +3390 +5490 +8610 +2690 +3772 +3991 +5365 +7134 +3757 +6308 +5187 +2018 +6907 +9415 +6148 +6917 +5854 +195 +3327 +8889 +8209 +9551 +8175 +6687 +3636 +8290 +1982 +5955 +7646 +9921 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_0.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_0.csv new file mode 100644 index 0000000..ba3cd15 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_0.csv @@ -0,0 +1,101 @@ +id,target +4317 +4567 +412 +565 +9944 +5821 +3898 +1970 +2832 +2952 +1594 +5936 +951 +1270 +8520 +5054 +1331 +2618 +2445 +1531 +2394 +2291 +3410 +8272 +367 +1640 +6019 +4379 +2084 +5913 +1998 +2998 +449 +3985 +4194 +5972 +4382 +1571 +250 +4695 +827 +3776 +4000 +3235 +1272 +2559 +5832 +2587 +2447 +3133 +125 +417 +9423 +671 +3145 +4511 +1782 +8186 +890 +1516 +447 +5842 +4974 +2898 +2339 +3437 +5867 +9656 +3995 +740 +333 +24 +5239 +3718 +4963 +5922 +3146 +3702 +6053 +2151 +7643 +4323 +1343 +4263 +3943 +5926 +4808 +8146 +62 +2105 +389 +1299 +4120 +8624 +2068 +2858 +3490 +9941 +4575 +4271 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_03.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_03.csv new file mode 100644 index 0000000..391926d --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_03.csv @@ -0,0 +1,101 @@ +id,target +2607 +1782 +3751 +3767 +2437 +7432 +4966 +1754 +9696 +2325 +1500 +4248 +3946 +8416 +1184 +4860 +1640 +3410 +2063 +1516 +1283 +175 +4000 +1270 +9613 +4886 +6019 +5268 +3808 +774 +2394 +691 +24 +4695 +6912 +9686 +542 +6700 +3333 +1192 +7226 +1718 +1429 +3133 +1551 +5832 +898 +3796 +2998 +5239 +4379 +2457 +2245 +3761 +1412 +7447 +4433 +3337 +523 +4511 +3429 +3838 +3618 +667 +827 +1594 +9544 +3196 +2366 +417 +6026 +4856 +1571 +2917 +551 +363 +9649 +716 +7402 +9700 +4498 +4816 +9538 +3851 +9831 +990 +1998 +565 +2426 +3976 +1096 +43 +4563 +2519 +8231 +9733 +7232 +671 +3850 +5979 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_cnn.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_cnn.csv new file mode 100644 index 0000000..833f49e --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_cnn.csv @@ -0,0 +1,101 @@ +id,target +3926 +3333 +606 +3206 +578 +628 +1718 +7130 +9944 +5835 +9914 +4785 +9733 +9905 +9544 +4075 +1096 +3618 +591 +4068 +2851 +2024 +4306 +565 +3716 +1173 +8358 +1594 +1012 +9217 +8272 +417 +73 +1204 +3477 +5654 +2393 +2921 +1200 +5757 +4567 +5299 +5649 +2229 +1743 +488 +335 +876 +8520 +3597 +4341 +5936 +1270 +2780 +3160 +4856 +5961 +2832 +2438 +8146 +1234 +80 +5268 +1601 +9925 +3376 +1970 +5714 +3475 +5634 +5821 +2858 +4294 +1597 +2618 +1640 +3856 +8952 +349 +4657 +5600 +1384 +9943 +4695 +9769 +2853 +5866 +5054 +3410 +3898 +9655 +464 +2394 +2424 +4256 +119 +2589 +1531 +3062 +5887 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_dd.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_dd.csv new file mode 100644 index 0000000..1be5593 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_dd.csv @@ -0,0 +1,101 @@ +id,target +2607 +1594 +565 +5936 +417 +4567 +2394 +1184 +2437 +9696 +3838 +4695 +3410 +2998 +6019 +3767 +3437 +6700 +8520 +3133 +7432 +827 +5832 +1270 +3235 +9613 +1571 +250 +3808 +4382 +9634 +1998 +1192 +1754 +8416 +8146 +4379 +9538 +24 +4000 +4966 +9423 +7402 +2898 +3333 +3985 +1640 +2858 +2084 +4511 +389 +3337 +4886 +760 +2457 +1500 +1343 +5268 +1718 +1283 +4194 +671 +1516 +4963 +4974 +2361 +951 +7332 +7447 +3718 +3580 +7226 +9544 +1924 +3946 +2325 +3796 +4507 +8312 +9686 +7680 +9733 +3477 +1865 +5561 +3136 +9208 +5926 +3821 +523 +4856 +3086 +5068 +3288 +789 +3490 +5001 +5239 +9211 +1412 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_mlp.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_mlp.csv new file mode 100644 index 0000000..ccd50d3 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla/nodes_mnist_mlp.csv @@ -0,0 +1,101 @@ +id,target +2437 +1184 +9696 +3767 +9613 +1754 +6700 +3808 +3337 +565 +9686 +3946 +1551 +523 +9634 +7432 +1968 +1838 +5677 +1594 +320 +9538 +7402 +4194 +8416 +1998 +2063 +3132 +990 +3796 +4379 +9831 +6912 +7447 +3751 +2325 +2394 +2189 +4695 +1412 +363 +542 +5832 +5068 +300 +3207 +3580 +1640 +9649 +1270 +3333 +3133 +2998 +5691 +551 +1571 +4384 +8320 +898 +9587 +9612 +3005 +3976 +4974 +43 +4567 +417 +740 +3437 +7226 +8095 +1429 +1433 +4498 +4425 +2245 +6743 +1283 +4507 +4433 +9544 +5268 +4995 +125 +1320 +3850 +3477 +3410 +250 +2366 +6445 +5617 +1096 +3030 +5926 +3851 +4350 +4265 +1226 +154 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_cnn.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_cnn.csv new file mode 100644 index 0000000..b9d3cdd --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_cnn.csv @@ -0,0 +1,101 @@ +id,target +8477 +8988 +3541 +5606 +6297 +7334 +8235 +2165 +3598 +6711 +832 +5725 +9025 +248 +7511 +33 +584 +3508 +3650 +8017 +2801 +5055 +1685 +3714 +1964 +6323 +7807 +8446 +7165 +7776 +9354 +8526 +8684 +9196 +5124 +8776 +1935 +1081 +2468 +7985 +4453 +9809 +6120 +2997 +3654 +5682 +2949 +8757 +5617 +91 +9082 +1105 +2010 +1650 +5312 +5714 +3251 +6210 +3446 +2106 +4860 +4140 +3725 +7475 +4457 +9017 +338 +3830 +8649 +1782 +6756 +2157 +8806 +945 +5228 +8324 +7374 +9164 +8661 +9992 +4089 +5845 +1067 +5193 +9495 +7397 +1252 +2674 +2691 +6106 +4476 +835 +4699 +7430 +5450 +8697 +7885 +6002 +4885 +4352 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_dd.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_dd.csv new file mode 100644 index 0000000..903a81f --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_dd.csv @@ -0,0 +1,101 @@ +id,target +5312 +6339 +3830 +1976 +9590 +1303 +205 +5183 +8684 +4176 +9448 +8477 +5606 +5178 +3381 +2516 +8410 +8020 +6592 +3945 +9157 +8070 +7618 +3598 +8235 +4259 +9017 +705 +5876 +9809 +4674 +6278 +998 +2803 +9928 +5450 +2133 +7019 +7265 +1112 +7165 +2358 +2698 +7291 +5828 +1513 +8889 +8096 +6605 +2453 +2214 +2768 +854 +67 +2044 +1163 +8789 +6297 +3541 +474 +3455 +6323 +232 +3063 +832 +6727 +9779 +460 +1070 +6546 +190 +4778 +1992 +7528 +6151 +8365 +3650 +9503 +2468 +8651 +8832 +8192 +5370 +8597 +4624 +1373 +6206 +6873 +6120 +6002 +7475 +187 +8401 +5116 +8847 +1558 +4886 +7244 +4155 +8428 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_mlp.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_mlp.csv new file mode 100644 index 0000000..6c50ce5 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_cifar_mlp.csv @@ -0,0 +1,101 @@ +id,target +942 +7244 +1479 +4394 +1327 +7438 +8684 +5606 +8697 +5312 +9596 +8365 +426 +1494 +8245 +1373 +9385 +1252 +2746 +7334 +7118 +9477 +474 +8627 +2850 +5728 +6261 +1976 +8587 +6297 +3784 +1560 +7968 +6026 +2653 +147 +3860 +3927 +3454 +7985 +371 +9164 +1555 +1992 +7291 +6920 +5617 +7211 +8458 +3830 +7540 +7704 +8743 +4248 +3455 +6860 +3000 +8810 +9530 +3771 +4275 +1067 +2037 +6853 +7153 +7202 +3672 +4307 +5743 +4623 +9431 +7262 +9745 +4130 +3570 +3209 +7536 +6019 +9295 +9180 +7612 +5837 +8189 +4088 +444 +8232 +8477 +9071 +9213 +3532 +4900 +6054 +273 +21 +5349 +3323 +5115 +9508 +6873 +7460 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_0.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_0.csv new file mode 100644 index 0000000..0ab8672 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_0.csv @@ -0,0 +1,101 @@ +id,target +9129 +4981 +9082 +3142 +4418 +9072 +4905 +9111 +2679 +503 +4402 +4999 +9154 +236 +9480 +5716 +2354 +6132 +5498 +5216 +2967 +7308 +5267 +514 +9131 +3772 +4474 +2482 +5024 +3933 +3904 +4666 +3775 +3057 +5287 +4965 +9820 +1056 +2864 +7163 +6309 +5629 +5541 +9285 +4387 +5402 +2977 +7977 +3668 +5785 +6111 +5684 +8455 +8460 +5436 +2108 +8691 +331 +5465 +5192 +9995 +6828 +6307 +9573 +1915 +3703 +7528 +8734 +4961 +291 +911 +7321 +5863 +1771 +7825 +816 +467 +5385 +2837 +2327 +9032 +4070 +5829 +643 +8482 +4241 +5157 +82 +4274 +3013 +5883 +9985 +921 +7518 +1432 +7744 +6234 +6345 +1799 +5108 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_03.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_03.csv new file mode 100644 index 0000000..fdbdb97 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_03.csv @@ -0,0 +1,101 @@ +id,target +183 +7792 +9498 +9818 +7225 +9066 +7705 +5030 +8869 +1231 +8464 +2337 +8474 +3768 +2466 +9475 +8446 +8637 +5031 +9352 +9004 +3660 +2352 +7712 +6082 +5574 +8123 +3447 +7415 +546 +246 +7667 +5421 +8179 +1808 +4348 +4999 +5411 +2328 +7709 +2217 +8448 +9753 +9297 +9457 +665 +1589 +2060 +5116 +5686 +5810 +7953 +9801 +6130 +1457 +5990 +6429 +3614 +6167 +3827 +2714 +3849 +514 +8338 +3602 +8307 +192 +5259 +8657 +2294 +1061 +545 +6325 +3545 +9178 +9865 +8579 +8554 +7485 +8528 +8929 +5133 +5684 +5497 +6526 +8513 +7335 +3540 +311 +123 +5834 +5415 +9761 +5568 +1362 +8117 +9952 +6114 +6016 +1698 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_cnn.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_cnn.csv new file mode 100644 index 0000000..e574e4c --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_cnn.csv @@ -0,0 +1,101 @@ +id,target +8697 +5497 +6258 +5884 +8789 +7122 +4115 +4499 +5441 +6933 +3712 +6350 +6804 +2231 +5030 +3603 +7667 +3886 +9178 +6315 +8884 +592 +3545 +8554 +8211 +384 +8148 +8739 +9138 +6402 +5810 +9221 +8474 +8446 +5481 +5353 +8399 +6130 +6325 +5385 +2737 +8870 +8625 +2791 +6633 +4418 +7576 +8869 +4875 +1590 +9355 +7681 +5337 +7705 +4008 +5694 +5484 +5581 +7485 +6685 +5898 +6355 +7787 +2486 +6328 +2170 +6207 +8077 +5568 +9066 +9555 +5844 +4336 +7593 +9148 +1203 +7939 +9312 +392 +8454 +8622 +8828 +503 +732 +6104 +5938 +5276 +6261 +9457 +1977 +517 +973 +6354 +2458 +2826 +3036 +353 +8990 +1662 +7778 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_dd.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_dd.csv new file mode 100644 index 0000000..6b08c48 --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_dd.csv @@ -0,0 +1,101 @@ +id,target +9178 +54 +8454 +7485 +7712 +2337 +6354 +6633 +2791 +4999 +9424 +6399 +8622 +8548 +5884 +7709 +7705 +6425 +6355 +7613 +5695 +5660 +7787 +9504 +3603 +1061 +6933 +9066 +973 +2873 +2231 +9130 +6130 +5938 +2535 +6406 +9555 +3849 +6096 +5295 +183 +8614 +732 +8697 +7346 +1733 +8179 +5684 +2321 +2518 +4416 +9457 +7699 +7761 +5497 +2486 +4686 +7681 +5481 +7122 +6104 +8629 +3220 +8668 +5276 +7792 +10 +9360 +6177 +5694 +9647 +586 +2703 +8595 +7576 +8117 +246 +1698 +1898 +3036 +8206 +2170 +4499 +1977 +7295 +5844 +4008 +7723 +2644 +6319 +2996 +8572 +8619 +1293 +2657 +8448 +8587 +517 +3058 +9148 diff --git a/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_mlp.csv b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_mlp.csv new file mode 100644 index 0000000..924c0fa --- /dev/null +++ b/evaluation/CLEVER/normal_spade_nodelist/vanilla_reverse/nodes_mnist_mlp.csv @@ -0,0 +1,101 @@ +id,target +6932 +592 +2376 +6798 +296 +6760 +7566 +4278 +4241 +9414 +5686 +5337 +2592 +7846 +3052 +1610 +9060 +1457 +3849 +7882 +4227 +9129 +4992 +3359 +3179 +1535 +7516 +8194 +5990 +3170 +3624 +4348 +7780 +8595 +8514 +5531 +3982 +1203 +5259 +4070 +440 +4959 +8858 +7913 +8481 +1558 +6259 +5006 +8023 +5031 +7933 +7526 +6779 +8921 +5079 +157 +4604 +6766 +6489 +136 +3768 +148 +1261 +6342 +9757 +4641 +9004 +9223 +4320 +3178 +7835 +5684 +2568 +7168 +2384 +7703 +3830 +6097 +5435 +4965 +8179 +6127 +8528 +5592 +3445 +172 +6280 +1723 +6181 +9387 +6016 +7269 +4611 +5088 +7337 +7129 +3325 +2894 +6333 +7485 diff --git a/evaluation/CLEVER/process_log.py b/evaluation/CLEVER/process_log.py new file mode 100644 index 0000000..0827740 --- /dev/null +++ b/evaluation/CLEVER/process_log.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +process_log.py + +process log files generated by clever.py + +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang + +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" + +import os, glob +import sys +import scipy +import scipy.io as sio +from scipy.stats import weibull_min +import numpy as np +import pandas as pd +import argparse +import matplotlib.pyplot as plt + + +def readDebugLog2array(filename): + f = open(filename) + + # last 3 lines are L0 verbosity, the first line is comment + #lines = f.readlines()[0:-3] + lines = f.readlines()[0:] # read all the lines and check if belongs to [DEBUG][L1] below + # if the files are ended with "_grep" + #lines = f.readlines() + f.close() + + data_arr = {"[DEBUG][L1] id":[],"true_label":[],"target_label":[],"info":[],"bnd_norm":[], + "bnd":[],"ks":[],"pVal":[],"shape":[],"loc":[],"scale":[], "g_x0":[]} + + #Ncols = len(lines[0].strip().split(',')); + + for line in lines: + # split ',' into columns + subline = line.strip().split(',') + + #print(subline) + #print('-reading lines-') + + # only save the info when the line is [DEBUG][L1] + if subline[0].split('=')[0] == '[DEBUG][L1] id ': + for elems in subline: + temp = elems.split('='); + key = temp[0].strip() + val = temp[1].strip() + # save key and val to array + data_arr[key].append(val) + + return data_arr + + +table_results = {} + +if __name__ == "__main__": + # parse command line parameters + parser = argparse.ArgumentParser(description='Process experiment data.') + parser.add_argument('data_folder', nargs='+', help='log file(s) or directory') + parser.add_argument('--save_pickle', + action='store_true', + help='save result to pickle') + args = vars(parser.parse_args()) + print(args) + # process all the log files in the folder + flag_process_all_dir = False + # save result to pickle + is_save2pickle = args['save_pickle'] + + files = [] + # the argument is a list of paths + for path in args['data_folder']: + # if the path is a directory, look for all log files inside + if os.path.isdir(path): + files.extend(glob.glob(os.path.join(path, "*.log"))) + # if the path is a file, include it directly + else: + files.append(path) + print(files) + + for file in files: + # datas is a dictionary + datas = readDebugLog2array(file) + + # convert the dictionary to dataframe df + df = pd.DataFrame.from_dict(datas) + + # convert the string type columns to numeric + df['bnd'] = pd.to_numeric(df['bnd']) + df['g_x0'] = pd.to_numeric(df['g_x0']) + df['ks'] = pd.to_numeric(df['ks']) + df['loc'] = pd.to_numeric(df['loc']) + df['pVal'] = pd.to_numeric(df['pVal']) + df['scale'] = pd.to_numeric(df['scale']) + df['shape'] = pd.to_numeric(df['shape']) + + tag1 = os.path.basename(file).split('_')[1] + + # cifar, mnist, imagenet + if (tag1 == 'cifar') or (tag1 == 'mnist'): + modelName = tag1 + '_' + os.path.basename(file).split('_')[2].split('.')[0] + else: + modelName = tag1 + + if modelName not in table_results: + nan = float('nan') + table_results[modelName] = {'least': {'1': [nan,nan], '2': [nan,nan], 'i': [nan,nan]}, + 'random':{'1': [nan,nan], '2': [nan,nan], 'i': [nan,nan]}, + 'top2': {'1': [nan,nan], '2': [nan,nan], 'i': [nan,nan]}} + + for label in ['least','random','top2']: + for bnd_norm in ['1','2','i']: + df_out = df[(df["info"]==label) & (df["bnd_norm"]==bnd_norm)] + + if not df_out.empty: + out_name = 'pickle_'+modelName+'_'+label+'_norm'+bnd_norm + + + if is_save2pickle: + # save selected df to pickle files + df_out.to_pickle(out_name) + + # obtain statistics and print out + descrb_0 = df_out.describe() + descrb_1 = df_out[(df_out["pVal"]>0.05)&(df_out["shape"]<1000)].describe() + + + bnd_0 = descrb_0["bnd"]["mean"] + count_0 = descrb_0["bnd"]["count"] + + bnd_1 = descrb_1["bnd"]["mean"] + count_1 = descrb_1["bnd"]["count"] + + table_results[modelName][label][bnd_norm][0] = bnd_1 + table_results[modelName][label][bnd_norm][1] = count_1 * 100.0 / count_0 + print("[L0] model = {}, Nimg = {}, bnd_avg = {:.5g}, pVal>0.05 & shape<1000 gives" + "Nimg = {}, bnd_avg = {:.5g}, useable = {:.1f} %".format(out_name,count_0,bnd_0,count_1,bnd_1,count_1/count_0*100)) + # print out table for easy pasting to LaTeX + output = sys.stdout + print('Generating LaTeX table...') + order=['mnist_2-layer', 'mnist_normal', 'mnist_distilled', 'mnist_brelu', 'cifar_2-layer', 'cifar_normal', 'cifar_distilled', 'cifar_brelu', 'inception', 'resnet', 'mobilenet'] + + def gen_table(elem_index, precision=3): + for label in ['least','random','top2']: + output.write("{:15s} &\t{:7s} &\t{:7s} &\n".format(label, '2', 'i')) + for model in order: + if model in table_results: + output.write("{:15s} &\t".format(model)) + for bnd_norm in ['2','i']: + output.write(("{:7."+str(precision)+"f} &\t").format(table_results[model][label][bnd_norm][elem_index])) + output.write("\n") + + print('\n%%% Table for bounds %%%') + gen_table(0) + print('\n%%% Table for p-values %%%') + gen_table(1,1) + diff --git a/evaluation/CLEVER/randsphere.py b/evaluation/CLEVER/randsphere.py new file mode 100644 index 0000000..76bb45b --- /dev/null +++ b/evaluation/CLEVER/randsphere.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import numpy as np +import time +from shmemarray import ShmemRawArray, NpShmemArray +from scipy.special import gammainc +from defense import defend_reduce, defend_jpeg, defend_tv, defend_none, defend_png +from functools import partial + +""" +Original Matlab code (for L2 sampling): + +function X = randsphere(m,n,r) + +X = randn(m,n); +s2 = sum(X.^2,2); +X = X.*repmat(r*(gammainc(s2/2,n/2).^(1/n))./sqrt(s2),1,n); +""" + +# generate random signs +def randsign(N): + n_bytes = (N + 7) // 8 + rbytes = np.random.randint(0, 255, dtype=np.uint8, size=n_bytes) + return (np.unpackbits(rbytes)[:N] - 0.5) * 2 + +def l2_samples(m,n): + X = np.random.randn(m, n) + s2 = np.sum(X * X, axis = 1) + return X * (np.tile(1.0*np.power(gammainc(n/2,s2/2), 1/n) / np.sqrt(s2), (n,1))).T + +def linf_samples(m, n): + return np.random.uniform(-1.0, 1.0, (m,n)) + +def l1_samples(m, n): + # U is uniform random between 0, 1 + U = np.random.uniform(0, 1.0, (m,n-1)) + V = np.empty(shape=(m,n+1)) + # V is sorted U, with 0 and 1 added to the begin and the end + V[:,0] = 0.0 + V[:,-1] = 1.0 + V[:,1:-1] = np.sort(U) + # X is the interval between each V_i + X = V[:,1:] - V[:,:-1] + # randomly flip the sign of each X + s = randsign(m * n).reshape(m,n) + return X * s + +def randsphere(idx, n, r, total_size, scale_size, tag_prefix, input_shape, norm, transform = None): + # currently we assume r = 1.0 and rescale using the array "scale" + assert r == 1.0 + result_arr = NpShmemArray(np.float32, (total_size, n), tag_prefix + "randsphere", False) + # for scale, we may want a different starting point for imagenet, which is scale_start + scale = NpShmemArray(np.float32, (scale_size, 1), tag_prefix + "scale", False) + input_example = NpShmemArray(np.float32, input_shape, tag_prefix + "input_example", False) + all_inputs = NpShmemArray(np.float32, (total_size,) + input_example.shape, tag_prefix + "all_inputs", False) + # m is the number of items, off is the offset + m, offset, scale_start = idx + # n is the dimension + if norm == "l2": + samples = l2_samples(m, n) + if norm == "l1": + samples = l1_samples(m, n) + if norm == "li": + samples = linf_samples(m, n) + result_arr[offset : offset + m] = samples + # make a scaling + result_arr[offset : offset + m] *= scale[offset + scale_start : offset + scale_start + m] + # add to input example + all_inputs[offset : offset + m] = input_example + result_arr = result_arr.reshape(-1, *input_shape) + all_inputs[offset : offset + m] += result_arr[offset : offset + m] + # apply an transformation + if transform: + transform_new = lambda x: (eval(transform)(np.squeeze(x) + 0.5) - 0.5).reshape(all_inputs[offset].shape) + # before transformation we need to clip first + np.clip(all_inputs[offset : offset + m], -0.5, 0.5, out = all_inputs[offset : offset + m]) + # run transformation + for i in range(offset, offset + m): + all_inputs[i] = transform_new(all_inputs[i]) + # we need to clip again after transformation (this is in the caller) + return + diff --git a/evaluation/CLEVER/run.sh b/evaluation/CLEVER/run.sh new file mode 100644 index 0000000..ed52b46 --- /dev/null +++ b/evaluation/CLEVER/run.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +""" +run.sh + +Run file: collect the samples of gradient norms + +Copyright (C) 2017-2018, IBM Corp. +Copyright (C) 2017, Lily Weng + and Huan Zhang + +This program is licenced under the Apache 2.0 licence, +contained in the LICENCE file in this directory. +""" +# run file for running 1st order and 2nd order clever + +if [ "$#" -le 6 ]; then + echo "Usage: $0 model modeltype nsamp niters activation order target gpuNum" + echo "model={mnist, cifar}; modeltype={2-layer, normal}; activation={tanh, sigmoid, softplus}; order={1,2}" + echo "target={top2, rand, least, all}, gpuNum={0,1,2} on frigg, {0,1} on apollo, {0} local" + exit 1 +fi + +## set up parameters +# mnist or cifar +model=$1 +# 2-layer, normal(7-layer) +modeltype=$2 +# 200 +nsamp=$3 +# 10, 20, 50, 100, 200, 300 +niters=$4 +# tanh, sigmoid, softplus +activation=$5 +# 1, 2 +order=$6 +# 1 (top2), 2 (rand), 4 (least), 15 (all) +target=$7 + +if [ "$#" -le 7 ]; then + gpuNum="0" + #echo "Number of args = $#" + echo "Using GPU 0" +else + gpuNum=$8 + #echo "Number of args = $#" + echo "Using GPU $gpuNum" +fi + +if [ "$target" == "top2" ]; then + target_type="1" +elif [ "$target" == "rand" ]; then + target_type="2" +elif [ "$target" == "least" ]; then + target_type="4" +elif [ "$target" == "all" ]; then + target_type="15" +else + echo "Wrong target $target: should be one in {top2, rand, least, all}" + exit 1 +fi + +output="${model}_${modeltype}_${activation}_ord${order}_ns${nsamp}_nit${niters}_${target}_$(date +%m%d_%H%M%S)" +dir="logs/$model/$modeltype" +mkdir -p $dir +logfile=$dir/$output.log +echo $logfile + +CMD="python3 collect_gradients.py --numimg 20 -d $model --model_name $modeltype --batch_size $nsamp --Nsamps $nsamp --Niters $niters --activation $activation --target_type $target_type --order $order" +echo $CMD + +## run on frigg +###CUDA_VISIBLE_DEVICES=$gpuNum +CUDA_VISIBLE_DEVICES=$gpuNum $CMD 2>&1 | tee $logfile +##$CMD 2>&1 | tee $logfile + +#NUMBA_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 $CMD 2>&1 | tee $logfile +# NUMBA_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 $CMD >$logfile 2>$logfile.err +echo "Done $logfile" + + + diff --git a/evaluation/CLEVER/run_clever_16.sh b/evaluation/CLEVER/run_clever_16.sh new file mode 100644 index 0000000..6b2207e --- /dev/null +++ b/evaluation/CLEVER/run_clever_16.sh @@ -0,0 +1,52 @@ +declare -A model_list1 +declare -A model_list2 +declare -A categ_list +categ_list=(["vanilla"]="" ["vanilla_reverse"]="reverse_") +dataset_list1="mnist" +model_list1=(["mnist_0"]="mnist_0" ["mnist_03"]="mnist_03") +dataset_list2="mnist cifar" +model_list2=(["2-layer"]="mlp" ["normal"]="cnn" ["distilled"]="dd") +lines=1 +curtime=`date +"%Y-%m-%d-%I-%M-%p"` +numimg=20 +tempfname="tempfile_`echo $curtime`.txt" + +for categ in ${!categ_list[@]}; do + for dataset in $dataset_list1; do + for model in ${!model_list1[@]}; do + python3 collect_gradients.py --numimg $numimg --target_type 16 --dataset $dataset --model_name $model --save ./lipschitz_mat/$categ --ids cknn_spade_nodelist/`echo ${model_list1[$model]}`_`echo ${categ_list[$categ]}`TopNode.csv + done + done + + for dataset in $dataset_list2; do + for model in ${!model_list2[@]}; do + python3 collect_gradients.py --numimg $numimg --target_type 16 --dataset $dataset --model_name $model --save ./lipschitz_mat/$categ --ids cknn_spade_nodelist/`echo $dataset`_`echo ${model_list2[$model]}`_`echo ${categ_list[$categ]}`TopNode.csv + done + done +done + + +echo "">$tempfname +for categ in ${!categ_list[@]}; do + fname=clever_score/clever_score_`echo $categ`_num`echo $numimg`_`echo $curtime`.txt + echo "">$fname + for dataset in $dataset_list1; do + for model in ${!model_list1[@]}; do + echo $dataset $categ $model >> $fname + python3 clever.py lipschitz_mat/$categ/`echo $dataset`_`echo $model` > $tempfname + echo "$(tail -$lines $tempfname)">>$fname + # echo lipschitz_mat/$categ/`echo $dataset`_`echo $model` + done + done + + for dataset in $dataset_list2; do + for model in ${!model_list2[@]}; do + echo $dataset $categ $model >> $fname + python3 clever.py lipschitz_mat/$categ/`echo $dataset`_`echo $model` > $tempfname + echo "$(tail -$lines $tempfname)">>$fname + # echo lipschitz_mat/$categ/`echo $dataset`_`echo $model` + done + done +done + +rm $tempfname \ No newline at end of file diff --git a/evaluation/CLEVER/setup_cifar.py b/evaluation/CLEVER/setup_cifar.py new file mode 100644 index 0000000..66fe648 --- /dev/null +++ b/evaluation/CLEVER/setup_cifar.py @@ -0,0 +1,173 @@ +## setup_cifar.py -- cifar data and model loading code +## +## Copyright (C) 2017-2018, IBM Corp. +## Copyright (C) 2017, Lily Weng +## and Huan Zhang +## Copyright (C) 2016, Nicholas Carlini . +## +## This program is licenced under the Apache 2.0 licence, +## contained in the LICENCE file in this directory. + + +import tensorflow as tf +import numpy as np +import os +import pickle +import gzip +import pickle +import urllib.request + +from tensorflow.contrib.keras.api.keras.models import Sequential +from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout, Activation, Flatten +from tensorflow.contrib.keras.api.keras.layers import Conv2D, MaxPooling2D +from tensorflow.contrib.keras.api.keras.layers import Lambda +from tensorflow.contrib.keras.api.keras.models import load_model +from tensorflow.contrib.keras.api.keras import backend as K + +def load_batch(fpath, label_key='labels'): + f = open(fpath, 'rb') + d = pickle.load(f, encoding="bytes") + for k, v in d.items(): + del(d[k]) + d[k.decode("utf8")] = v + f.close() + data = d["data"] + labels = d[label_key] + + data = data.reshape(data.shape[0], 3, 32, 32) + final = np.zeros((data.shape[0], 32, 32, 3),dtype=np.float32) + final[:,:,:,0] = data[:,0,:,:] + final[:,:,:,1] = data[:,1,:,:] + final[:,:,:,2] = data[:,2,:,:] + + final /= 255 + final -= .5 + labels2 = np.zeros((len(labels), 10)) + labels2[np.arange(len(labels2)), labels] = 1 + + return final, labels + +def load_batch(fpath): + f = open(fpath,"rb").read() + size = 32*32*3+1 + labels = [] + images = [] + for i in range(10000): + arr = np.fromstring(f[i*size:(i+1)*size],dtype=np.uint8) + lab = np.identity(10)[arr[0]] + img = arr[1:].reshape((3,32,32)).transpose((1,2,0)) + + labels.append(lab) + images.append((img/255)-.5) + return np.array(images),np.array(labels) + + +class CIFAR: + def __init__(self): + train_data = [] + train_labels = [] + + if not os.path.exists("cifar-10-batches-bin"): + urllib.request.urlretrieve("https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz", + "cifar-data.tar.gz") + os.popen("tar -xzf cifar-data.tar.gz").read() + + + for i in range(5): + r,s = load_batch("cifar-10-batches-bin/data_batch_"+str(i+1)+".bin") + train_data.extend(r) + train_labels.extend(s) + + train_data = np.array(train_data,dtype=np.float32) + train_labels = np.array(train_labels) + + self.test_data, self.test_labels = load_batch("cifar-10-batches-bin/test_batch.bin") + + VALIDATION_SIZE = 5000 + + self.validation_data = train_data[:VALIDATION_SIZE, :, :, :] + self.validation_labels = train_labels[:VALIDATION_SIZE] + self.train_data = train_data[VALIDATION_SIZE:, :, :, :] + self.train_labels = train_labels[VALIDATION_SIZE:] + +class CIFARModel: + def __init__(self, restore=None, session=None, use_softmax=False, use_brelu = False, activation = "relu"): + def bounded_relu(x): + return K.relu(x, max_value=1) + if use_brelu: + activation = bounded_relu + else: + activation = activation + + print("inside CIFARModel: activation = {}".format(activation)) + + self.num_channels = 3 + self.image_size = 32 + self.num_labels = 10 + + model = Sequential() + + model.add(Conv2D(64, (3, 3), + input_shape=(32, 32, 3))) + model.add(Activation(activation)) + model.add(Conv2D(64, (3, 3))) + model.add(Activation(activation)) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Conv2D(128, (3, 3))) + model.add(Activation(activation)) + model.add(Conv2D(128, (3, 3))) + model.add(Activation(activation)) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Flatten()) + model.add(Dense(256)) + model.add(Activation(activation)) + model.add(Dense(256)) + model.add(Activation(activation)) + model.add(Dense(10)) + if use_softmax: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.layer_outputs = layer_outputs + self.model = model + + def predict(self, data): + return self.model(data) + + +class TwoLayerCIFARModel: + def __init__(self, restore = None, session=None, use_softmax=False): + self.num_channels = 3 + self.image_size = 32 + self.num_labels = 10 + + model = Sequential() + model.add(Flatten(input_shape=(32, 32, 3))) + model.add(Dense(1024)) + model.add(Activation('softplus')) + model.add(Dense(10)) + # output log probability, used for black-box attack + if use_softmax: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.layer_outputs = layer_outputs + self.model = model + + def predict(self, data): + + return self.model(data) diff --git a/evaluation/CLEVER/setup_imagenet.py b/evaluation/CLEVER/setup_imagenet.py new file mode 100644 index 0000000..6c8bbe5 --- /dev/null +++ b/evaluation/CLEVER/setup_imagenet.py @@ -0,0 +1,498 @@ +## Modified by Huan Zhang for ResNet, Inception v1, v2, v4, VGG, MobileNet, Densenet, Alexnet and NASnet +## Modified by Huan Zhang for the updated Inception-v3 model (inception_v3_2016_08_28.tar.gz) +## Modified by Nicholas Carlini to match model structure for attack code. +## Original copyright license follows. + + +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Simple image classification with an ImageNet Classifier. + +Run image classification with an ImageNet Classifier (Inception, ResNet, AlexNet, etc) trained on ImageNet 2012 Challenge data +set. + +This program creates a graph from a saved GraphDef protocol buffer, +and runs inference on an input JPEG image. It outputs human readable +strings of the top 5 predictions along with their probabilities. + +Change the --image_file argument to any jpg image to compute a +classification of that image. + +Please see the tutorial and website for a detailed description of how +to use this script to perform image recognition. + +https://tensorflow.org/tutorials/image_recognition/ +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os.path +import re +import sys +from functools import partial +import random +import tarfile +import scipy.misc + +import numpy as np +from six.moves import urllib +import tensorflow as tf + +import PIL +from PIL import Image + + +model_params = {} + + +"""Add a new new entry to ImageNet models + +Parameters: +name: name of the new model, like "resnet" +url: URL to download the model +image_size: image size, usually 224 or 299 +model_filename: model protobuf file name (.pb) +label_filename: a text file contains the mapping from class ID to human readable string +input_tensor: input tensor of the network defined by protobuf, like "input:0" +logit: logit output tensor of the network, like "resnet_v2_50/predictions/Reshape:0" +prob: probability output tensor of the network, like "resnet_v2_50/predictions/Reshape_1:0" +shape: tensor for reshaping the final output, like "resnet_v2_50/predictions/Shape:0". + Set to None if no reshape needed. + +All the tensor names can be viewed and found in TensorBoard. +""" +def AddModel(name, url, model_filename, image_size, label_filename, input_tensor, logit, prob, shape): + global model_params + param = {} + param['url'] = url + param['model_filename'] = model_filename + param['size'] = image_size + param['input'] = input_tensor + param['logit'] = logit + param['prob'] = prob + param['shape'] = shape + param['label_filename'] = label_filename + param['name'] = name + model_params[name] = param + +# pylint: disable=line-too-long +AddModel('resnet_v2_50', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_resnet_v2_50.pb', 299, 'labels.txt', 'input:0', + 'resnet_v2_50/predictions/Reshape:0', 'resnet_v2_50/predictions/Reshape_1:0', 'resnet_v2_50/predictions/Shape:0') +AddModel('resnet_v2_101', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_resnet_v2_101.pb', 299, 'labels.txt', 'input:0', + 'resnet_v2_101/predictions/Reshape:0', 'resnet_v2_101/predictions/Reshape_1:0', 'resnet_v2_101/predictions/Shape:0') +AddModel('resnet_v2_152', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_resnet_v2_152.pb', 299, 'labels.txt', 'input:0', + 'resnet_v2_152/predictions/Reshape:0', 'resnet_v2_152/predictions/Reshape_1:0', 'resnet_v2_152/predictions/Shape:0') +AddModel('inception_v1', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_v1.pb', 224, 'labels.txt', 'input:0', + 'InceptionV1/Logits/Predictions/Reshape:0', 'InceptionV1/Logits/Predictions/Reshape_1:0', 'InceptionV1/Logits/Predictions/Shape:0') +AddModel('inception_v2', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_v2.pb', 224, 'labels.txt', 'input:0', + 'InceptionV2/Predictions/Reshape:0', 'InceptionV2/Predictions/Reshape_1:0', 'InceptionV2/Predictions/Shape:0') +AddModel('inception_v3', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_v3.pb', 299, 'labels.txt', 'input:0', + 'InceptionV3/Predictions/Reshape:0', 'InceptionV3/Predictions/Softmax:0', 'InceptionV3/Predictions/Shape:0') +AddModel('inception_v4', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_v4.pb', 299, 'labels.txt', 'input:0', + 'InceptionV4/Logits/Logits/BiasAdd:0', 'InceptionV4/Logits/Predictions:0', '') +AddModel('inception_resnet_v2', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_inception_resnet_v2.pb', 299, 'labels.txt', 'input:0', + 'InceptionResnetV2/Logits/Logits/BiasAdd:0', 'InceptionResnetV2/Logits/Predictions:0', '') +AddModel('vgg_16', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_vgg_16.pb', 224, 'labels.txt', 'input:0', + 'vgg_16/fc8/squeezed:0', 'vgg_16/fc8/squeezed:0', '') +AddModel('vgg_19', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_vgg_19.pb', 224, 'labels.txt', 'input:0', + 'vgg_19/fc8/squeezed:0', 'vgg_19/fc8/squeezed:0', '') +AddModel('mobilenet_v1_025', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_mobilenet_v1_025.pb', 224, 'labels.txt', 'input:0', + 'MobilenetV1/Predictions/Reshape:0', 'MobilenetV1/Predictions/Reshape_1:0', 'MobilenetV1/Predictions/Shape:0') +AddModel('mobilenet_v1_050', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_mobilenet_v1_050.pb', 224, 'labels.txt', 'input:0', + 'MobilenetV1/Predictions/Reshape:0', 'MobilenetV1/Predictions/Reshape_1:0', 'MobilenetV1/Predictions/Shape:0') +AddModel('mobilenet_v1_100', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_mobilenet_v1_100.pb', 224, 'labels.txt', 'input:0', + 'MobilenetV1/Predictions/Reshape:0', 'MobilenetV1/Predictions/Reshape_1:0', 'MobilenetV1/Predictions/Shape:0') +AddModel('nasnet_large', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'frozen_nasnet_large.pb', 331, 'labels.txt', 'input:0', + 'final_layer/FC/BiasAdd:0', 'final_layer/predictions:0', '') +AddModel('densenet121_k32', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'densenet121_k32_frozen.pb', 224, 'labels.txt', 'input:0', + 'densenet121/predictions/Reshape:0', 'densenet121/predictions/Reshape_1:0', 'densenet121/predictions/Shape:0') +AddModel('densenet169_k32', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'densenet169_k32_frozen.pb', 224, 'labels.txt', 'input:0', + 'densenet169/predictions/Reshape:0', 'densenet169/predictions/Reshape_1:0', 'densenet169/predictions/Shape:0') +AddModel('densenet161_k48', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'densenet161_k48_frozen.pb', 224, 'labels.txt', 'input:0', + 'densenet161/predictions/Reshape:0', 'densenet161/predictions/Reshape_1:0', 'densenet161/predictions/Shape:0') +AddModel('alexnet', 'http://jaina.cs.ucdavis.edu/datasets/adv/imagenet/frozen_imagenet_models_v1.1.tar.gz', + 'alexnet_frozen.pb', 227, 'labels.txt', 'Placeholder:0', + 'fc8/fc8:0', 'Softmax:0', '') + +# pylint: enable=line-too-long + + +class NodeLookup(object): + """Converts integer node ID's to human readable labels.""" + + def __init__(self, + label_lookup_path=None): + if not label_lookup_path: + label_lookup_path = os.path.join( + FLAGS.model_dir, 'labels.txt') + self.node_lookup = self.load(label_lookup_path) + + def load(self, label_lookup_path): + """Loads a human readable English name for each softmax node. + + Args: + label_lookup_path: string UID to integer node ID. + uid_lookup_path: string UID to human-readable string. + + Returns: + dict from integer node ID to human-readable string. + """ + if not tf.gfile.Exists(label_lookup_path): + tf.logging.fatal('File does not exist %s', label_lookup_path) + + # Loads mapping from string UID to integer node ID. + node_id_to_name = {} + proto_as_ascii = tf.gfile.GFile(label_lookup_path).readlines() + for line in proto_as_ascii: + if line: + words = line.split(':') + target_class = int(words[0]) + name = words[1] + node_id_to_name[target_class] = name + + return node_id_to_name + + def id_to_string(self, node_id): + if node_id not in self.node_lookup: + return '' + return self.node_lookup[node_id] + +LOADED_GRAPH = None + +def create_graph(model_param): + """Creates a graph from saved GraphDef file and returns a saver.""" + # Creates graph from saved graph_def.pb. + global LOADED_GRAPH + with tf.gfile.FastGFile(os.path.join( + # FLAGS.model_dir, 'classify_image_graph_def.pb'), 'rb') as f: + FLAGS.model_dir, model_param['model_filename']), 'rb') as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + #for line in repr(graph_def).split("\n"): + # if "tensor_content" not in line: + # print(line) + LOADED_GRAPH = graph_def + + +class ImageNetModelPrediction: + def __init__(self, sess, use_softmax = False, model_name = "resnet_v2_50", softmax_tensor = None): + self.sess = sess + self.use_softmax = use_softmax + model_param = model_params[model_name] + self.output_name = model_param['prob'] if self.use_softmax else model_param['logit'] + self.input_name = model_param['input'] + self.shape_name = model_param['shape'] + self.model_name = model_param['name'] + self.image_size = model_param['size'] + self.img = tf.placeholder(tf.float32, (None, self.image_size, self.image_size, 3)) + if not softmax_tensor: + # no existing graph + self.softmax_tensor = tf.import_graph_def( + LOADED_GRAPH, + # sess.graph.as_graph_def(), + input_map={self.input_name: self.img}, + return_elements=[self.output_name]) + if 'vgg' in self.model_name and use_softmax == True: + # the pretrained VGG network output is logits, need an extra softmax + self.softmax_tensor = tf.nn.softmax(self.softmax_tensor) + else: + # use an existing graph + self.softmax_tensor = softmax_tensor + print("GraphDef Size:", self.sess.graph_def.ByteSize()) + + def predict(self, dat): + dat = np.squeeze(dat) + if 'vgg' in self.model_name: + # VGG uses 0 - 255 image as input + dat = (0.5 + dat) * 255.0 + imagenet_mean = np.array([123.68, 116.78, 103.94], dtype=np.float32) + dat -= imagenet_mean + elif 'alexnet' in self.model_name: + if dat.ndim == 3: + dat = dat[:,:,::-1] + else: + dat = dat[:,:,:,::-1] # change RGB to BGR + dat = (0.5 + dat) * 255.0 + imagenet_mean = np.array([104., 117., 124.], dtype=np.float32) + dat -= imagenet_mean + elif 'densenet' in self.model_name: + dat = (0.5 + dat) * 255.0 + imagenet_mean = np.array([123.68, 116.78, 103.94], dtype=np.float32) + dat -= imagenet_mean + dat = dat * 0.017 + else: + dat = dat * 2.0 + + + if dat.ndim == 3: + scaled = dat.reshape((1,) + dat.shape) + else: + scaled = dat + # print(scaled.shape) + predictions = self.sess.run(self.softmax_tensor, + {self.img: scaled}) + predictions = np.squeeze(predictions) + return predictions + # Creates node ID --> English string lookup. + node_lookup = NodeLookup() + top_k = predictions.argsort()#[-FLAGS.num_top_predictions:][::-1] + for node_id in top_k: + print('id',node_id) + human_string = node_lookup.id_to_string(node_id) + score = predictions[node_id] + print('%s (score = %.5f)' % (human_string, score)) + return top_k[-1] + + +CREATED_GRAPH = False +class ImageNetModel: + def __init__(self, sess, use_softmax = False, model_name = "resnet_v2_50", create_prediction = True): + global CREATED_GRAPH + self.sess = sess + self.use_softmax = use_softmax + model_param = model_params[model_name] + maybe_download_and_extract(model_param) + + if not CREATED_GRAPH: + create_graph(model_param) + CREATED_GRAPH = True + self.num_channels = 3 + self.output_name = model_param['prob'] if self.use_softmax else model_param['logit'] + self.input_name = model_param['input'] + self.shape_name = model_param['shape'] + self.model_name = model_param['name'] + self.num_labels = 1000 if 'vgg' in self.model_name or 'densenet' in self.model_name or 'alexnet' in self.model_name else 1001 + self.image_size = model_param['size'] + self.use_softmax = use_softmax + if create_prediction: + self.model = ImageNetModelPrediction(sess, use_softmax, model_name) + + def predict(self, img): + if 'vgg' in self.model_name: + # VGG uses 0 - 255 image as input + img = (0.5 + img) * 255.0 + imagenet_mean = np.array([123.68, 116.78, 103.94], dtype=np.float32) + img -= imagenet_mean + elif 'alexnet' in self.model_name: + img = tf.reverse(img,axis=[-1])# change RGB to BGR + img = (0.5 + img) * 255.0 + imagenet_mean = np.array([104., 117., 124.], dtype=np.float32) + img -= imagenet_mean + elif 'densenet' in self.model_name: + # convert to 0 - 255 image as input + img = (0.5 + img) * 255.0 + imagenet_mean = np.array([123.68, 116.78, 103.94], dtype=np.float32) + img -= imagenet_mean + img = img * 0.017 + else: + img = img * 2.0 + + if img.shape.is_fully_defined() and img.shape.as_list()[0] and self.shape_name: + # check if a shape has been specified explicitly + shape = (int(img.shape[0]), self.num_labels) + self.softmax_tensor = tf.import_graph_def( + LOADED_GRAPH, + # self.sess.graph.as_graph_def(), + input_map={self.input_name: img, self.shape_name: shape}, + return_elements=[self.output_name]) + if 'vgg' in self.model_name and self.use_softmax == True: + # the pretrained VGG network output is logitimport_graph_defs, need an extra softmax + self.softmax_tensor = tf.nn.softmax(self.softmax_tensor) + else: + # placeholder shape + self.softmax_tensor = tf.import_graph_def( + LOADED_GRAPH, + # self.sess.graph.as_graph_def(), + input_map={self.input_name: img}, + return_elements=[self.output_name]) + if 'vgg' in self.model_name and self.use_softmax == True: + # the pretrained VGG network output is logits, need an extra softmax + self.softmax_tensor = tf.nn.softmax(self.softmax_tensor) + print("GraphDef Size:", self.sess.graph_def.ByteSize()) + return self.softmax_tensor[0] + + +def maybe_download_and_extract(model_param): + """Download and extract model tar file.""" + dest_directory = FLAGS.model_dir + if not os.path.exists(dest_directory): + os.makedirs(dest_directory) + filename = model_param['url'].split('/')[-1] + filepath = os.path.join(dest_directory, filename) + modelname = model_param['model_filename'].split('/')[-1] + modelpath = os.path.join(dest_directory, modelname) + if not os.path.exists(modelpath): + def _progress(count, block_size, total_size): + sys.stdout.write('\r>> Downloading %s %.1f%%' % ( + filename, float(count * block_size) / float(total_size) * 100.0)) + sys.stdout.flush() + filepath, _ = urllib.request.urlretrieve(model_param['url'], filepath, _progress) + print() + statinfo = os.stat(filepath) + print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.') + if os.path.splitext(filename)[1] != '.pb': + tarfile.open(filepath, 'r:gz').extractall(dest_directory) + + +def main(_): + param = model_params[FLAGS.model_name] + maybe_download_and_extract(param) + image = (FLAGS.image_file if FLAGS.image_file else + os.path.join(FLAGS.model_dir, 'cropped_panda.jpg')) + # run_inference_on_image(image) + create_graph(param) + image_size = param['size'] + with tf.Session() as sess: + dat = np.array(scipy.misc.imresize(scipy.misc.imread(image),(image_size, image_size)), dtype = np.float32) + dat /= 255.0 + dat -= 0.5 + # print(dat) + model = ImageNetModelPrediction(sess, True, FLAGS.model_name) + predictions = model.predict(dat) + # Creates node ID --> English string lookup. + node_lookup = NodeLookup() + top_k = predictions.argsort()#[-FLAGS.num_top_predictions:][::-1] + for node_id in top_k: + score = predictions[node_id] + if 'vgg' in FLAGS.model_name or 'densenet' in FLAGS.model_name or 'alexnet' in FLAGS.model_name: + node_id += 1 + print('id',node_id) + human_string = node_lookup.id_to_string(node_id) + print('%s (score = %.5f)' % (human_string, score)) + + +def keep_aspect_ratio_transform(img, img_size): + + s_0, s_1 = img.size + if s_0 < s_1: + ratio = (img_size / float(s_0)) + size_1 = int((float(img.size[1]) * float(ratio))) + img = img.resize((img_size, size_1), PIL.Image.ANTIALIAS) + else: + ratio = (img_size / float(s_1)) + size_0 = int((float(img.size[0]) * float(ratio))) + img = img.resize((size_0, img_size), PIL.Image.ANTIALIAS) + + c_0 = img.size[0] // 2 + c_1 = img.size[1] // 2 + + if img_size % 2 == 0: + w_left = h_top = img_size // 2 + w_right = h_bottom = img_size // 2 + else: + w_left = h_top = img_size // 2 + w_right = h_bottom = img_size // 2 + 1 + + transformed_img = img.crop( + ( + c_0 - w_left, + c_1 - h_top, + c_0 + w_right, + c_1 + h_bottom + ) + ) + + return transformed_img + +def readimg(ff, img_size): + f = "../imagenetdata/imgs/"+ff + # img = scipy.misc.imread(f) + # skip small images (image should be at least img_size X img_size) + + # if img.shape[0] < img_size or img.shape[1] < img_size: + # return None + + # img = np.array(scipy.misc.imresize(img,(img_size, img_size)),dtype=np.float32)/255.0-.5 + img = Image.open(f) + transformed_img = keep_aspect_ratio_transform(img, img_size) + + img = np.array(transformed_img)/255.0-.5 + if img.shape != (img_size, img_size, 3): + # grayscale image + if img.shape == (img_size, img_size): + img = np.repeat(np.expand_dims(img, axis = 2), 3, axis = 2) + return [img, int(ff.split(".")[0])] + return None + return [img, int(ff.split(".")[0])] + +class ImageNet: + def __init__(self, img_size, load_total_imgs = 1000): + from multiprocessing import Pool, cpu_count + pool = Pool(cpu_count()) + file_list = sorted(os.listdir("../imagenetdata/imgs/")) + random.shuffle(file_list) + # for efficiency, we only load first 1000 images + # You can pass load_total_imgs to load all images + short_file_list = file_list[:load_total_imgs] + r = pool.map(partial(readimg, img_size=img_size), short_file_list) + print(short_file_list) + print("Loaded imagenet", len(short_file_list), "of", len(file_list), "images") + + r = [x for x in r if x != None] + test_data, test_labels = zip(*r) + self.test_data = np.array(test_data) + self.test_labels = np.zeros((len(test_labels), 1001)) + self.test_labels[np.arange(len(test_labels)), test_labels] = 1 + + pool.close() + pool.join() + +if __name__ == '__main__': + FLAGS = tf.app.flags.FLAGS + # classify_image_graph_def.pb: + # Binary representation of the GraphDef protocol buffer. + # imagenet_synset_to_human_label_map.txt: + # Map from synset ID to a human readable string. + # imagenet_2012_challenge_label_map_proto.pbtxt: + # Text representation of a protocol buffer mapping a label to synset ID. + tf.app.flags.DEFINE_string( + 'model_dir', 'tmp/imagenet', + """Path to classify_image_graph_def.pb, """ + """imagenet_synset_to_human_label_map.txt, and """ + """imagenet_2012_challenge_label_map_proto.pbtxt.""") + tf.app.flags.DEFINE_string('image_file', '', + """Absolute path to image file.""") + tf.app.flags.DEFINE_string('model_name', 'resnet_v2_101', + """Absolute path to image file.""") + tf.app.flags.DEFINE_integer('num_top_predictions', 5, + """Display this many predictions.""") + tf.app.run() +else: + # starting from TF 1.5, an parameter unkown by tf.app.flags will raise an error + # so we cannot use tf.app.flags when loading this file as a module, because the + # main program may define other options. + from argparse import Namespace + FLAGS = Namespace(model_dir="tmp/imagenet") + diff --git a/evaluation/CLEVER/setup_mnist.py b/evaluation/CLEVER/setup_mnist.py new file mode 100644 index 0000000..ed9ef63 --- /dev/null +++ b/evaluation/CLEVER/setup_mnist.py @@ -0,0 +1,238 @@ +## setup_mnist.py -- mnist data and model loading code +## +## Copyright (C) 2017-2018, IBM Corp. +## Copyright (C) 2017, Lily Weng +## and Huan Zhang +## Copyright (C) 2016, Nicholas Carlini . +## +## This program is licenced under the Apache 2.0 licence, +## contained in the LICENCE file in this directory. + +import tensorflow as tf +import numpy as np +import os +import pickle +import gzip +import urllib.request + +from tensorflow.contrib.keras.api.keras.models import Sequential +from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout, Activation, Flatten +from tensorflow.contrib.keras.api.keras.layers import Conv2D, MaxPooling2D +from tensorflow.contrib.keras.api.keras.layers import Lambda +from tensorflow.contrib.keras.api.keras.models import load_model +from tensorflow.contrib.keras.api.keras import backend as K + +def extract_data(filename, num_images): + with gzip.open(filename) as bytestream: + bytestream.read(16) + buf = bytestream.read(num_images*28*28) + data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32) + data = (data / 255) - 0.5 + data = data.reshape(num_images, 28, 28, 1) + return data + +def extract_labels(filename, num_images): + with gzip.open(filename) as bytestream: + bytestream.read(8) + buf = bytestream.read(1 * num_images) + labels = np.frombuffer(buf, dtype=np.uint8) + return (np.arange(10) == labels[:, None]).astype(np.float32) + +class MNIST: + def __init__(self): + if not os.path.exists("data"): + os.mkdir("data") + files = ["train-images-idx3-ubyte.gz", + "t10k-images-idx3-ubyte.gz", + "train-labels-idx1-ubyte.gz", + "t10k-labels-idx1-ubyte.gz"] + for name in files: + + urllib.request.urlretrieve('http://yann.lecun.com/exdb/mnist/' + name, "data/"+name) + + train_data = extract_data("data/train-images-idx3-ubyte.gz", 60000) + train_labels = extract_labels("data/train-labels-idx1-ubyte.gz", 60000) + self.test_data = extract_data("data/t10k-images-idx3-ubyte.gz", 10000) + self.test_labels = extract_labels("data/t10k-labels-idx1-ubyte.gz", 10000) + + VALIDATION_SIZE = 5000 + + self.validation_data = train_data[:VALIDATION_SIZE, :, :, :] + self.validation_labels = train_labels[:VALIDATION_SIZE] + self.train_data = train_data[VALIDATION_SIZE:, :, :, :] + self.train_labels = train_labels[VALIDATION_SIZE:] + + +class MNISTModel: + def __init__(self, restore = None, session=None, use_softmax=False, use_brelu = False, activation = "relu"): + def bounded_relu(x): + return K.relu(x, max_value=1) + if use_brelu: + activation = bounded_relu + + print("inside MNISTModel: activation = {}".format(activation)) + + self.num_channels = 1 + self.image_size = 28 + self.num_labels = 10 + + model = Sequential() + + model.add(Conv2D(32, (3, 3), + input_shape=(28, 28, 1))) + model.add(Activation(activation)) + model.add(Conv2D(32, (3, 3))) + model.add(Activation(activation)) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Conv2D(64, (3, 3))) + model.add(Activation(activation)) + model.add(Conv2D(64, (3, 3))) + model.add(Activation(activation)) + model.add(MaxPooling2D(pool_size=(2, 2))) + + model.add(Flatten()) + model.add(Dense(200)) + model.add(Activation(activation)) + model.add(Dense(200)) + model.add(Activation(activation)) + model.add(Dense(10)) + # output log probability, used for black-box attack + if use_softmax: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.model = model + self.layer_outputs = layer_outputs + + def predict(self, data): + return self.model(data) + +class TwoLayerMNISTModel: + def __init__(self, restore = None, session=None, use_softmax=False): + self.num_channels = 1 + self.image_size = 28 + self.num_labels = 10 + + model = Sequential() + model.add(Flatten(input_shape=(28, 28, 1))) + model.add(Dense(1024)) + model.add(Lambda(lambda x: x * 10)) + model.add(Activation('softplus')) + model.add(Lambda(lambda x: x * 0.1)) + model.add(Dense(10)) + # output log probability, used for black-box attack + if use_softmax: + model.add(Activation('softmax')) + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.layer_outputs = layer_outputs + self.model = model + + def predict(self, data): + + return self.model(data) + + +class mnist_0: + def __init__(self, restore = None, session=None, use_softmax=False, use_brelu = False, activation = "relu"): + def bounded_relu(x): + return K.relu(x, max_value=1) + if use_brelu: + activation = bounded_relu + + print("inside MNISTModel: activation = {}".format(activation)) + + self.num_channels = 1 + self.image_size = 28 + self.num_labels = 10 + + labels = tf.placeholder(tf.int64, shape=[None], name='labels') + logits = tf.placeholder(tf.float32, shape=[None, 10], name='logits') + + model = Sequential() + conv1 = Conv2D(32, (5,5), input_shape=(28,28,1), activation='relu', padding='SAME', use_bias=True) + model.add(conv1) + model.add(MaxPooling2D(pool_size=(2,2),padding='SAME')) + conv2 = Conv2D(64, (5,5), activation='relu', padding='SAME', use_bias=True) + model.add(conv2) + model.add(MaxPooling2D(pool_size=(2,2),padding='SAME')) + model.add(Flatten()) + + fc1 = Dense(1024, activation='relu', use_bias=True) + model.add(fc1) + fc2 = Dense(10, activation='softmax', use_bias=True) + model.add(fc2) + + # output log probability, used for black-box attack + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.model = model + self.layer_outputs = layer_outputs + + def predict(self, data): + return self.model(data) + + +class mnist_03: + def __init__(self, restore = None, session=None, use_softmax=False, use_brelu = False, activation = "relu"): + def bounded_relu(x): + return K.relu(x, max_value=1) + if use_brelu: + activation = bounded_relu + + print("inside MNISTModel: activation = {}".format(activation)) + + self.num_channels = 1 + self.image_size = 28 + self.num_labels = 10 + + labels = tf.placeholder(tf.int64, shape=[None], name='labels') + logits = tf.placeholder(tf.float32, shape=[None, 10], name='logits') + + model = Sequential() + conv1 = Conv2D(32, (5,5), input_shape=(28,28,1), activation='relu', padding='SAME', use_bias=True) + model.add(conv1) + model.add(MaxPooling2D(pool_size=(2,2),padding='SAME')) + conv2 = Conv2D(64, (5,5), activation='relu', padding='SAME', use_bias=True) + model.add(conv2) + model.add(MaxPooling2D(pool_size=(2,2),padding='SAME')) + model.add(Flatten()) + + fc1 = Dense(1024, activation='relu', use_bias=True) + model.add(fc1) + fc2 = Dense(10, activation='softmax', use_bias=True) + model.add(fc2) + + # output log probability, used for black-box attack + if restore: + model.load_weights(restore) + + layer_outputs = [] + for layer in model.layers: + if isinstance(layer, Conv2D) or isinstance(layer, Dense): + layer_outputs.append(K.function([model.layers[0].input], [layer.output])) + + self.model = model + self.layer_outputs = layer_outputs + + def predict(self, data): + return self.model(data) diff --git a/evaluation/CLEVER/shmemarray.py b/evaluation/CLEVER/shmemarray.py new file mode 100644 index 0000000..aa561a6 --- /dev/null +++ b/evaluation/CLEVER/shmemarray.py @@ -0,0 +1,104 @@ +# +# Based on multiprocessing.sharedctypes.RawArray +# +# Uses posix_ipc (http://semanchuk.com/philip/posix_ipc/) to allow shared ctypes arrays +# among unrelated processors +# +# Usage Notes: +# * The first two args (typecode_or_type and size_or_initializer) should work the same as with RawArray. +# * The shared array is accessible by any process, as long as tag matches. +# * The shared memory segment is unlinked when the origin array (that returned +# by ShmemRawArray(..., create=True)) is deleted/gc'ed +# * Creating an shared array using a tag that currently exists will raise an ExistentialError +# * Accessing a shared array using a tag that doesn't exist (or one that has been unlinked) will also +# raise an ExistentialError +# +# Author: Shawn Chin (http://shawnchin.github.com) +# +# Edited for python 3 by: Adam Stooke +# + +import numpy as np +# import os +import sys +import mmap +import ctypes +import posix_ipc +# from _multiprocessing import address_of_buffer # (not in python 3) +from string import ascii_letters, digits + +valid_chars = frozenset("/-_. %s%s" % (ascii_letters, digits)) + +typecode_to_type = { + 'c': ctypes.c_char, 'u': ctypes.c_wchar, + 'b': ctypes.c_byte, 'B': ctypes.c_ubyte, + 'h': ctypes.c_short, 'H': ctypes.c_ushort, + 'i': ctypes.c_int, 'I': ctypes.c_uint, + 'l': ctypes.c_long, 'L': ctypes.c_ulong, + 'f': ctypes.c_float, 'd': ctypes.c_double +} + + +def address_of_buffer(buf): # (python 3) + return ctypes.addressof(ctypes.c_char.from_buffer(buf)) + + +class ShmemBufferWrapper(object): + + def __init__(self, tag, size, create=True): + # default vals so __del__ doesn't fail if __init__ fails to complete + self._mem = None + self._map = None + self._owner = create + self.size = size + + assert 0 <= size < sys.maxsize # sys.maxint (python 3) + flag = (0, posix_ipc.O_CREX)[create] + self._mem = posix_ipc.SharedMemory(tag, flags=flag, size=size) + self._map = mmap.mmap(self._mem.fd, self._mem.size) + self._mem.close_fd() + + def get_address(self): + # addr, size = address_of_buffer(self._map) + # assert size == self.size + assert self._map.size() == self.size # (changed for python 3) + addr = address_of_buffer(self._map) + return addr + + def __del__(self): + if self._map is not None: + self._map.close() + if self._mem is not None and self._owner: + self._mem.unlink() + + +def ShmemRawArray(typecode_or_type, size_or_initializer, tag, create=True): + assert frozenset(tag).issubset(valid_chars) + if tag[0] != "/": + tag = "/%s" % (tag,) + + type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) + if isinstance(size_or_initializer, int): + type_ = type_ * size_or_initializer + else: + type_ = type_ * len(size_or_initializer) + + buffer = ShmemBufferWrapper(tag, ctypes.sizeof(type_), create=create) + obj = type_.from_address(buffer.get_address()) + obj._buffer = buffer + + if not isinstance(size_or_initializer, int): + obj.__init__(*size_or_initializer) + + return obj + + +############################################################################### +# New Additions (by Adam) # + + +def NpShmemArray(dtype, shape, tag, create=True): + size = int(np.prod(shape)) + nbytes = size * np.dtype(dtype).itemsize + shmem = ShmemRawArray(ctypes.c_char, nbytes, tag, create) + return np.frombuffer(shmem, dtype=dtype, count=size).reshape(shape) diff --git a/evaluation/GAIRAT/GAIR.py b/evaluation/GAIRAT/GAIR.py new file mode 100644 index 0000000..caaf792 --- /dev/null +++ b/evaluation/GAIRAT/GAIR.py @@ -0,0 +1,17 @@ +from models import * +import torch +import numpy as np + +def GAIR(num_steps, Kappa, Lambda, func): + # Weight assign + if func == "Tanh": + reweight = ((Lambda+(int(num_steps/2)-Kappa)*5/(int(num_steps/2))).tanh()+1)/2 + normalized_reweight = reweight * len(reweight) / reweight.sum() + elif func == "Sigmoid": + reweight = (Lambda+(int(num_steps/2)-Kappa)*5/(int(num_steps/2))).sigmoid() + normalized_reweight = reweight * len(reweight) / reweight.sum() + elif func == "Discrete": + reweight = ((num_steps+1)-Kappa)/(num_steps+1) + normalized_reweight = reweight * len(reweight) / reweight.sum() + + return normalized_reweight diff --git a/evaluation/GAIRAT/GAIRAT.py b/evaluation/GAIRAT/GAIRAT.py new file mode 100644 index 0000000..d1840d7 --- /dev/null +++ b/evaluation/GAIRAT/GAIRAT.py @@ -0,0 +1,359 @@ +""" +The code is adapted from the Jingfeng Zhang's code: +https://github.com/zjfheart/Geometry-aware-Instance-reweighted-Adversarial-Training/blob/main/GAIRAT.py +""" + + + + +import os +import argparse +import torchvision +import torch.optim as optim +from torchvision import transforms +from models import * +from GAIR import GAIR +import numpy as np +import attack_generator as attack +from utils import Logger + +parser = argparse.ArgumentParser(description='GAIRAT: Geometry-aware instance-dependent adversarial training') +parser.add_argument('--epochs', type=int, default=120, metavar='N', help='number of epochs to train') +parser.add_argument('--weight-decay', '--wd', default=2e-4, type=float, metavar='W') +parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='SGD momentum') +parser.add_argument('--epsilon', type=float, default=0.031, help='perturbation bound') +parser.add_argument('--num-steps', type=int, default=10, help='maximum perturbation step K') +parser.add_argument('--step-size', type=float, default=0.007, help='step size') +parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed') +parser.add_argument('--net', type=str, default="WRN",help="decide which network to use,choose from smallcnn,resnet18,WRN") +parser.add_argument('--dataset', type=str, default="cifar10", help="choose from cifar10,svhn,cifar100,mnist") +parser.add_argument('--random',type=bool,default=True,help="whether to initiat adversarial sample with random noise") +parser.add_argument('--depth',type=int,default=32,help='WRN depth') +parser.add_argument('--width-factor',type=int,default=10,help='WRN width factor') +parser.add_argument('--drop-rate',type=float,default=0.0, help='WRN drop rate') +parser.add_argument('--resume',type=str,default=None,help='whether to resume training') +parser.add_argument('--out-dir',type=str,default='./GAIRAT_result',help='dir of output') +parser.add_argument('--lr-schedule', default='piecewise', choices=['superconverge', 'piecewise', 'linear', 'onedrop', 'multipledecay', 'cosine']) +parser.add_argument('--lr-max', default=0.1, type=float) +parser.add_argument('--lr-one-drop', default=0.01, type=float) +parser.add_argument('--lr-drop-epoch', default=100, type=int) +parser.add_argument('--Lambda',type=str, default='-1.0', help='parameter for GAIR') +parser.add_argument('--Lambda_max',type=float, default=float('inf'), help='max Lambda') +parser.add_argument('--Lambda_schedule', default='fixed', choices=['linear', 'piecewise', 'fixed']) +parser.add_argument('--weight_assignment_function', default='Tanh', choices=['Discrete','Sigmoid','Tanh']) +parser.add_argument('--begin_epoch', type=int, default=60, help='when to use GAIR') +args = parser.parse_args() + +# Training settings +seed = args.seed +momentum = args.momentum +weight_decay = args.weight_decay +depth = args.depth +width_factor = args.width_factor +drop_rate = args.drop_rate +resume = args.resume +out_dir = args.out_dir + +torch.manual_seed(seed) +np.random.seed(seed) +torch.cuda.manual_seed_all(seed) +torch.backends.cudnn.benchmark = True +torch.backends.cudnn.deterministic = True + +# Models and optimizer +if args.net == "smallcnn": + model = SmallCNN().cuda() + net = "smallcnn" +if args.net == "resnet18": + model = ResNet18().cuda() + net = "resnet18" +if args.net == "preactresnet18": + model = PreActResNet18().cuda() + net = "preactresnet18" +if args.net == "WRN": + model = Wide_ResNet_Madry(depth=depth, num_classes=10, widen_factor=width_factor, dropRate=drop_rate).cuda() + net = "WRN{}-{}-dropout{}".format(depth,width_factor,drop_rate) + +model = torch.nn.DataParallel(model) +optimizer = optim.SGD(model.parameters(), lr=args.lr_max, momentum=momentum, weight_decay=weight_decay) + +# Learning schedules +if args.lr_schedule == 'superconverge': + lr_schedule = lambda t: np.interp([t], [0, args.epochs * 2 // 5, args.epochs], [0, args.lr_max, 0])[0] +elif args.lr_schedule == 'piecewise': + def lr_schedule(t): + if args.epochs >= 110: + # Train Wide-ResNet + if t / args.epochs < 0.5: + return args.lr_max + elif t / args.epochs < 0.75: + return args.lr_max / 10. + elif t / args.epochs < (11/12): + return args.lr_max / 100. + else: + return args.lr_max / 200. + else: + # Train ResNet + if t / args.epochs < 0.3: + return args.lr_max + elif t / args.epochs < 0.6: + return args.lr_max / 10. + else: + return args.lr_max / 100. +elif args.lr_schedule == 'linear': + lr_schedule = lambda t: np.interp([t], [0, args.epochs // 3, args.epochs * 2 // 3, args.epochs], [args.lr_max, args.lr_max, args.lr_max / 10, args.lr_max / 100])[0] +elif args.lr_schedule == 'onedrop': + def lr_schedule(t): + if t < args.lr_drop_epoch: + return args.lr_max + else: + return args.lr_one_drop +elif args.lr_schedule == 'multipledecay': + def lr_schedule(t): + return args.lr_max - (t//(args.epochs//10))*(args.lr_max/10) +elif args.lr_schedule == 'cosine': + def lr_schedule(t): + return args.lr_max * 0.5 * (1 + np.cos(t / args.epochs * np.pi)) + +# Store path +if not os.path.exists(out_dir): + os.makedirs(out_dir) + +# Save checkpoint +def save_checkpoint(state, checkpoint=out_dir, filename='checkpoint.pth.tar'): + filepath = os.path.join(checkpoint, filename) + torch.save(state, filepath) + +# Get adversarially robust network +def train(epoch, end_epoch, model, train_loader, test_loader, optimizer, Lambda): + + lr = 0 + num_data = 0 + train_robust_loss = 0 + + x_adv_res = np.zeros((1,3*32*32)) + data_res = np.zeros((1,3*32*32)) + kappa_res = np.zeros((1)) + logit_res = np.zeros((1,10)) + logit_clean_res = np.zeros((1,10)) + weight_res = np.zeros((1)) + labels_res = np.zeros((1)) + + for batch_idx, (data, target) in enumerate(train_loader): + loss = 0 + data, target = data.cuda(), target.cuda() + + # Get adversarial data and geometry value + x_adv, Kappa = attack.GA_PGD(model,data,target,args.epsilon,args.step_size,args.num_steps,loss_fn="cent",category="Madry",rand_init=True) + + model.train() + lr = lr_schedule(epoch + 1) + optimizer.param_groups[0].update(lr=lr) + optimizer.zero_grad() + + logit = model(x_adv) + logit_clean = model(data) + + shp = x_adv.shape + if len(shp) == 2: + x_adv_out = x_adv + data_out = data + else: + x_adv_out = x_adv.reshape(shp[0], shp[1]*shp[2]*shp[3]) + data_out = data.reshape(shp[0], shp[1]*shp[2]*shp[3]) + x_adv_res = np.vstack((x_adv_res, x_adv_out.detach().cpu().numpy())) + kappa_res = np.append(kappa_res, Kappa.cpu().numpy()) + logit_res = np.vstack((logit_res, logit.detach().cpu().numpy())) + data_res = np.vstack((data_res, data_out.detach().cpu().numpy())) + logit_clean_res = np.vstack((logit_clean_res, logit_clean.detach().cpu().numpy())) + labels_res = np.append(labels_res,target.cpu().numpy()) + + if (epoch + 1) >= args.begin_epoch: + Kappa = Kappa.cuda() + loss = nn.CrossEntropyLoss(reduce=False)(logit, target) + # Calculate weight assignment according to geometry value + normalized_reweight = GAIR(args.num_steps, Kappa, Lambda, args.weight_assignment_function) + loss = loss.mul(normalized_reweight).mean() + + weight_res = np.append(weight_res, normalized_reweight.cpu().numpy()) + + else: + loss = nn.CrossEntropyLoss(reduce="mean")(logit, target) + + train_robust_loss += loss.item() * len(x_adv) + + loss.backward() + optimizer.step() + + num_data += len(data) + + train_robust_loss = train_robust_loss / num_data + + x_adv_res = x_adv_res[1:] + kappa_res = kappa_res[1:] + logit_res = logit_res[1:] + logit_clean_res = logit_clean_res[1:] + data_res = data_res[1:] + weight_res = weight_res[1:] + labels_res = labels_res[1:] + + return train_robust_loss, lr, x_adv_res, kappa_res, logit_res, data_res, logit_clean_res, weight_res, labels_res + +# Adjust lambda for weight assignment using epoch +def adjust_Lambda(epoch): + Lam = float(args.Lambda) + if args.epochs >= 110: + # Train Wide-ResNet + Lambda = args.Lambda_max + if args.Lambda_schedule == 'linear': + if epoch >= 60: + Lambda = args.Lambda_max - (epoch/args.epochs) * (args.Lambda_max - Lam) + elif args.Lambda_schedule == 'piecewise': + if epoch >= 60: + Lambda = Lam + elif epoch >= 90: + Lambda = Lam-1.0 + elif epoch >= 110: + Lambda = Lam-1.5 + elif args.Lambda_schedule == 'fixed': + if epoch >= 60: + Lambda = Lam + else: + # Train ResNet + Lambda = args.Lambda_max + if args.Lambda_schedule == 'linear': + if epoch >= 30: + Lambda = args.Lambda_max - (epoch/args.epochs) * (args.Lambda_max - Lam) + elif args.Lambda_schedule == 'piecewise': + if epoch >= 30: + Lambda = Lam + elif epoch >= 60: + Lambda = Lam-2.0 + elif args.Lambda_schedule == 'fixed': + if epoch >= 30: + Lambda = Lam + return Lambda + +# Setup data loader +transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), +]) +transform_test = transforms.Compose([ + transforms.ToTensor(), +]) + +if args.dataset == "cifar10": + trainset = torchvision.datasets.CIFAR10(root='./data/cifar-10', train=True, download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=False, num_workers=2) + testset = torchvision.datasets.CIFAR10(root='./data/cifar-10', train=False, download=True, transform=transform_test) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2) +if args.dataset == "svhn": + trainset = torchvision.datasets.SVHN(root='./data/SVHN', split='train', download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=False, num_workers=2) + testset = torchvision.datasets.SVHN(root='./data/SVHN', split='test', download=True, transform=transform_test) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2) +if args.dataset == "mnist": + trainset = torchvision.datasets.MNIST(root='./data/MNIST', train=True, download=True, transform=transforms.ToTensor()) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=False, num_workers=1,pin_memory=True) + testset = torchvision.datasets.MNIST(root='./data/MNIST', train=False, download=True, transform=transforms.ToTensor()) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=1,pin_memory=True) + +# Resume +title = 'GAIRAT' +best_acc = 0 +start_epoch = 0 +if resume: + # Resume directly point to checkpoint.pth.tar + print ('==> GAIRAT Resuming from checkpoint ..') + print(resume) + assert os.path.isfile(resume) + out_dir = os.path.dirname(resume) + checkpoint = torch.load(resume) + start_epoch = checkpoint['epoch'] + best_acc = checkpoint['test_pgd20_acc'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + logger_test = Logger(os.path.join(out_dir, 'log_results.txt'), title=title, resume=True) +else: + print('==> GAIRAT') + logger_test = Logger(os.path.join(out_dir, 'log_results.txt'), title=title) + logger_test.set_names(['Epoch', 'Natural Test Acc', 'PGD20 Acc']) + +## Training get started +test_nat_acc = 0 +test_pgd20_acc = 0 + +best_x_adv_res = np.array([]) +best_kappa_res = np.array([]) +best_logit_res = np.array([]) +best_data_res = np.array([]) +best_logit_clean_res = np.array([]) +best_weight_res = np.array([]) +best_labels_res = np.array([]) + +output_ad = r'{}/'.format(args.out_dir) +if not os.path.exists(output_ad): + os.makedirs(output_ad) + +for epoch in range(start_epoch, args.epochs): + + # Get lambda + Lambda = adjust_Lambda(epoch + 1) + + # Adversarial training + train_robust_loss, lr, x_adv_res, kappa_res, logit_res, data_res, logit_clean_res, weight_res, labels_res = train(epoch, args.epochs, model, train_loader, test_loader, optimizer, Lambda) + + # Evalutions similar to DAT. + _, test_nat_acc = attack.eval_clean(model, test_loader) + _, test_pgd20_acc = attack.eval_robust(model, test_loader, perturb_steps=20, epsilon=0.031, step_size=0.031 / 4,loss_fn="cent", category="Madry", random=True) + + + print( + 'Epoch: [%d | %d] | Learning Rate: %f | Natural Test Acc %.2f | PGD20 Test Acc %.2f |\n' % ( + epoch, + args.epochs, + lr, + test_nat_acc, + test_pgd20_acc) + ) + + logger_test.append([epoch + 1, test_nat_acc, test_pgd20_acc]) + + # Save the best checkpoint + if test_pgd20_acc > best_acc: + best_acc = test_pgd20_acc + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'test_nat_acc': test_nat_acc, + 'test_pgd20_acc': test_pgd20_acc, + 'optimizer' : optimizer.state_dict(), + },filename='bestpoint.pth.tar') + + best_x_adv_res = x_adv_res + best_kappa_res=kappa_res + best_logit_res = logit_res + best_data_res = data_res + best_logit_clean_res = logit_clean_res + best_weight_res = weight_res + best_labels_res = labels_res + + # Save the last checkpoint + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'test_nat_acc': test_nat_acc, + 'test_pgd20_acc': test_pgd20_acc, + 'optimizer' : optimizer.state_dict(), + }) + + np.savetxt("{}best_x_adv.csv".format(output_ad),best_x_adv_res, "%.3f") + np.savetxt("{}best_kappa.csv".format(output_ad),best_kappa_res, "%.1f") + np.savetxt("{}best_logit.csv".format(output_ad),best_logit_res, "%.3f") + np.savetxt("{}best_logit_clean.csv".format(output_ad),best_logit_clean_res, "%.3f") + np.savetxt("{}best_data.csv".format(output_ad),best_data_res, "%.3f") + np.savetxt("{}best_weight_res.csv".format(output_ad),best_weight_res, "%.3f") + np.savetxt("{}best_labels_res.csv".format(output_ad),best_labels_res, "%.3f") +logger_test.close() diff --git a/evaluation/GAIRAT/GAIR_FAT.py b/evaluation/GAIRAT/GAIR_FAT.py new file mode 100644 index 0000000..0df7285 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_FAT.py @@ -0,0 +1,327 @@ +import os +import argparse +import torchvision +import torch.optim as optim +from torchvision import transforms +from models import * +from earlystop import GA_earlystop +from GAIR import GAIR +import numpy as np +import attack_generator as attack +from utils import Logger + +parser = argparse.ArgumentParser(description='GAIR-FAT') +parser.add_argument('--epochs', type=int, default=120, metavar='N', help='number of epochs to train') +parser.add_argument('--weight-decay', '--wd', default=2e-4, type=float, metavar='W') +parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='SGD momentum') +parser.add_argument('--epsilon', type=float, default=0.031, help='perturbation bound') +parser.add_argument('--num-steps', type=int, default=10, help='maximum perturbation step K') +parser.add_argument('--step-size', type=float, default=0.007, help='step size') +parser.add_argument('--seed', type=int, default=3, metavar='S', help='random seed') +parser.add_argument('--net', type=str, default="WRN",help="decide which network to use,choose from smallcnn,resnet18,WRN") +parser.add_argument('--tau',type=int,default=0,help='slippery step tau') +parser.add_argument('--dataset', type=str, default="cifar10", help="choose from cifar10,svhn,cifar100,mnist") +parser.add_argument('--random',type=bool,default=True,help="whether to initiat adversarial sample with random noise") +parser.add_argument('--omega',type=float,default=0.0,help="random sample parameter") +parser.add_argument('--dynamictau',type=bool,default=True,help='whether to use dynamic tau') +parser.add_argument('--depth',type=int,default=32,help='WRN depth') +parser.add_argument('--width-factor',type=int,default=10,help='WRN width factor') +parser.add_argument('--drop-rate',type=float,default=0.0, help='WRN drop rate') +parser.add_argument('--resume',type=str,default=None,help='whether to resume training') +parser.add_argument('--out-dir',type=str,default='./GAIR_FAT_result',help='dir of output') +parser.add_argument('--lr-schedule', default='piecewise', choices=['superconverge', 'piecewise', 'linear', 'onedrop', 'multipledecay', 'cosine']) +parser.add_argument('--lr-max', default=0.1, type=float) +parser.add_argument('--lr-one-drop', default=0.01, type=float) +parser.add_argument('--lr-drop-epoch', default=100, type=int) +parser.add_argument('--Lambda',type=str, default='0.0', help='parameter for join loss') +parser.add_argument('--Lambda_max',type=float, default=float('inf'), help='max Lambda') +parser.add_argument('--Lambda_schedule', default='fixed', choices=['linear', 'piecewise', 'fixed']) +parser.add_argument('--weight_assignment_function', default='Tanh', choices=['Discrete','Sigmoid','Tanh']) +parser.add_argument('--begin_epoch', type=int, default=60, help='when to use GAIR') +args = parser.parse_args() + +# Training settings +seed = args.seed +epsilon = args.epsilon +step_size = args.step_size +num_steps = args.num_steps +dynamictau = args.dynamictau +momentum = args.momentum +weight_decay = args.weight_decay +omega = args.omega +random = args.random +depth = args.depth +width_factor = args.width_factor +drop_rate = args.drop_rate +resume = args.resume +out_dir = args.out_dir + +torch.manual_seed(seed) +np.random.seed(seed) +torch.cuda.manual_seed_all(seed) +torch.backends.cudnn.benchmark = False +torch.backends.cudnn.deterministic = True + +# Models and optimizer +if args.net == "smallcnn": + model = SmallCNN().cuda() + net = "smallcnn" +if args.net == "resnet18": + model = ResNet18().cuda() + net = "resnet18" +if args.net == "preactresnet18": + model = PreActResNet18().cuda() + net = "preactresnet18" +if args.net == "WRN": + model = Wide_ResNet_Madry(depth=depth, num_classes=10, widen_factor=width_factor, dropRate=drop_rate).cuda() + net = "WRN{}-{}-dropout{}".format(depth,width_factor,drop_rate) + +model = torch.nn.DataParallel(model) +optimizer = optim.SGD(model.parameters(), lr=args.lr_max, momentum=momentum, weight_decay=weight_decay) + +# Learning schedules +if args.lr_schedule == 'superconverge': + lr_schedule = lambda t: np.interp([t], [0, args.epochs * 2 // 5, args.epochs], [0, args.lr_max, 0])[0] +elif args.lr_schedule == 'piecewise': + def lr_schedule(t): + if args.epochs >= 110: + # Train Wide-ResNet + if t / args.epochs < 0.5: + return args.lr_max + elif t / args.epochs < 0.75: + return args.lr_max / 10. + elif t / args.epochs < (11/12): + return args.lr_max / 100. + else: + return args.lr_max / 200. + else: + # Train ResNet + if t / args.epochs < 0.3: + return args.lr_max + elif t / args.epochs < 0.6: + return args.lr_max / 10. + else: + return args.lr_max / 100. +elif args.lr_schedule == 'linear': + lr_schedule = lambda t: np.interp([t], [0, args.epochs // 3, args.epochs * 2 // 3, args.epochs], [args.lr_max, args.lr_max, args.lr_max / 10, args.lr_max / 100])[0] +elif args.lr_schedule == 'onedrop': + def lr_schedule(t): + if t < args.lr_drop_epoch: + return args.lr_max + else: + return args.lr_one_drop +elif args.lr_schedule == 'multipledecay': + def lr_schedule(t): + return args.lr_max - (t//(args.epochs//10))*(args.lr_max/10) +elif args.lr_schedule == 'cosine': + def lr_schedule(t): + return args.lr_max * 0.5 * (1 + np.cos(t / args.epochs * np.pi)) + +# Store path +if not os.path.exists(out_dir): + os.makedirs(out_dir) + +# Save checkpoint +def save_checkpoint(state, checkpoint=out_dir, filename='checkpoint.pth.tar'): + filepath = os.path.join(checkpoint, filename) + torch.save(state, filepath) + +# Get adversarially robust network +def train(epoch, end_epoch, model, train_loader, test_loader, optimizer, tau, Lambda): + + lr = 0 + num_data = 0 + train_robust_loss = 0 + + if epoch >= end_epoch - 1: + fp_x = open("x_adv.csv","a+") + fp_k = open("kappa.csv", "a+") + + for batch_idx, (data, target) in enumerate(train_loader): + + loss = 0 + data, target = data.cuda(), target.cuda() + + # Get adversarial data and geometry value + output_adv, output_target, output_natural, count, Kappa = GA_earlystop(model,data,target,step_size=args.step_size,epsilon=args.epsilon,perturb_steps=args.num_steps,tau=tau,type="fat",random=random,omega=omega) + + + shp = output_adv.shape + print(Kappa.shape) + if epoch >= end_epoch - 1: + if shp.shape == 2: + op_adv_out = output_adv + else: + op_adv_out = np.reshape(output_adv, (shp[0], shp[1]*shp[2]*shp[3])) + np.savetxt(fp_x, op_adv_out) + np.savetxt(fp_k, Kappa) + + model.train() + lr = lr_schedule(epoch + 1) + optimizer.param_groups[0].update(lr=lr) + optimizer.zero_grad() + + logit = model(output_adv) + + if (epoch + 1) >= args.begin_epoch: + Kappa = Kappa.cuda() + loss = nn.CrossEntropyLoss(reduce=False)(logit, target) + # Calculate weight assignment according to geometry value + normalized_reweight = GAIR(args.num_steps, Kappa, Lambda, args.weight_assignment_function) + loss = loss.mul(normalized_reweight).mean() + else: + loss = nn.CrossEntropyLoss(reduce="mean")(logit, output_target) + train_robust_loss += loss.item() * len(output_adv) + + loss.backward() + optimizer.step() + + num_data += len(data) + + train_robust_loss = train_robust_loss / num_data + + return train_robust_loss, lr + +# Adjust tau +def adjust_tau(epoch, dynamictau): + tau = args.tau + if dynamictau: + if epoch <= 40: + tau = 0 + elif epoch <= 70: + tau = 3 + else: + tau = 6 + return tau + +def adjust_Lambda(epoch): + Lam = float(args.Lambda) + if args.epochs >= 110: + # Train Wide-ResNet + Lambda = args.Lambda_max + if args.Lambda_schedule == 'linear': + if epoch >= 60: + Lambda = args.Lambda_max - (epoch/args.epochs) * (args.Lambda_max - Lam) + elif args.Lambda_schedule == 'piecewise': + if epoch >= 60: + Lambda = Lam + elif epoch >= 90: + Lambda = Lam-1.0 + elif epoch >= 110: + Lambda = Lam-1.5 + elif args.Lambda_schedule == 'fixed': + if epoch >= 60: + Lambda = Lam + else: + # Train ResNet + Lambda = args.Lambda_max + if args.Lambda_schedule == 'linear': + if epoch >= 30: + Lambda = args.Lambda_max - (epoch/args.epochs) * (args.Lambda_max - Lam) + elif args.Lambda_schedule == 'piecewise': + if epoch >= 30: + Lambda = Lam + elif epoch >= 60: + Lambda = Lam-2.0 + elif args.Lambda_schedule == 'fixed': + if epoch >= 30: + Lambda = Lam + return Lambda + +# Setup data loader +transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), +]) +transform_test = transforms.Compose([ + transforms.ToTensor(), +]) + +if args.dataset == "cifar10": + trainset = torchvision.datasets.CIFAR10(root='./data/cifar-10', train=True, download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2) + testset = torchvision.datasets.CIFAR10(root='./data/cifar-10', train=False, download=True, transform=transform_test) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2) +if args.dataset == "svhn": + trainset = torchvision.datasets.SVHN(root='./data/SVHN', split='train', download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2) + testset = torchvision.datasets.SVHN(root='./data/SVHN', split='test', download=True, transform=transform_test) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2) +if args.dataset == "mnist": + trainset = torchvision.datasets.MNIST(root='./data/MNIST', train=True, download=True, transform=transforms.ToTensor()) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=1,pin_memory=True) + testset = torchvision.datasets.MNIST(root='./data/MNIST', train=False, download=True, transform=transforms.ToTensor()) + test_loader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=1,pin_memory=True) + +# Resume +title = 'GAIR-FAT' +best_acc = 0 +start_epoch = 0 +if resume: + # Resume directly point to checkpoint.pth.tar + print ('==> GAIR-FAT Resuming from checkpoint ..') + print(resume) + assert os.path.isfile(resume) + out_dir = os.path.dirname(resume) + checkpoint = torch.load(resume) + start_epoch = checkpoint['epoch'] + best_acc = checkpoint['test_pgd20_acc'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + logger_test = Logger(os.path.join(out_dir, 'log_results.txt'), title=title, resume=True) +else: + print('==> GAIR-FAT') + logger_test = Logger(os.path.join(out_dir, 'log_results.txt'), title=title) + logger_test.set_names(['Epoch', 'Natural Test Acc', 'PGD20 Acc']) + +## Training get started +test_nat_acc = 0 +test_pgd20_acc = 0 + +for epoch in range(start_epoch, args.epochs): + + # Get tau and lambda + tau = adjust_tau(epoch + 1, dynamictau) + Lambda = adjust_Lambda(epoch + 1) + + # Adversarial training + train_robust_loss, lr = train(epoch, args.epochs, model, train_loader, test_loader, optimizer, tau, Lambda) + + # Evalutions similar to DAT. + _, test_nat_acc = attack.eval_clean(model, test_loader) + _, test_pgd20_acc = attack.eval_robust(model, test_loader, perturb_steps=20, epsilon=0.031, step_size=0.031 / 4,loss_fn="cent", category="Madry", random=True) + + + print( + 'Epoch: [%d | %d] | Learning Rate: %f | Natural Test Acc %.2f | PGD20 Test Acc %.2f |\n' % ( + epoch, + args.epochs, + lr, + test_nat_acc, + test_pgd20_acc) + ) + + logger_test.append([epoch + 1, test_nat_acc, test_pgd20_acc]) + + # Save the best checkpoint + if test_pgd20_acc > best_acc: + best_acc = test_pgd20_acc + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'test_nat_acc': test_nat_acc, + 'test_pgd20_acc': test_pgd20_acc, + 'optimizer' : optimizer.state_dict(), + },filename='bestpoint.pth.tar') + + # Save the last checkpoint + save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': model.state_dict(), + 'test_nat_acc': test_nat_acc, + 'test_pgd20_acc': test_pgd20_acc, + 'optimizer' : optimizer.state_dict(), + }) + +logger_test.close() \ No newline at end of file diff --git a/evaluation/GAIRAT/GAIR_RST/attack_cw.py b/evaluation/GAIRAT/GAIR_RST/attack_cw.py new file mode 100644 index 0000000..4411f8d --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/attack_cw.py @@ -0,0 +1,365 @@ +""" +Implementation of L_infty Carlini-Wagner attack based on the L2 implementation +in FoolBox v1.9 (with many dependencies on that pakage) +https://github.com/bethgelab/foolbox +""" + +try: + from foolbox.models import PyTorchModel + from foolbox.attacks import Attack + from foolbox.attacks.base import call_decorator + from foolbox.utils import onehot_like + from foolbox.distances import Linf + HAVE_FOOLBOX = True +except ImportError: + HAVE_FOOLBOX = False + Attack = object + call_decorator = lambda x: x +import numpy as np +import logging + + +def cw(model, + X, + y, + binary_search_steps=5, + max_iterations=1000, + learning_rate=5E-3, + initial_const=1E-2, + tau_decrease_factor=0.9 + ): + if not HAVE_FOOLBOX: + raise ImportError('Could not import FoolBox') + + foolbox_model = PyTorchModel(model, bounds=(0, 1), num_classes=10) + attack = CarliniWagnerLIAttack(foolbox_model, distance=Linf) + linf_distances = [] + for i in range(len(X)): + logging.info('Example: %g', i) + image = X[i, :].detach().cpu().numpy() + label = y[i].cpu().numpy() + adversarial = attack(image, label, + binary_search_steps=binary_search_steps, + max_iterations=max_iterations, + learning_rate=learning_rate, + initial_const=initial_const, + tau_decrease_factor=tau_decrease_factor) + logging.info('Linf distance: %g', np.max(np.abs(adversarial - image))) + linf_distances.append(np.max(np.abs(adversarial - image))) + return linf_distances + + +class CarliniWagnerLIAttack(Attack): + """The Linf version of the Carlini & Wagner attack. + + This attack is described in [1]_. This implementation + is based on the reference implementation by Carlini [2]_. + For bounds \ne (0, 1), it differs from [2]_ because we + normalize the squared L2 loss with the bounds. + + References + ---------- + .. [1] Nicholas Carlini, David Wagner: "Towards Evaluating the + Robustness of Neural Networks", https://arxiv.org/abs/1608.04644 + .. [2] https://github.com/carlini/nn_robust_attacks + + """ + + @call_decorator + def __call__(self, input_or_adv, label=None, unpack=True, + binary_search_steps=5, + tau_decrease_factor=0.9, + max_iterations=1000, + confidence=0, learning_rate=5e-3, + initial_const=1e-2, abort_early=True): + + """The L_infty version of the Carlini & Wagner attack. + + Parameters + ---------- + input_or_adv : `numpy.ndarray` or :class:`Adversarial` + The original, unperturbed input as a `numpy.ndarray` or + an :class:`Adversarial` instance. + label : int + The reference label of the original input. Must be passed + if `a` is a `numpy.ndarray`, must not be passed if `a` is + an :class:`Adversarial` instance. + unpack : bool + If true, returns the adversarial input, otherwise returns + the Adversarial object. + binary_search_steps : int + The number of steps for the binary search used to + find the optimal tradeoff-constant between distance and confidence. + max_iterations : int + The maximum number of iterations. Larger values are more + accurate; setting it too small will require a large learning rate + and will produce poor results. + confidence : int or float + Confidence of adversarial examples: a higher value produces + adversarials that are further away, but more strongly classified + as adversarial. + learning_rate : float + The learning rate for the attack algorithm. Smaller values + produce better results but take longer to converge. + initial_const : float + The initial tradeoff-constant to use to tune the relative + importance of distance and confidence. If `binary_search_steps` + is large, the initial constant is not important. + abort_early : bool + If True, Adam will be aborted if the loss hasn't decreased + for some time (a tenth of max_iterations). + + """ + + a = input_or_adv + + del input_or_adv + del label + del unpack + + if not a.has_gradient(): + logging.fatal('Applied gradient-based attack to model that ' + 'does not provide gradients.') + return + + min_, max_ = a.bounds() + + def to_attack_space(x): + # map from [min_, max_] to [-1, +1] + a = (min_ + max_) / 2 + b = (max_ - min_) / 2 + x = (x - a) / b + + # from [-1, +1] to approx. (-1, +1) + x = x * 0.999999 + + # from (-1, +1) to (-inf, +inf) + return np.arctanh(x) + + def to_model_space(x): + """Transforms an input from the attack space + to the model space. This transformation and + the returned gradient are elementwise.""" + + # from (-inf, +inf) to (-1, +1) + x = np.tanh(x) + + grad = 1 - np.square(x) + + # map from (-1, +1) to (min_, max_) + a = (min_ + max_) / 2 + b = (max_ - min_) / 2 + x = x * b + a + + grad = grad * b + return x, grad + + # variables representing inputs in attack space will be + # prefixed with att_ + att_original = to_attack_space(a.original_image) + + # will be close but not identical to a.original_image + reconstructed_original, _ = to_model_space(att_original) + + # the binary search finds the smallest const for which we + # find an adversarial + const = initial_const + lower_bound = 0 + upper_bound = np.inf + true_logits, _ = a.predictions(reconstructed_original) + true_label = np.argmax(true_logits) + # Binary search for constant + start_tau = 1.0 + for binary_search_step in range(binary_search_steps): + if binary_search_step == binary_search_steps - 1 and \ + binary_search_steps >= 10: + # in the last binary search step, use the upper_bound instead + # TODO: find out why... it's not obvious why this is useful + const = upper_bound + + logging.info( + 'starting optimization with const = {}, best overall distance = {}'.format( + const, a.distance)) + + # found adv with the current const + + att_warmstart = att_original + tau = start_tau + while tau > 1. / 256: + found_adv = False + att_perturbation = np.zeros_like(att_original) + # create a new optimizer to minimize the perturbation + optimizer = AdamOptimizer(att_perturbation.shape) + loss_at_previous_check = np.inf + # logging.info('Running with const:{}, tau:{}'.format(const, tau)) + for iteration in range(max_iterations): + + x, dxdp = to_model_space(att_warmstart + att_perturbation) + logits, i = a.predictions(x) + false_label = np.argmax(logits) + is_adv = not (false_label == true_label) + + loss, dldx, adv_loss, distance = self.loss_function( + const, tau, a, x, logits, reconstructed_original, + confidence, min_, max_) + + check_loss = logits[true_label] - logits[false_label] + + # logging.info('Iteration:{}, loss: {}; adv_loss:{}; distance:{}'.format( + # iteration, loss, adv_loss, distance)) + # backprop the gradient of the loss w.r.t. x further + # to get the gradient of the loss w.r.t. att_perturbation + assert dldx.shape == x.shape + assert dxdp.shape == x.shape + # we can do a simple elementwise multiplication, because + # grad_x_wrt_p is a matrix of elementwise derivatives + # (i.e. each x[i] w.r.t. p[i] only, for all i) and + # grad_loss_wrt_x is a real gradient reshaped as a matrix + gradient = dldx * dxdp + + att_perturbation += optimizer(gradient, learning_rate) + x, dxdp = to_model_space(att_warmstart + att_perturbation) + + if is_adv: + # Tau + binary search was successful but continuing opt + found_adv = True + + if abort_early and \ + iteration % (np.ceil(max_iterations / 10)) == 0: + # after each tenth of the iterations, check progress + # logging.info('Iteration:{}, loss: {}; best overall distance: {}; is_adv:{}'.format( + # iteration, loss, a.distance, is_adv)) + if not (loss_at_previous_check - loss > 0.0001): + break # stop Adam if there has not been progress + loss_at_previous_check = loss + + if not found_adv: + # This constant is fine, just that we broke out of tau + if tau < start_tau: + found_adv = True + start_tau = tau + break + + else: + actualtau = np.max(np.abs(x - reconstructed_original)) + if actualtau < tau: + tau = actualtau + tau = tau * tau_decrease_factor + att_warmstart = to_attack_space(x) + + if found_adv: + # logging.info('found adversarial with const = {}'.format(const)) + upper_bound = const + else: + # logging.info('failed to find adversarial ' + # 'with const = {}'.format(const)) + lower_bound = const + + if upper_bound == np.inf: + # exponential search + const *= 10 + else: + # binary search + const = (lower_bound + upper_bound) / 2 + + @classmethod + def loss_function(cls, const, tau, a, x, logits, reconstructed_original, + confidence, min_, max_): + """Returns the loss and the gradient of the loss w.r.t. x, + assuming that logits = model(x).""" + + targeted = a.target_class() is not None + if targeted: + c_minimize = cls.best_other_class(logits, a.target_class()) + c_maximize = a.target_class() + else: + c_minimize = a.original_class + c_maximize = cls.best_other_class(logits, a.original_class) + + is_adv_loss = logits[c_minimize] - logits[c_maximize] + + # is_adv is True as soon as the is_adv_loss goes below 0 + # but sometimes we want additional confidence + is_adv_loss += confidence + is_adv_loss = max(0, is_adv_loss) + + s = max_ - min_ + # squared_l2_distance = np.sum((x - reconstructed_original)**2) / s**2 + linf_distance = np.sum( + np.maximum(np.abs(x - reconstructed_original) - tau, 0)) + + total_loss = linf_distance + const * is_adv_loss + + # calculate the gradient of total_loss w.r.t. x + logits_diff_grad = np.zeros_like(logits) + logits_diff_grad[c_minimize] = 1 + logits_diff_grad[c_maximize] = -1 + is_adv_loss_grad = a.backward(logits_diff_grad, x) + assert is_adv_loss >= 0 + if is_adv_loss == 0: + is_adv_loss_grad = 0 + + squared_l2_distance_grad = (2 / s ** 2) * (x - reconstructed_original) + linf_distance_grad = np.sign(x - reconstructed_original) * ( + np.abs(x - reconstructed_original) - tau > 0) + total_loss_grad = linf_distance_grad + const * is_adv_loss_grad + return total_loss, total_loss_grad, is_adv_loss, linf_distance + + @staticmethod + def best_other_class(logits, exclude): + """Returns the index of the largest logit, ignoring the class that + is passed as `exclude`.""" + other_logits = logits - onehot_like(logits, exclude, value=np.inf) + return np.argmax(other_logits) + + +class AdamOptimizer: + """Basic Adam optimizer implementation that can minimize w.r.t. + a single variable. + + Parameters + ---------- + shape : tuple + shape of the variable w.r.t. which the loss should be minimized + + """ + + def __init__(self, shape): + self.m = np.zeros(shape) + self.v = np.zeros(shape) + self.t = 0 + + def __call__(self, gradient, learning_rate, + beta1=0.9, beta2=0.999, epsilon=10e-8): + """Updates internal parameters of the optimizer and returns + the change that should be applied to the variable. + + Parameters + ---------- + gradient : `np.ndarray` + the gradient of the loss w.r.t. to the variable + learning_rate: float + the learning rate in the current iteration + beta1: float + decay rate for calculating the exponentially + decaying average of past gradients + beta2: float + decay rate for calculating the exponentially + decaying average of past squared gradients + epsilon: float + small value to avoid division by zero + + """ + + self.t += 1 + + self.m = beta1 * self.m + (1 - beta1) * gradient + self.v = beta2 * self.v + (1 - beta2) * gradient ** 2 + + bias_correction_1 = 1 - beta1 ** self.t + bias_correction_2 = 1 - beta2 ** self.t + + m_hat = self.m / bias_correction_1 + v_hat = self.v / bias_correction_2 + + return -learning_rate * m_hat / (np.sqrt(v_hat) + epsilon) diff --git a/evaluation/GAIRAT/GAIR_RST/attack_evaluation.py b/evaluation/GAIRAT/GAIR_RST/attack_evaluation.py new file mode 100644 index 0000000..109f12f --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/attack_evaluation.py @@ -0,0 +1,246 @@ +""" +Evaluate robustness against specific attack. +Loosely based on code from https://github.com/yaodongyu/TRADES +""" + +import os +import json +import numpy as np +import argparse +import logging +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from torch.autograd import Variable +from datasets import SemiSupervisedDataset, DATASETS +from torchvision import transforms +from attack_pgd import pgd +from attack_cw import cw +import torch.backends.cudnn as cudnn +from utils import get_model + + +def eval_adv_test(model, device, test_loader, attack, attack_params, + results_dir, num_eval_batches): + """ + evaluate model by white-box attack + """ + model.eval() + if attack == 'pgd': + restarts_matrices = [] + for restart in range(attack_params['num_restarts']): + is_correct_adv_rows = [] + count = 0 + batch_num = 0 + natural_num_correct = 0 + for data, target in test_loader: + batch_num = batch_num + 1 + if num_eval_batches and batch_num > num_eval_batches: + break + data, target = data.to(device), target.to(device) + count += len(target) + X, y = Variable(data, requires_grad=True), Variable(target) + # is_correct_adv has batch_size*num_iterations dimensions + is_correct_natural, is_correct_adv = pgd( + model, X, y, + epsilon=attack_params['epsilon'], + num_steps=attack_params['num_steps'], + step_size=attack_params['step_size'], + random_start=attack_params['random_start']) + natural_num_correct += is_correct_natural.sum() + is_correct_adv_rows.append(is_correct_adv) + + is_correct_adv_matrix = np.concatenate(is_correct_adv_rows, axis=0) + restarts_matrices.append(is_correct_adv_matrix) + + is_correct_adv_over_restarts = np.stack(restarts_matrices, axis=-1) + num_correct_adv = is_correct_adv_over_restarts.prod( + axis=-1).prod(axis=-1).sum() + + logging.info("Accuracy after %d restarts: %.4g%%" % + (restart + 1, 100 * num_correct_adv / count)) + stats = {'attack': 'pgd', + 'count': count, + 'attack_params': attack_params, + 'natural_accuracy': natural_num_correct / count, + 'is_correct_adv_array': is_correct_adv_over_restarts, + 'robust_accuracy': num_correct_adv / count, + 'restart_num': restart + } + + np.save(os.path.join(results_dir, 'pgd_results.npy'), stats) + + elif attack == 'cw': + all_linf_distances = [] + count = 0 + for data, target in test_loader: + logging.info('Batch: %g', count) + count = count + 1 + if num_eval_batches and count > num_eval_batches: + break + data, target = data.to(device), target.to(device) + X, y = Variable(data, requires_grad=True), Variable(target) + batch_linf_distances = cw(model, X, y, + binary_search_steps=attack_params[ + 'binary_search_steps'], + max_iterations=attack_params[ + 'max_iterations'], + learning_rate=attack_params[ + 'learning_rate'], + initial_const=attack_params[ + 'initial_const'], + tau_decrease_factor=attack_params[ + 'tau_decrease_factor']) + all_linf_distances.append(batch_linf_distances) + stats = {'attack': 'cw', + 'attack_params': attack_params, + 'linf_distances': np.array(all_linf_distances), + } + np.save(os.path.join(results_dir, 'cw_results.npy'), stats) + + else: + raise ValueError('Unknown attack %s' % attack) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='PyTorch CIFAR Attack Evaluation') + parser.add_argument('--dataset', type=str, default='cifar10', + choices=DATASETS, + help='The dataset') + parser.add_argument('--model_path', + help='Model for attack evaluation') + parser.add_argument('--model', '-m', default='wrn-32-10', type=str, + help='Name of the model') + parser.add_argument('--output_suffix', default='', type=str, + help='String to add to log filename') + parser.add_argument('--batch_size', type=int, default=128, metavar='N', + help='Input batch size for testing (default: 200)') + parser.add_argument('--no_cuda', action='store_true', default=False, + help='Disables CUDA training') + parser.add_argument('--epsilon', default=0.031, type=float, + help='Attack perturbation magnitude') + parser.add_argument('--attack', default='pgd', type=str, + help='Attack type (CW requires FoolBox)', + choices=('pgd', 'cw')) + parser.add_argument('--num_steps', default=40, type=int, + help='Number of PGD steps') + parser.add_argument('--step_size', default=0.01, type=float, + help='PGD step size') + parser.add_argument('--num_restarts', default=5, type=int, + help='Number of restarts for PGD attack') + parser.add_argument('--no_random_start', dest='random_start', + action='store_false', + help='Disable random PGD initialization') + parser.add_argument('--binary_search_steps', default=5, type=int, + help='Number of binary search steps for CW attack') + parser.add_argument('--max_iterations', default=1000, type=int, + help='Max number of Adam iterations in each CW' + ' optimization') + parser.add_argument('--learning_rate', default=5E-3, type=float, + help='Learning rate for CW attack') + parser.add_argument('--initial_const', default=1E-2, type=float, + help='Initial constant for CW attack') + parser.add_argument('--tau_decrease_factor', default=0.9, type=float, + help='Tau decrease factor for CW attack') + parser.add_argument('--random_seed', default=0, type=int, + help='Random seed for permutation of test instances') + parser.add_argument('--num_eval_batches', default=None, type=int, + help='Number of batches to run evalaution on') + parser.add_argument('--shuffle_testset', action='store_true', default=False, + help='Shuffles the test set') + + args = parser.parse_args() + torch.manual_seed(args.random_seed) + + output_dir, checkpoint_name = os.path.split(args.model_path) + + results_dir = os.path.join('./', args.output_suffix) + if not os.path.isdir(results_dir): + os.mkdir(results_dir) + #epoch = int(re.search('epoch(\d+)', checkpoint_name).group(1)) + epoch = 0 + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(message)s", + handlers=[ + logging.FileHandler(os.path.join(results_dir, + 'attack_epoch%d%s.log' % + (epoch, args.output_suffix))), + logging.StreamHandler() + ]) + logger = logging.getLogger() + + + + logging.info('Attack evaluation') + logging.info('Args: %s' % args) + + # settings + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + dl_kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} + + # set up data loader + transform_test = transforms.Compose([transforms.ToTensor(), ]) + testset = SemiSupervisedDataset(base_dataset=args.dataset, + train=False, root='./data/cifar-10', + download=True, + transform=transform_test) + + if args.shuffle_testset: + np.random.seed(123) + logging.info("Permuting testset") + permutation = np.random.permutation(len(testset)) + testset.data = testset.data[permutation, :] + testset.targets = [testset.targets[i] for i in permutation] + + test_loader = torch.utils.data.DataLoader(testset, + batch_size=args.batch_size, + shuffle=False, **dl_kwargs) + + checkpoint = torch.load(args.model_path) + state_dict = checkpoint.get('state_dict', checkpoint) + num_classes = checkpoint.get('num_classes', 10) + normalize_input = checkpoint.get('normalize_input', False) + model = get_model(args.model, num_classes=num_classes, + normalize_input=normalize_input) + if use_cuda: + model = torch.nn.DataParallel(model).cuda() + cudnn.benchmark = True + if not all([k.startswith('module') for k in state_dict]): + state_dict = {'module.' + k: v for k, v in state_dict.items()} + else: + def strip_data_parallel(s): + if s.startswith('module'): + return s[len('module.'):] + else: + return s + state_dict = {strip_data_parallel(k): v for k, v in state_dict.items()} + model.load_state_dict(state_dict) + + attack_params = { + 'epsilon': args.epsilon, + 'seed': args.random_seed + } + if args.attack == 'pgd': + attack_params.update({ + 'num_restarts': args.num_restarts, + 'step_size': args.step_size, + 'num_steps': args.num_steps, + 'random_start': args.random_start, + }) + elif args.attack == 'cw': + attack_params.update({ + 'binary_search_steps': args.binary_search_steps, + 'max_iterations': args.max_iterations, + 'learning_rate': args.learning_rate, + 'initial_const': args.initial_const, + 'tau_decrease_factor': args.tau_decrease_factor + }) + + logging.info('Running %s' % attack_params) + eval_adv_test(model, device, test_loader, attack=args.attack, + attack_params=attack_params, results_dir=results_dir, + num_eval_batches=args.num_eval_batches) diff --git a/evaluation/GAIRAT/GAIR_RST/attack_pgd.py b/evaluation/GAIRAT/GAIR_RST/attack_pgd.py new file mode 100644 index 0000000..5180c4e --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/attack_pgd.py @@ -0,0 +1,53 @@ +### PGD implementation + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from torch.autograd import Variable +import torch.optim as optim +import logging + + +def pgd(model, + X, + y, + epsilon=8 / 255, + num_steps=20, + step_size=0.01, + random_start=True): + out = model(X) + is_correct_natural = (out.max(1)[1] == y).float().cpu().numpy() + perturbation = torch.zeros_like(X, requires_grad=True) + + if random_start: + perturbation = torch.rand_like(X, requires_grad=True) + perturbation.data = perturbation.data * 2 * epsilon - epsilon + + is_correct_adv = [] + opt = optim.SGD([perturbation], lr=1e-3) # This is just to clear the grad + + for _ in range(num_steps): + opt.zero_grad() + + with torch.enable_grad(): + loss = nn.CrossEntropyLoss()(model(X + perturbation), y) + + loss.backward() + + perturbation.data = ( + perturbation + step_size * perturbation.grad.detach().sign()).clamp( + -epsilon, epsilon) + perturbation.data = torch.min(torch.max(perturbation.detach(), -X), + 1 - X) # clip X+delta to [0,1] + X_pgd = Variable(torch.clamp(X.data + perturbation.data, 0, 1.0), + requires_grad=False) + is_correct_adv.append(np.reshape( + (model(X_pgd).max(1)[1] == y).float().cpu().numpy(), + [-1, 1])) + + is_correct_adv = np.concatenate(is_correct_adv, axis=1) + return is_correct_natural, is_correct_adv + + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/__init__.py b/evaluation/GAIRAT/GAIR_RST/autoattack/__init__.py new file mode 100644 index 0000000..00ac5fa --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/__init__.py @@ -0,0 +1 @@ +from .autoattack import AutoAttack diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/autoattack.py b/evaluation/GAIRAT/GAIR_RST/autoattack/autoattack.py new file mode 100644 index 0000000..9daab82 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/autoattack.py @@ -0,0 +1,252 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable +import numpy as np +import argparse +import time +from other_utils import Logger + +class AutoAttack(): + def __init__(self, model, norm='Linf', eps=.3, seed=None, verbose=True, + attacks_to_run=[], version='standard', is_tf_model=False, + device='cuda', log_path=None): + self.model = model + self.norm = norm + assert norm in ['Linf', 'L2'] + self.epsilon = eps + self.seed = seed + self.verbose = verbose + self.attacks_to_run = attacks_to_run + self.version = version + self.is_tf_model = is_tf_model + self.device = device + self.logger = Logger(log_path) + + if not self.is_tf_model: + from autopgd_pt import APGDAttack + self.apgd = APGDAttack(self.model, n_restarts=5, n_iter=100, verbose=False, + eps=self.epsilon, norm=self.norm, eot_iter=1, rho=.75, seed=self.seed, device=self.device) + + from fab_pt import FABAttack + self.fab = FABAttack(self.model, n_restarts=5, n_iter=100, eps=self.epsilon, seed=self.seed, + norm=self.norm, verbose=False, device=self.device) + + from square import SquareAttack + self.square = SquareAttack(self.model, p_init=.8, n_queries=5000, eps=self.epsilon, norm=self.norm, + n_restarts=1, seed=self.seed, verbose=False, device=self.device, resc_schedule=False) + + from autopgd_pt import APGDAttack_targeted + self.apgd_targeted = APGDAttack_targeted(self.model, n_restarts=1, n_iter=100, verbose=False, + eps=self.epsilon, norm=self.norm, eot_iter=1, rho=.75, seed=self.seed, device=self.device) + + else: + from autopgd_tf import APGDAttack + self.apgd = APGDAttack(self.model, n_restarts=5, n_iter=100, verbose=False, + eps=self.epsilon, norm=self.norm, eot_iter=1, rho=.75, seed=self.seed, device=self.device) + + from fab_tf import FABAttack + self.fab = FABAttack(self.model, n_restarts=5, n_iter=100, eps=self.epsilon, seed=self.seed, + norm=self.norm, verbose=False, device=self.device) + + from square import SquareAttack + self.square = SquareAttack(self.model.predict, p_init=.8, n_queries=5000, eps=self.epsilon, norm=self.norm, + n_restarts=1, seed=self.seed, verbose=False, device=self.device, resc_schedule=False) + + from autopgd_tf import APGDAttack_targeted + self.apgd_targeted = APGDAttack_targeted(self.model, n_restarts=1, n_iter=100, verbose=False, + eps=self.epsilon, norm=self.norm, eot_iter=1, rho=.75, seed=self.seed, device=self.device) + + if version in ['standard', 'plus', 'rand']: + self.set_version(version) + + def get_logits(self, x): + if not self.is_tf_model: + return self.model(x) + else: + return self.model.predict(x) + + def get_seed(self): + return time.time() if self.seed is None else self.seed + + def run_standard_evaluation(self, x_orig, y_orig, bs=250): + if self.verbose: + print('using {} version including {}'.format(self.version, + ', '.join(self.attacks_to_run))) + + with torch.no_grad(): + # calculate accuracy + n_batches = int(np.ceil(x_orig.shape[0] / bs)) + robust_flags = torch.zeros(x_orig.shape[0], dtype=torch.bool, device=x_orig.device) + for batch_idx in range(n_batches): + start_idx = batch_idx * bs + end_idx = min( (batch_idx + 1) * bs, x_orig.shape[0]) + + x = x_orig[start_idx:end_idx, :].clone().to(self.device) + y = y_orig[start_idx:end_idx].clone().to(self.device) + output = self.get_logits(x) + correct_batch = y.eq(output.max(dim=1)[1]) + robust_flags[start_idx:end_idx] = correct_batch.detach().to(robust_flags.device) + + robust_accuracy = torch.sum(robust_flags).item() / x_orig.shape[0] + + if self.verbose: + self.logger.log('initial accuracy: {:.2%}'.format(robust_accuracy)) + + x_adv = x_orig.clone().detach() + startt = time.time() + for attack in self.attacks_to_run: + # item() is super important as pytorch int division uses floor rounding + num_robust = torch.sum(robust_flags).item() + + if num_robust == 0: + break + + n_batches = int(np.ceil(num_robust / bs)) + + robust_lin_idcs = torch.nonzero(robust_flags, as_tuple=False) + if num_robust > 1: + robust_lin_idcs.squeeze_() + + for batch_idx in range(n_batches): + start_idx = batch_idx * bs + end_idx = min((batch_idx + 1) * bs, num_robust) + + batch_datapoint_idcs = robust_lin_idcs[start_idx:end_idx] + if len(batch_datapoint_idcs.shape) > 1: + batch_datapoint_idcs.squeeze_(-1) + x = x_orig[batch_datapoint_idcs, :].clone().to(self.device) + y = y_orig[batch_datapoint_idcs].clone().to(self.device) + + # make sure that x is a 4d tensor even if there is only a single datapoint left + if len(x.shape) == 3: + x.unsqueeze_(dim=0) + + # run attack + if attack == 'apgd-ce': + # apgd on cross-entropy loss + self.apgd.loss = 'ce' + self.apgd.seed = self.get_seed() + _, adv_curr = self.apgd.perturb(x, y, cheap=True) + + elif attack == 'apgd-dlr': + # apgd on dlr loss + self.apgd.loss = 'dlr' + self.apgd.seed = self.get_seed() + _, adv_curr = self.apgd.perturb(x, y, cheap=True) + + elif attack == 'fab': + # fab + self.fab.targeted = False + self.fab.seed = self.get_seed() + adv_curr = self.fab.perturb(x, y) + + elif attack == 'square': + # square + self.square.seed = self.get_seed() + adv_curr = self.square.perturb(x, y) + + elif attack == 'apgd-t': + # targeted apgd + self.apgd_targeted.seed = self.get_seed() + _, adv_curr = self.apgd_targeted.perturb(x, y, cheap=True) + + elif attack == 'fab-t': + # fab targeted + self.fab.targeted = True + self.fab.n_restarts = 1 + self.fab.seed = self.get_seed() + adv_curr = self.fab.perturb(x, y) + + else: + raise ValueError('Attack not supported') + + output = self.get_logits(adv_curr) + false_batch = ~y.eq(output.max(dim=1)[1]).to(robust_flags.device) + non_robust_lin_idcs = batch_datapoint_idcs[false_batch] + robust_flags[non_robust_lin_idcs] = False + + x_adv[non_robust_lin_idcs] = adv_curr[false_batch].detach().to(x_adv.device) + + if self.verbose: + num_non_robust_batch = torch.sum(false_batch) + self.logger.log('{} - {}/{} - {} out of {} successfully perturbed'.format( + attack, batch_idx + 1, n_batches, num_non_robust_batch, x.shape[0])) + + robust_accuracy = torch.sum(robust_flags).item() / x_orig.shape[0] + if self.verbose: + self.logger.log('robust accuracy after {}: {:.2%} (total time {:.1f} s)'.format( + attack.upper(), robust_accuracy, time.time() - startt)) + + # final check + if self.verbose: + if self.norm == 'Linf': + res = (x_adv - x_orig).abs().view(x_orig.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_adv - x_orig) ** 2).view(x_orig.shape[0], -1).sum(-1).sqrt() + self.logger.log('max {} perturbation: {:.5f}, nan in tensor: {}, max: {:.5f}, min: {:.5f}'.format( + self.norm, res.max(), (x_adv != x_adv).sum(), x_adv.max(), x_adv.min())) + self.logger.log('robust accuracy: {:.2%}'.format(robust_accuracy)) + + return x_adv + + def clean_accuracy(self, x_orig, y_orig, bs=250): + n_batches = x_orig.shape[0] // bs + acc = 0. + for counter in range(n_batches): + x = x_orig[counter * bs:min((counter + 1) * bs, x_orig.shape[0])].clone().to(self.device) + y = y_orig[counter * bs:min((counter + 1) * bs, x_orig.shape[0])].clone().to(self.device) + output = self.get_logits(x) + acc += (output.max(1)[1] == y).float().sum() + + if self.verbose: + print('clean accuracy: {:.2%}'.format(acc / x_orig.shape[0])) + + return acc.item() / x_orig.shape[0] + + def run_standard_evaluation_individual(self, x_orig, y_orig, bs=250): + if self.verbose: + print('using {} version including {}'.format(self.version, + ', '.join(self.attacks_to_run))) + + l_attacks = self.attacks_to_run + adv = {} + verbose_indiv = self.verbose + self.verbose = False + + for c in l_attacks: + startt = time.time() + self.attacks_to_run = [c] + adv[c] = self.run_standard_evaluation(x_orig, y_orig, bs=bs) + if verbose_indiv: + acc_indiv = self.clean_accuracy(adv[c], y_orig, bs=bs) + space = '\t \t' if c == 'fab' else '\t' + self.logger.log('robust accuracy by {} {} {:.2%} \t (time attack: {:.1f} s)'.format( + c.upper(), space, acc_indiv, time.time() - startt)) + + return adv + + def set_version(self, version='standard'): + if version == 'standard': + self.attacks_to_run = ['apgd-ce', 'apgd-t', 'fab-t', 'square'] + self.apgd.n_restarts = 1 + self.fab.n_restarts = 1 + self.apgd_targeted.n_restarts = 1 + self.fab.n_target_classes = 9 + self.apgd_targeted.n_target_classes = 9 + self.square.n_queries = 5000 + + elif version == 'plus': + self.attacks_to_run = ['apgd-ce', 'apgd-dlr', 'fab', 'square', 'apgd-t', 'fab-t'] + self.apgd.n_restarts = 5 + self.fab.n_restarts = 5 + self.apgd_targeted.n_restarts = 1 + self.fab.n_target_classes = 9 + self.apgd_targeted.n_target_classes = 9 + self.square.n_queries = 5000 + + elif version == 'rand': + self.attacks_to_run = ['apgd-ce', 'apgd-dlr'] + self.apgd.n_restarts = 1 + self.apgd.eot_iter = 20 + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/autopgd_pt.py b/evaluation/GAIRAT/GAIR_RST/autoattack/autopgd_pt.py new file mode 100644 index 0000000..8187954 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/autopgd_pt.py @@ -0,0 +1,431 @@ +import numpy as np +import time +import torch +#import scipy.io + +#import numpy.linalg as nl + +# +import os +import sys + +import torch.nn as nn +import torch.nn.functional as F + + +class APGDAttack(): + def __init__(self, model, n_iter=100, norm='Linf', n_restarts=1, eps=None, + seed=0, loss='ce', eot_iter=1, rho=.75, verbose=False, + device='cuda'): + self.model = model + self.n_iter = n_iter + self.eps = eps + self.norm = norm + self.n_restarts = n_restarts + self.seed = seed + self.loss = loss + self.eot_iter = eot_iter + self.thr_decr = rho + self.verbose = verbose + self.device = device + + def check_oscillation(self, x, j, k, y5, k3=0.75): + t = np.zeros(x.shape[1]) + for counter5 in range(k): + t += x[j - counter5] > x[j - counter5 - 1] + + return t <= k*k3*np.ones(t.shape) + + def check_shape(self, x): + return x if len(x.shape) > 0 else np.expand_dims(x, 0) + + def dlr_loss(self, x, y): + x_sorted, ind_sorted = x.sort(dim=1) + ind = (ind_sorted[:, -1] == y).float() + + return -(x[np.arange(x.shape[0]), y] - x_sorted[:, -2] * ind - x_sorted[:, -1] * (1. - ind)) / (x_sorted[:, -1] - x_sorted[:, -3] + 1e-12) + + def attack_single_run(self, x_in, y_in): + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + self.n_iter_2, self.n_iter_min, self.size_decr = max(int(0.22 * self.n_iter), 1), max(int(0.06 * self.n_iter), 1), max(int(0.03 * self.n_iter), 1) + if self.verbose: + print('parameters: ', self.n_iter, self.n_iter_2, self.n_iter_min, self.size_decr) + + if self.norm == 'Linf': + t = 2 * torch.rand(x.shape).to(self.device).detach() - 1 + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / (t.reshape([t.shape[0], -1]).abs().max(dim=1, keepdim=True)[0].reshape([-1, 1, 1, 1])) + elif self.norm == 'L2': + t = torch.randn(x.shape).to(self.device).detach() + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / ((t ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv = x_adv.clamp(0., 1.) + x_best = x_adv.clone() + x_best_adv = x_adv.clone() + loss_steps = torch.zeros([self.n_iter, x.shape[0]]) + loss_best_steps = torch.zeros([self.n_iter + 1, x.shape[0]]) + acc_steps = torch.zeros_like(loss_best_steps) + + if self.loss == 'ce': + criterion_indiv = nn.CrossEntropyLoss(reduce=False, reduction='none') + elif self.loss == 'dlr': + criterion_indiv = self.dlr_loss + else: + raise ValueError('unknowkn loss') + + x_adv.requires_grad_() + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + with torch.enable_grad(): + logits = self.model(x_adv) # 1 forward pass (eot_iter = 1) + loss_indiv = criterion_indiv(logits, y) + loss = loss_indiv.sum() + + grad += torch.autograd.grad(loss, [x_adv])[0].detach() # 1 backward pass (eot_iter = 1) + + grad /= float(self.eot_iter) + grad_best = grad.clone() + + acc = logits.detach().max(1)[1] == y + acc_steps[0] = acc + 0 + loss_best = loss_indiv.detach().clone() + + step_size = self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * torch.Tensor([2.0]).to(self.device).detach().reshape([1, 1, 1, 1]) + x_adv_old = x_adv.clone() + counter = 0 + k = self.n_iter_2 + 0 + u = np.arange(x.shape[0]) + counter3 = 0 + + loss_best_last_check = loss_best.clone() + reduced_last_check = np.zeros(loss_best.shape) == np.zeros(loss_best.shape) + n_reduced = 0 + + for i in range(self.n_iter): + ### gradient step + with torch.no_grad(): + x_adv = x_adv.detach() + grad2 = x_adv - x_adv_old + x_adv_old = x_adv.clone() + + a = 0.75 if i > 0 else 1.0 + + if self.norm == 'Linf': + x_adv_1 = x_adv + step_size * torch.sign(grad) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv_1, x - self.eps), x + self.eps), 0.0, 1.0) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a), x - self.eps), x + self.eps), 0.0, 1.0) + + elif self.norm == 'L2': + x_adv_1 = x_adv + step_size * grad / ((grad ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt()), 0.0, 1.0) + x_adv_1 = x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12), 0.0, 1.0) + + x_adv = x_adv_1 + 0. + + ### get gradient + x_adv.requires_grad_() + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + with torch.enable_grad(): + logits = self.model(x_adv) # 1 forward pass (eot_iter = 1) + loss_indiv = criterion_indiv(logits, y) + loss = loss_indiv.sum() + + grad += torch.autograd.grad(loss, [x_adv])[0].detach() # 1 backward pass (eot_iter = 1) + + + grad /= float(self.eot_iter) + + pred = logits.detach().max(1)[1] == y + acc = torch.min(acc, pred) + acc_steps[i + 1] = acc + 0 + x_best_adv[(pred == 0).nonzero().squeeze()] = x_adv[(pred == 0).nonzero().squeeze()] + 0. + if self.verbose: + print('iteration: {} - Best loss: {:.6f}'.format(i, loss_best.sum())) + + ### check step size + with torch.no_grad(): + y1 = loss_indiv.detach().clone() + loss_steps[i] = y1.cpu() + 0 + ind = (y1 > loss_best).nonzero().squeeze() + x_best[ind] = x_adv[ind].clone() + grad_best[ind] = grad[ind].clone() + loss_best[ind] = y1[ind] + 0 + loss_best_steps[i + 1] = loss_best + 0 + + counter3 += 1 + + if counter3 == k: + fl_oscillation = self.check_oscillation(loss_steps.detach().cpu().numpy(), i, k, loss_best.detach().cpu().numpy(), k3=self.thr_decr) + fl_reduce_no_impr = (~reduced_last_check) * (loss_best_last_check.cpu().numpy() >= loss_best.cpu().numpy()) + fl_oscillation = ~(~fl_oscillation * ~fl_reduce_no_impr) + reduced_last_check = np.copy(fl_oscillation) + loss_best_last_check = loss_best.clone() + + if np.sum(fl_oscillation) > 0: + step_size[u[fl_oscillation]] /= 2.0 + n_reduced = fl_oscillation.astype(float).sum() + + fl_oscillation = np.where(fl_oscillation) + + x_adv[fl_oscillation] = x_best[fl_oscillation].clone() + grad[fl_oscillation] = grad_best[fl_oscillation].clone() + + counter3 = 0 + k = np.maximum(k - self.size_decr, self.n_iter_min) + + return x_best, acc, loss_best, x_best_adv + + def perturb(self, x_in, y_in, best_loss=False, cheap=True): + assert self.norm in ['Linf', 'L2'] + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + adv = x.clone() + acc = self.model(x).max(1)[1] == y + loss = -1e10 * torch.ones_like(acc).float() + if self.verbose: + print('-------------------------- running {}-attack with epsilon {:.4f} --------------------------'.format(self.norm, self.eps)) + print('initial accuracy: {:.2%}'.format(acc.float().mean())) + startt = time.time() + + if not best_loss: + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not cheap: + raise ValueError('not implemented yet') + + else: + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + best_curr, acc_curr, loss_curr, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + ind_curr = (acc_curr == 0).nonzero().squeeze() + # + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - robust accuracy: {:.2%} - cum. time: {:.1f} s'.format( + counter, acc.float().mean(), time.time() - startt)) + + return acc, adv + + else: + adv_best = x.detach().clone() + loss_best = torch.ones([x.shape[0]]).to(self.device) * (-float('inf')) + for counter in range(self.n_restarts): + best_curr, _, loss_curr, _ = self.attack_single_run(x, y) + ind_curr = (loss_curr > loss_best).nonzero().squeeze() + adv_best[ind_curr] = best_curr[ind_curr] + 0. + loss_best[ind_curr] = loss_curr[ind_curr] + 0. + + if self.verbose: + print('restart {} - loss: {:.5f}'.format(counter, loss_best.sum())) + + return loss_best, adv_best + +class APGDAttack_targeted(): + def __init__(self, model, n_iter=100, norm='Linf', n_restarts=1, eps=None, + seed=0, eot_iter=1, rho=.75, verbose=False, device='cuda', + n_target_classes=9): + self.model = model + self.n_iter = n_iter + self.eps = eps + self.norm = norm + self.n_restarts = n_restarts + self.seed = seed + self.eot_iter = eot_iter + self.thr_decr = rho + self.verbose = verbose + self.target_class = None + self.device = device + self.n_target_classes = n_target_classes + + def check_oscillation(self, x, j, k, y5, k3=0.5): + t = np.zeros(x.shape[1]) + for counter5 in range(k): + t += x[j - counter5] > x[j - counter5 - 1] + + return t <= k*k3*np.ones(t.shape) + + def check_shape(self, x): + return x if len(x.shape) > 0 else np.expand_dims(x, 0) + + def dlr_loss_targeted(self, x, y, y_target): + x_sorted, ind_sorted = x.sort(dim=1) + + return -(x[np.arange(x.shape[0]), y] - x[np.arange(x.shape[0]), y_target]) / (x_sorted[:, -1] - .5 * x_sorted[:, -3] - .5 * x_sorted[:, -4] + 1e-12) + + def attack_single_run(self, x_in, y_in): + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + self.n_iter_2, self.n_iter_min, self.size_decr = max(int(0.22 * self.n_iter), 1), max(int(0.06 * self.n_iter), 1), max(int(0.03 * self.n_iter), 1) + if self.verbose: + print('parameters: ', self.n_iter, self.n_iter_2, self.n_iter_min, self.size_decr) + + if self.norm == 'Linf': + t = 2 * torch.rand(x.shape).to(self.device).detach() - 1 + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / (t.reshape([t.shape[0], -1]).abs().max(dim=1, keepdim=True)[0].reshape([-1, 1, 1, 1])) + elif self.norm == 'L2': + t = torch.randn(x.shape).to(self.device).detach() + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / ((t ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv = x_adv.clamp(0., 1.) + x_best = x_adv.clone() + x_best_adv = x_adv.clone() + loss_steps = torch.zeros([self.n_iter, x.shape[0]]) + loss_best_steps = torch.zeros([self.n_iter + 1, x.shape[0]]) + acc_steps = torch.zeros_like(loss_best_steps) + + output = self.model(x) + y_target = output.sort(dim=1)[1][:, -self.target_class] + + x_adv.requires_grad_() + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + with torch.enable_grad(): + logits = self.model(x_adv) # 1 forward pass (eot_iter = 1) + loss_indiv = self.dlr_loss_targeted(logits, y, y_target) + loss = loss_indiv.sum() + + grad += torch.autograd.grad(loss, [x_adv])[0].detach() # 1 backward pass (eot_iter = 1) + + grad /= float(self.eot_iter) + grad_best = grad.clone() + + acc = logits.detach().max(1)[1] == y + acc_steps[0] = acc + 0 + loss_best = loss_indiv.detach().clone() + + step_size = self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * torch.Tensor([2.0]).to(self.device).detach().reshape([1, 1, 1, 1]) + x_adv_old = x_adv.clone() + counter = 0 + k = self.n_iter_2 + 0 + u = np.arange(x.shape[0]) + counter3 = 0 + + loss_best_last_check = loss_best.clone() + reduced_last_check = np.zeros(loss_best.shape) == np.zeros(loss_best.shape) + n_reduced = 0 + + for i in range(self.n_iter): + ### gradient step + with torch.no_grad(): + x_adv = x_adv.detach() + grad2 = x_adv - x_adv_old + x_adv_old = x_adv.clone() + + a = 0.75 if i > 0 else 1.0 + + if self.norm == 'Linf': + x_adv_1 = x_adv + step_size * torch.sign(grad) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv_1, x - self.eps), x + self.eps), 0.0, 1.0) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv + (x_adv_1 - x_adv)*a + grad2*(1 - a), x - self.eps), x + self.eps), 0.0, 1.0) + + elif self.norm == 'L2': + x_adv_1 = x_adv + step_size[0] * grad / ((grad ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt()), 0.0, 1.0) + x_adv_1 = x_adv + (x_adv_1 - x_adv)*a + grad2*(1 - a) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12), 0.0, 1.0) + + x_adv = x_adv_1 + 0. + + ### get gradient + x_adv.requires_grad_() + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + with torch.enable_grad(): + logits = self.model(x_adv) # 1 forward pass (eot_iter = 1) + loss_indiv = self.dlr_loss_targeted(logits, y, y_target) + loss = loss_indiv.sum() + + grad += torch.autograd.grad(loss, [x_adv])[0].detach() # 1 backward pass (eot_iter = 1) + + grad /= float(self.eot_iter) + + pred = logits.detach().max(1)[1] == y + acc = torch.min(acc, pred) + acc_steps[i + 1] = acc + 0 + x_best_adv[(pred == 0).nonzero().squeeze()] = x_adv[(pred == 0).nonzero().squeeze()] + 0. + if self.verbose: + print('iteration: {} - Best loss: {:.6f}'.format(i, loss_best.sum())) + + ### check step size + with torch.no_grad(): + y1 = loss_indiv.detach().clone() + loss_steps[i] = y1.cpu() + 0 + ind = (y1 > loss_best).nonzero().squeeze() + x_best[ind] = x_adv[ind].clone() + grad_best[ind] = grad[ind].clone() + loss_best[ind] = y1[ind] + 0 + loss_best_steps[i + 1] = loss_best + 0 + + counter3 += 1 + + if counter3 == k: + fl_oscillation = self.check_oscillation(loss_steps.detach().cpu().numpy(), i, k, loss_best.detach().cpu().numpy(), k3=self.thr_decr) + fl_reduce_no_impr = (~reduced_last_check) * (loss_best_last_check.cpu().numpy() >= loss_best.cpu().numpy()) + fl_oscillation = ~(~fl_oscillation * ~fl_reduce_no_impr) + reduced_last_check = np.copy(fl_oscillation) + loss_best_last_check = loss_best.clone() + + if np.sum(fl_oscillation) > 0: + step_size[u[fl_oscillation]] /= 2.0 + n_reduced = fl_oscillation.astype(float).sum() + + fl_oscillation = np.where(fl_oscillation) + + x_adv[fl_oscillation] = x_best[fl_oscillation].clone() + grad[fl_oscillation] = grad_best[fl_oscillation].clone() + + counter3 = 0 + k = np.maximum(k - self.size_decr, self.n_iter_min) + + return x_best, acc, loss_best, x_best_adv + + def perturb(self, x_in, y_in, best_loss=False, cheap=True): + assert self.norm in ['Linf', 'L2'] + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + adv = x.clone() + acc = self.model(x).max(1)[1] == y + loss = -1e10 * torch.ones_like(acc).float() + if self.verbose: + print('-------------------------- running {}-attack with epsilon {:.4f} --------------------------'.format(self.norm, self.eps)) + print('initial accuracy: {:.2%}'.format(acc.float().mean())) + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not cheap: + raise ValueError('not implemented yet') + + else: + for target_class in range(2, self.n_target_classes + 2): + self.target_class = target_class + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + best_curr, acc_curr, loss_curr, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + ind_curr = (acc_curr == 0).nonzero().squeeze() + # + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, self.target_class, acc.float().mean(), self.eps, time.time() - startt)) + + return acc, adv + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/autopgd_tf.py b/evaluation/GAIRAT/GAIR_RST/autoattack/autopgd_tf.py new file mode 100644 index 0000000..728c836 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/autopgd_tf.py @@ -0,0 +1,389 @@ +import numpy as np +import time +import torch +#import scipy.io + +#import numpy.linalg as nl + +# +import os +import sys + +import torch.nn as nn +import torch.nn.functional as F + + +class APGDAttack(): + def __init__(self, model, n_iter=100, norm='Linf', n_restarts=1, eps=None, + seed=0, loss='ce', eot_iter=1, rho=.75, verbose=False, + device='cuda'): + self.model = model + self.n_iter = n_iter + self.eps = eps + self.norm = norm + self.n_restarts = n_restarts + self.seed = seed + self.loss = loss + self.eot_iter = eot_iter + self.thr_decr = rho + self.verbose = verbose + self.device = device + + def check_oscillation(self, x, j, k, y5, k3=0.75): + t = np.zeros(x.shape[1]) + for counter5 in range(k): + t += x[j - counter5] > x[j - counter5 - 1] + + return t <= k*k3*np.ones(t.shape) + + def check_shape(self, x): + return x if len(x.shape) > 0 else np.expand_dims(x, 0) + + def attack_single_run(self, x_in, y_in): + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + self.n_iter_2, self.n_iter_min, self.size_decr = max(int(0.22 * self.n_iter), 1), max(int(0.06 * self.n_iter), 1), max(int(0.03 * self.n_iter), 1) + if self.verbose: + print('parameters: ', self.n_iter, self.n_iter_2, self.n_iter_min, self.size_decr) + + if self.norm == 'Linf': + t = 2 * torch.rand(x.shape).to(self.device).detach() - 1 + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / (t.reshape([t.shape[0], -1]).abs().max(dim=1, keepdim=True)[0].reshape([-1, 1, 1, 1])) + elif self.norm == 'L2': + t = torch.randn(x.shape).to(self.device).detach() + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / ((t ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv = x_adv.clamp(0., 1.) + x_best = x_adv.clone() + x_best_adv = x_adv.clone() + loss_steps = torch.zeros([self.n_iter, x.shape[0]]) + loss_best_steps = torch.zeros([self.n_iter + 1, x.shape[0]]) + acc_steps = torch.zeros_like(loss_best_steps) + + if self.loss == 'ce': + criterion_indiv = self.model.get_logits_loss_grad_xent + elif self.loss == 'dlr': + criterion_indiv = self.model.get_logits_loss_grad_dlr + else: + raise ValueError('unknowkn loss') + + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + logits, loss_indiv, grad_curr = criterion_indiv(x_adv, y) + grad += grad_curr + + grad /= float(self.eot_iter) + grad_best = grad.clone() + + acc = logits.detach().max(1)[1] == y + acc_steps[0] = acc + 0 + loss_best = loss_indiv.detach().clone() + + step_size = self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * torch.Tensor([2.0]).to(self.device).detach().reshape([1, 1, 1, 1]) + x_adv_old = x_adv.clone() + counter = 0 + k = self.n_iter_2 + 0 + u = np.arange(x.shape[0]) + counter3 = 0 + + loss_best_last_check = loss_best.clone() + reduced_last_check = np.zeros(loss_best.shape) == np.zeros(loss_best.shape) + n_reduced = 0 + + for i in range(self.n_iter): + ### gradient step + with torch.no_grad(): + x_adv = x_adv.detach() + grad2 = x_adv - x_adv_old + x_adv_old = x_adv.clone() + + a = 0.75 if i > 0 else 1.0 + + if self.norm == 'Linf': + x_adv_1 = x_adv + step_size * torch.sign(grad) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv_1, x - self.eps), x + self.eps), 0.0, 1.0) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a), x - self.eps), x + self.eps), 0.0, 1.0) + + elif self.norm == 'L2': + x_adv_1 = x_adv + step_size * grad / ((grad ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt()), 0.0, 1.0) + x_adv_1 = x_adv + (x_adv_1 - x_adv) * a + grad2 * (1 - a) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12), 0.0, 1.0) + + x_adv = x_adv_1 + 0. + + ### get gradient + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + logits, loss_indiv, grad_curr = criterion_indiv(x_adv, y) + grad += grad_curr + + grad /= float(self.eot_iter) + + pred = logits.detach().max(1)[1] == y + acc = torch.min(acc, pred) + acc_steps[i + 1] = acc + 0 + x_best_adv[(pred == 0).nonzero().squeeze()] = x_adv[(pred == 0).nonzero().squeeze()] + 0. + if self.verbose: + print('iteration: {} - Best loss: {:.6f}'.format(i, loss_best.sum())) + + ### check step size + with torch.no_grad(): + y1 = loss_indiv.detach().clone() + loss_steps[i] = y1.cpu() + 0 + ind = (y1 > loss_best).nonzero().squeeze() + x_best[ind] = x_adv[ind].clone() + grad_best[ind] = grad[ind].clone() + loss_best[ind] = y1[ind] + 0 + loss_best_steps[i + 1] = loss_best + 0 + + counter3 += 1 + + if counter3 == k: + fl_oscillation = self.check_oscillation(loss_steps.detach().cpu().numpy(), i, k, loss_best.detach().cpu().numpy(), k3=self.thr_decr) + fl_reduce_no_impr = (~reduced_last_check) * (loss_best_last_check.cpu().numpy() >= loss_best.cpu().numpy()) + fl_oscillation = ~(~fl_oscillation * ~fl_reduce_no_impr) + reduced_last_check = np.copy(fl_oscillation) + loss_best_last_check = loss_best.clone() + + if np.sum(fl_oscillation) > 0: + step_size[u[fl_oscillation]] /= 2.0 + n_reduced = fl_oscillation.astype(float).sum() + + fl_oscillation = np.where(fl_oscillation) + + x_adv[fl_oscillation] = x_best[fl_oscillation].clone() + grad[fl_oscillation] = grad_best[fl_oscillation].clone() + + counter3 = 0 + k = np.maximum(k - self.size_decr, self.n_iter_min) + + return x_best, acc, loss_best, x_best_adv + + def perturb(self, x_in, y_in, cheap=True): + assert self.norm in ['Linf', 'L2'] + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + adv = x.clone() + acc = self.model.predict(x).max(1)[1] == y + loss = -1e10 * torch.ones_like(acc).float() + if self.verbose: + print('-------------------------- running {}-attack with epsilon {:.4f} --------------------------'.format(self.norm, self.eps)) + print('initial accuracy: {:.2%}'.format(acc.float().mean())) + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not cheap: + raise ValueError('not implemented yet') + + else: + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + best_curr, acc_curr, loss_curr, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + ind_curr = (acc_curr == 0).nonzero().squeeze() + # + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - robust accuracy: {:.2%} - cum. time: {:.1f} s'.format( + counter, acc.float().mean(), time.time() - startt)) + + return acc, adv + +class APGDAttack_targeted(): + def __init__(self, model, n_iter=100, norm='Linf', n_restarts=1, eps=None, + seed=0, eot_iter=1, rho=.75, verbose=False, device='cuda', + n_target_classes=9): + self.model = model + self.n_iter = n_iter + self.eps = eps + self.norm = norm + self.n_restarts = n_restarts + self.seed = seed + self.eot_iter = eot_iter + self.thr_decr = rho + self.verbose = verbose + self.target_class = None + self.device = device + self.n_target_classes = n_target_classes + + def check_oscillation(self, x, j, k, y5, k3=0.5): + t = np.zeros(x.shape[1]) + for counter5 in range(k): + t += x[j - counter5] > x[j - counter5 - 1] + + return t <= k*k3*np.ones(t.shape) + + def check_shape(self, x): + return x if len(x.shape) > 0 else np.expand_dims(x, 0) + + def custom_loss_targeted(self, x, y, y_target): + x_sorted, ind_sorted = x.sort(dim=1) + + return -(x[np.arange(x.shape[0]), y] - x[np.arange(x.shape[0]), y_target]) / (x_sorted[:, -1] - .5 * x_sorted[:, -3] - .5 * x_sorted[:, -4] + 1e-12) + + def attack_single_run(self, x_in, y_in): + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + self.n_iter_2, self.n_iter_min, self.size_decr = max(int(0.22 * self.n_iter), 1), max(int(0.06 * self.n_iter), 1), max(int(0.03 * self.n_iter), 1) + if self.verbose: + print('parameters: ', self.n_iter, self.n_iter_2, self.n_iter_min, self.size_decr) + + if self.norm == 'Linf': + t = 2 * torch.rand(x.shape).to(self.device).detach() - 1 + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / (t.reshape([t.shape[0], -1]).abs().max(dim=1, keepdim=True)[0].reshape([-1, 1, 1, 1])) + elif self.norm == 'L2': + t = torch.randn(x.shape).to(self.device).detach() + x_adv = x.detach() + self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * t / ((t ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv = x_adv.clamp(0., 1.) + x_best = x_adv.clone() + x_best_adv = x_adv.clone() + loss_steps = torch.zeros([self.n_iter, x.shape[0]]) + loss_best_steps = torch.zeros([self.n_iter + 1, x.shape[0]]) + acc_steps = torch.zeros_like(loss_best_steps) + + output = self.model.predict(x) + y_target = output.sort(dim=1)[1][:, -self.target_class] + + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + logits, loss_indiv, grad_curr = self.model.get_logits_loss_grad_target(x_adv, y, y_target) + grad += grad_curr + + grad /= float(self.eot_iter) + grad_best = grad.clone() + + acc = logits.detach().max(1)[1] == y + acc_steps[0] = acc + 0 + loss_best = loss_indiv.detach().clone() + + step_size = self.eps * torch.ones([x.shape[0], 1, 1, 1]).to(self.device).detach() * torch.Tensor([2.0]).to(self.device).detach().reshape([1, 1, 1, 1]) + x_adv_old = x_adv.clone() + counter = 0 + k = self.n_iter_2 + 0 + u = np.arange(x.shape[0]) + counter3 = 0 + + loss_best_last_check = loss_best.clone() + reduced_last_check = np.zeros(loss_best.shape) == np.zeros(loss_best.shape) + n_reduced = 0 + + for i in range(self.n_iter): + ### gradient step + with torch.no_grad(): + x_adv = x_adv.detach() + grad2 = x_adv - x_adv_old + x_adv_old = x_adv.clone() + + a = 0.75 if i > 0 else 1.0 + + if self.norm == 'Linf': + x_adv_1 = x_adv + step_size * torch.sign(grad) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv_1, x - self.eps), x + self.eps), 0.0, 1.0) + x_adv_1 = torch.clamp(torch.min(torch.max(x_adv + (x_adv_1 - x_adv)*a + grad2*(1 - a), x - self.eps), x + self.eps), 0.0, 1.0) + + elif self.norm == 'L2': + x_adv_1 = x_adv + step_size[0] * grad / ((grad ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt()), 0.0, 1.0) + x_adv_1 = x_adv + (x_adv_1 - x_adv)*a + grad2*(1 - a) + x_adv_1 = torch.clamp(x + (x_adv_1 - x) / (((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12) * torch.min( + self.eps * torch.ones(x.shape).to(self.device).detach(), ((x_adv_1 - x) ** 2).sum(dim=(1, 2, 3), keepdim=True).sqrt() + 1e-12), 0.0, 1.0) + + x_adv = x_adv_1 + 0. + + ### get gradient + grad = torch.zeros_like(x) + for _ in range(self.eot_iter): + logits, loss_indiv, grad_curr = self.model.get_logits_loss_grad_target(x_adv, y, y_target) + grad += grad_curr + + grad /= float(self.eot_iter) + + pred = logits.detach().max(1)[1] == y + acc = torch.min(acc, pred) + acc_steps[i + 1] = acc + 0 + x_best_adv[(pred == 0).nonzero().squeeze()] = x_adv[(pred == 0).nonzero().squeeze()] + 0. + if self.verbose: + print('iteration: {} - Best loss: {:.6f}'.format(i, loss_best.sum())) + + ### check step size + with torch.no_grad(): + y1 = loss_indiv.detach().clone() + loss_steps[i] = y1.cpu() + 0 + ind = (y1 > loss_best).nonzero().squeeze() + x_best[ind] = x_adv[ind].clone() + grad_best[ind] = grad[ind].clone() + loss_best[ind] = y1[ind] + 0 + loss_best_steps[i + 1] = loss_best + 0 + + counter3 += 1 + + if counter3 == k: + fl_oscillation = self.check_oscillation(loss_steps.detach().cpu().numpy(), i, k, loss_best.detach().cpu().numpy(), k3=self.thr_decr) + fl_reduce_no_impr = (~reduced_last_check) * (loss_best_last_check.cpu().numpy() >= loss_best.cpu().numpy()) + fl_oscillation = ~(~fl_oscillation * ~fl_reduce_no_impr) + reduced_last_check = np.copy(fl_oscillation) + loss_best_last_check = loss_best.clone() + + if np.sum(fl_oscillation) > 0: + step_size[u[fl_oscillation]] /= 2.0 + n_reduced = fl_oscillation.astype(float).sum() + + fl_oscillation = np.where(fl_oscillation) + + x_adv[fl_oscillation] = x_best[fl_oscillation].clone() + grad[fl_oscillation] = grad_best[fl_oscillation].clone() + + counter3 = 0 + k = np.maximum(k - self.size_decr, self.n_iter_min) + + return x_best, acc, loss_best, x_best_adv + + def perturb(self, x_in, y_in, best_loss=False, cheap=True): + assert self.norm in ['Linf', 'L2'] + x = x_in.clone() if len(x_in.shape) == 4 else x_in.clone().unsqueeze(0) + y = y_in.clone() if len(y_in.shape) == 1 else y_in.clone().unsqueeze(0) + + adv = x.clone() + acc = self.model.predict(x).max(1)[1] == y + loss = -1e10 * torch.ones_like(acc).float() + if self.verbose: + print('-------------------------- running {}-attack with epsilon {:.4f} --------------------------'.format(self.norm, self.eps)) + print('initial accuracy: {:.2%}'.format(acc.float().mean())) + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not cheap: + raise ValueError('not implemented yet') + + else: + for target_class in range(2, self.n_target_classes + 2): + self.target_class = target_class + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + best_curr, acc_curr, loss_curr, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + ind_curr = (acc_curr == 0).nonzero().squeeze() + # + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, self.target_class, acc.float().mean(), self.eps, time.time() - startt)) + + return acc, adv + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/examples/eval.py b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/eval.py new file mode 100644 index 0000000..d1802b1 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/eval.py @@ -0,0 +1,88 @@ +import os +import argparse +import torch +import torch.nn as nn +import torchvision.datasets as datasets +import torch.utils.data as data +import torchvision.transforms as transforms +from wideresnet import * + +import sys +sys.path.insert(0, '..') + +from resnet import * + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='~/dataset/cifar-10') + parser.add_argument('--norm', type=str, default='Linf') + parser.add_argument('--epsilon', type=float, default=8./255.) + parser.add_argument('--model', type=str, default='./model_test.pt') + parser.add_argument('--n_ex', type=int, default=10000) + parser.add_argument('--individual', action='store_true') + parser.add_argument('--save_dir', type=str, default='./results') + parser.add_argument('--batch_size', type=int, default=500) + parser.add_argument('--log_path', type=str, default='./log_file.txt') + parser.add_argument('--version', type=str, default='standard') + + args = parser.parse_args() + + # load model + + model = WideResNet().cuda() + model = nn.DataParallel(model) + ckpt = torch.load(args.model) + model.load_state_dict(ckpt['state_dict']) + + model.cuda() + model.eval() + ''' + model = ResNet18() + ckpt = torch.load(args.model) + model.load_state_dict(ckpt) + model.cuda() + model.eval() + ''' + # load data + transform_list = [transforms.ToTensor()] + transform_chain = transforms.Compose(transform_list) + item = datasets.CIFAR10(root=args.data_dir, train=False, transform=transform_chain, download=True) + test_loader = data.DataLoader(item, batch_size=1000, shuffle=False, num_workers=0) + + # create save dir + if not os.path.exists(args.save_dir): + os.makedirs(args.save_dir) + + # load attack + from autoattack import AutoAttack + adversary = AutoAttack(model, norm=args.norm, eps=args.epsilon, log_path=args.log_path, + version=args.version) + + l = [x for (x, y) in test_loader] + x_test = torch.cat(l, 0) + l = [y for (x, y) in test_loader] + y_test = torch.cat(l, 0) + + # example of custom version + if args.version == 'custom': + adversary.attacks_to_run = ['apgd-ce', 'fab'] + adversary.apgd.n_restarts = 2 + adversary.fab.n_restarts = 2 + + # run attack and save images + with torch.no_grad(): + if not args.individual: + adv_complete = adversary.run_standard_evaluation(x_test[:args.n_ex], y_test[:args.n_ex], + bs=args.batch_size) + + torch.save({'adv_complete': adv_complete}, '{}/{}_{}_1_{}_eps_{:.5f}.pth'.format( + args.save_dir, 'aa', args.version, adv_complete.shape[0], args.epsilon)) + + else: + # individual version, each attack is run on all test points + adv_complete = adversary.run_standard_evaluation_individual(x_test[:args.n_ex], + y_test[:args.n_ex], bs=args.batch_size) + + torch.save(adv_complete, '{}/{}_{}_individual_1_{}_eps_{:.5f}_plus_{}_cheap.pth'.format( + args.save_dir, 'aa', args.version, args.n_ex, args.epsilon)) + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/examples/eval_tf1.py b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/eval_tf1.py new file mode 100644 index 0000000..c2f5472 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/eval_tf1.py @@ -0,0 +1,150 @@ +#%% +from argparse import ArgumentParser + +import numpy as np +import tensorflow as tf + +import torch +import torch.nn as nn +import torchvision.datasets as datasets +import torch.utils.data as data +import torchvision.transforms as transforms + +import sys +#sys.path.insert(0,'..') + +from autoattack import AutoAttack, utils_tf +# + +#%% +class mnist_loader: + def __init__(self): + + self.n_class = 10 + self.dim_x = 28 + self.dim_y = 28 + self.dim_z = 1 + self.img_min = 0.0 + self.img_max = 1.0 + self.epsilon = 0.3 + + def download(self): + (trainX, trainY), (testX, testY) = tf.keras.datasets.mnist.load_data() + + trainX = trainX.astype(np.float32) + testX = testX.astype(np.float32) + + # ont-hot + trainY = tf.keras.utils.to_categorical(trainY, self.n_class) + testY = tf.keras.utils.to_categorical(testY , self.n_class) + + # get validation sets + training_size = 55000 + validX = trainX[training_size:,:] + validY = trainY[training_size:,:] + + trainX = trainX[:training_size,:] + trainY = trainY[:training_size,:] + + # expand dimesion + trainX = np.expand_dims(trainX, axis=3) + validX = np.expand_dims(validX, axis=3) + testX = np.expand_dims(testX , axis=3) + + return trainX, trainY, validX, validY, testX, testY + + def get_raw_data(self): + return self.download() + + def get_normalized_data(self): + trainX, trainY, validX, validY, testX, testY = self.get_raw_data() + trainX = trainX / 255.0 * (self.img_max - self.img_min) + self.img_min + validX = validX / 255.0 * (self.img_max - self.img_min) + self.img_min + testX = testX / 255.0 * (self.img_max - self.img_min) + self.img_min + trainY = trainY + validY = validY + testY = testY + return trainX, trainY, validX, validY, testX, testY + +#%% +def mnist_model(): + # declare variables + model_layers = [ tf.keras.layers.Input(shape=(28,28,1), name="model/input"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c1"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c2"), + tf.keras.layers.MaxPooling2D(pool_size=(2, 2), name="clf/p1"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c3"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c4"), + tf.keras.layers.MaxPooling2D(pool_size=(2, 2), name="clf/p2"), + tf.keras.layers.Flatten(name="clf/f1"), + tf.keras.layers.Dense(256, activation="relu", kernel_initializer='he_normal', name="clf/d1"), + tf.keras.layers.Dense(10 , activation=None , kernel_initializer='he_normal', name="clf/d2"), + tf.keras.layers.Activation('softmax', name="clf_output") + ] + + # clf_model + clf_model = tf.keras.Sequential() + for ii in model_layers: + clf_model.add(ii) + + clf_model.compile(loss='categorical_crossentropy', optimizer='Nadam', metrics=['accuracy']) + clf_model.summary() + + return clf_model + +#%% +def arg_parser(parser): + + parser.add_argument("--path" , dest ="path", type=str, default='./', help="path of tf.keras model's wieghts") + args, unknown = parser.parse_known_args() + if unknown: + msg = " ".join(unknown) + print('[Warning] Unrecognized arguments: {:s}'.format(msg) ) + + return args + +#%% +if __name__ == '__main__': + + # get arguments + parser = ArgumentParser() + args = arg_parser(parser) + + # MODEL PATH + MODEL_PATH = args.path + + # init tf/keras + tf.compat.v1.keras.backend.clear_session() + gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) + sess = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) + tf.compat.v1.keras.backend.set_session(sess) + tf.compat.v1.keras.backend.set_learning_phase(0) + + # load data + batch_size = 1000 + epsilon = mnist_loader().epsilon + _, _, _, _, testX, testY = mnist_loader().get_normalized_data() + + # convert to pytorch format + testY = np.argmax(testY, axis=1) + torch_testX = torch.from_numpy( np.transpose(testX, (0, 3, 1, 2)) ).float().cuda() + torch_testY = torch.from_numpy( testY ).float() + + # load model from saved weights + print('[INFO] MODEL_PATH: {:s}'.format(MODEL_PATH) ) + tf_model = mnist_model() + tf_model.load_weights(MODEL_PATH) + + # remove 'softmax layer' and put it into adapter + atk_model = tf.keras.models.Model(inputs=tf_model.input, outputs=tf_model.get_layer(index=-2).output) + atk_model.summary() + y_input = tf.placeholder(tf.int64, shape = [None]) + x_input = atk_model.input + logits = atk_model.output + model_adapted = utils_tf.ModelAdapter(logits, x_input, y_input, sess) + + # run attack + adversary = AutoAttack(model_adapted, norm='Linf', eps=epsilon, plus=False, is_tf_model=True) + x_adv = adversary.run_standard_evaluation(torch_testX, torch_testY, bs=batch_size) + np_x_adv = np.moveaxis(x_adv.cpu().numpy(), 1, 3) + np.save("./output/mnist_adv.npy", np_x_adv) diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/examples/eval_tf2.py b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/eval_tf2.py new file mode 100644 index 0000000..189e2ca --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/eval_tf2.py @@ -0,0 +1,145 @@ +#%% +from argparse import ArgumentParser + +import numpy as np +import tensorflow as tf + +import torch +import torch.nn as nn +import torchvision.datasets as datasets +import torch.utils.data as data +import torchvision.transforms as transforms + +import sys +sys.path.insert(0, '..') + +from autoattack import AutoAttack, utils_tf2 + + +#%% +class mnist_loader: + def __init__(self): + + self.n_class = 10 + self.dim_x = 28 + self.dim_y = 28 + self.dim_z = 1 + self.img_min = 0.0 + self.img_max = 1.0 + self.epsilon = 0.3 + + def download(self): + (trainX, trainY), (testX, testY) = tf.keras.datasets.mnist.load_data() + + trainX = trainX.astype(np.float32) + testX = testX.astype(np.float32) + + # ont-hot + trainY = tf.keras.utils.to_categorical(trainY, self.n_class) + testY = tf.keras.utils.to_categorical(testY , self.n_class) + + # get validation sets + training_size = 55000 + validX = trainX[training_size:,:] + validY = trainY[training_size:,:] + + trainX = trainX[:training_size,:] + trainY = trainY[:training_size,:] + + # expand dimesion + trainX = np.expand_dims(trainX, axis=3) + validX = np.expand_dims(validX, axis=3) + testX = np.expand_dims(testX , axis=3) + + return trainX, trainY, validX, validY, testX, testY + + def get_raw_data(self): + return self.download() + + def get_normalized_data(self): + trainX, trainY, validX, validY, testX, testY = self.get_raw_data() + trainX = trainX / 255.0 * (self.img_max - self.img_min) + self.img_min + validX = validX / 255.0 * (self.img_max - self.img_min) + self.img_min + testX = testX / 255.0 * (self.img_max - self.img_min) + self.img_min + trainY = trainY + validY = validY + testY = testY + return trainX, trainY, validX, validY, testX, testY + +#%% +def mnist_model(): + # declare variables + model_layers = [ tf.keras.layers.Input(shape=(28,28,1), name="model/input"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c1"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c2"), + tf.keras.layers.MaxPooling2D(pool_size=(2, 2), name="clf/p1"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c3"), + tf.keras.layers.Conv2D(16, (3, 3), padding="same", activation="relu", kernel_initializer='he_normal', name="clf/c4"), + tf.keras.layers.MaxPooling2D(pool_size=(2, 2), name="clf/p2"), + tf.keras.layers.Flatten(name="clf/f1"), + tf.keras.layers.Dense(256, activation="relu", kernel_initializer='he_normal', name="clf/d1"), + tf.keras.layers.Dense(10 , activation=None , kernel_initializer='he_normal', name="clf/d2"), + tf.keras.layers.Activation('softmax', name="clf_output") + ] + + # clf_model + clf_model = tf.keras.Sequential() + for ii in model_layers: + clf_model.add(ii) + + clf_model.compile(loss='categorical_crossentropy', optimizer='Nadam', metrics=['accuracy']) + clf_model.summary() + + return clf_model + +#%% +def arg_parser(parser): + + parser.add_argument("--path" , dest ="path", type=str, default='./', help="path of tf.keras model's wieghts") + args, unknown = parser.parse_known_args() + if unknown: + msg = " ".join(unknown) + print('[Warning] Unrecognized arguments: {:s}'.format(msg) ) + + return args + +#%% +if __name__ == '__main__': + + # get arguments + parser = ArgumentParser() + args = arg_parser(parser) + + # MODEL PATH + MODEL_PATH = args.path + + # init tf/keras + gpus = tf.config.list_physical_devices('GPU') + for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) + + # load data + batch_size = 1000 + epsilon = mnist_loader().epsilon + _, _, _, _, testX, testY = mnist_loader().get_normalized_data() + + # convert to pytorch format + testY = np.argmax(testY, axis=1) + torch_testX = torch.from_numpy( np.transpose(testX, (0, 3, 1, 2)) ).float().cuda() + torch_testY = torch.from_numpy( testY ).float() + + # load model from saved weights + print('[INFO] MODEL_PATH: {:s}'.format(MODEL_PATH) ) + tf_model = mnist_model() + tf_model.load_weights(MODEL_PATH) + + # remove 'softmax layer' and put it into adapter + atk_model = tf.keras.models.Model(inputs=tf_model.input, outputs=tf_model.get_layer(index=-2).output) + atk_model.summary() + model_adapted = utils_tf2.ModelAdapter(atk_model) + + # run attack + adversary = AutoAttack(model_adapted, norm='Linf', eps=epsilon, plus=False, is_tf_model=True) + x_adv = adversary.run_standard_evaluation(torch_testX, torch_testY, bs=batch_size) + np_x_adv = np.moveaxis(x_adv.cpu().numpy(), 1, 3) + np.save("./output/mnist_adv.npy", np_x_adv) diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/examples/resnet.py b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/resnet.py new file mode 100644 index 0000000..cef5876 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/resnet.py @@ -0,0 +1,115 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion * planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512 * block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def ResNet18(): + return ResNet(BasicBlock, [2, 2, 2, 2]) + + +def ResNet34(): + return ResNet(BasicBlock, [3, 4, 6, 3]) + + +def ResNet50(): + return ResNet(Bottleneck, [3, 4, 6, 3]) + + +def ResNet101(): + return ResNet(Bottleneck, [3, 4, 23, 3]) + + +def ResNet152(): + return ResNet(Bottleneck, [3, 8, 36, 3]) + + +def test(): + net = ResNet18() + y = net(torch.randn(1, 3, 32, 32)) + print(y.size()) diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/examples/run_eval.sh b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/run_eval.sh new file mode 100644 index 0000000..34a4d4b --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/run_eval.sh @@ -0,0 +1 @@ +CUDA_VISIBLE_DEVICES='0' python3 eval.py --model='../../GAIR_RST/GAIR_RST_results/checkpoint-epoch200.pt' --log_path='./GAIR_RST_AA_results.txt' --save_dir='./GAIR_RST_AA_results' & diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/examples/wideresnet.py b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/wideresnet.py new file mode 100644 index 0000000..5a2eb19 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/wideresnet.py @@ -0,0 +1,98 @@ +"""Based on code from https://github.com/yaodongyu/TRADES""" + +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class WideResNet(nn.Module): + def __init__(self, depth=28, num_classes=10, widen_factor=10, dropRate=0.0): + super(WideResNet, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 4) % 6 == 0) + n = (depth - 4) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x, return_prelogit=False): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + if return_prelogit: + return self.fc(out), out + else: + return self.fc(out) + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/examples/wrn_madry.py b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/wrn_madry.py new file mode 100644 index 0000000..08473ee --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/examples/wrn_madry.py @@ -0,0 +1,101 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class Wide_ResNet_Madry(nn.Module): + def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0): + super(Wide_ResNet_Madry, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 4) % 6 == 0) + n = (depth - 4) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + # self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + return self.fc(out) +def test(): + net = Wide_ResNet_Madry() + y = net(Variable(torch.randn(1, 3, 32, 32))) + #print(y.size()) + print(net) +# test() + +def wideresnet(): + return Wide_ResNet_Madry(depth=28, num_classes=10, widen_factor=10, dropRate=0.0) \ No newline at end of file diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/fab_pt.py b/evaluation/GAIRAT/GAIR_RST/autoattack/fab_pt.py new file mode 100644 index 0000000..90e1ccf --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/fab_pt.py @@ -0,0 +1,748 @@ +# Copyright (c) 2019-present, Francesco Croce +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import torch +from torch.autograd.gradcheck import zero_gradients +import time + + +DEFAULT_EPS_DICT_BY_NORM = {'Linf': .3, 'L2': 1., 'L1': 5.0} + + +class FABAttack(): + """ + Fast Adaptive Boundary Attack (Linf, L2, L1) + https://arxiv.org/abs/1907.02044 + + :param predict: forward pass function + :param norm: Lp-norm to minimize ('Linf', 'L2', 'L1' supported) + :param n_restarts: number of random restarts + :param n_iter: number of iterations + :param eps: epsilon for the random restarts + :param alpha_max: alpha_max + :param eta: overshooting + :param beta: backward step + """ + + def __init__( + self, + predict, + norm='Linf', + n_restarts=1, + n_iter=100, + eps=None, + alpha_max=0.1, + eta=1.05, + beta=0.9, + loss_fn=None, + verbose=False, + seed=0, + targeted=False, + device=None, + n_target_classes=9): + """ FAB-attack implementation in pytorch """ + + self.predict = predict + self.norm = norm + self.n_restarts = n_restarts + self.n_iter = n_iter + self.eps = eps if eps is not None else DEFAULT_EPS_DICT_BY_NORM[norm] + self.alpha_max = alpha_max + self.eta = eta + self.beta = beta + self.targeted = False + self.verbose = verbose + self.seed = seed + self.target_class = None + self.device = device + self.n_target_classes = n_target_classes + + def _get_predicted_label(self, x): + with torch.no_grad(): + outputs = self.predict(x) + _, y = torch.max(outputs, dim=1) + return y + + def check_shape(self, x): + return x if len(x.shape) > 0 else x.unsqueeze(0) + + def get_diff_logits_grads_batch(self, imgs, la): + im = imgs.clone().requires_grad_() + with torch.enable_grad(): + y = self.predict(im) + + g2 = torch.zeros([y.shape[-1], *imgs.size()]).to(self.device) + grad_mask = torch.zeros_like(y) + for counter in range(y.shape[-1]): + zero_gradients(im) + grad_mask[:, counter] = 1.0 + y.backward(grad_mask, retain_graph=True) + grad_mask[:, counter] = 0.0 + g2[counter] = im.grad.data + + g2 = torch.transpose(g2, 0, 1).detach() + #y2 = self.predict(imgs).detach() + y2 = y.detach() + df = y2 - y2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + dg = g2 - g2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + df[torch.arange(imgs.shape[0]), la] = 1e10 + + return df, dg + + def get_diff_logits_grads_batch_targeted(self, imgs, la, la_target): + u = torch.arange(imgs.shape[0]) + im = imgs.clone().requires_grad_() + with torch.enable_grad(): + y = self.predict(im) + diffy = -(y[u, la] - y[u, la_target]) + sumdiffy = diffy.sum() + + zero_gradients(im) + sumdiffy.backward() + graddiffy = im.grad.data + df = diffy.detach().unsqueeze(1) + dg = graddiffy.unsqueeze(1) + + return df, dg + + def projection_linf(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + ind2 = ((w * t).sum(1) - b < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + b[ind2] *= -1 + + c5 = (w < 0).float() + a = torch.ones(t.shape).to(self.device) + d = (a * c5 - t) * (w != 0).float() + a -= a * (1 - c5) + + p = torch.ones(t.shape).to(self.device) * c5 - t * (2 * c5 - 1) + indp = torch.argsort(p, dim=1) + + b = b - (w * t).sum(1) + b0 = (w * d).sum(1) + b1 = b0.clone() + + counter = 0 + indp2 = indp.unsqueeze(-1).flip(dims=(1, 2)).squeeze() + u = torch.arange(0, w.shape[0]) + ws = w[u.unsqueeze(1), indp2] + bs2 = - ws * d[u.unsqueeze(1), indp2] + + s = torch.cumsum(ws.abs(), dim=1) + sb = torch.cumsum(bs2, dim=1) + b0.unsqueeze(1) + + c = b - b1 > 0 + b2 = sb[u, -1] - s[u, -1] * p[u, indp[u, 0]] + c_l = (b - b2 > 0).nonzero().squeeze() + c2 = ((b - b1 > 0) * (b - b2 <= 0)).nonzero().squeeze() + c_l = self.check_shape(c_l) + c2 = self.check_shape(c2) + + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(w.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + indcurr = indp[c2, -counter2 - 1] + b2 = sb[c2, counter2] - s[c2, counter2] * p[c2, indcurr] + c = b[c2] - b2 > 0 + ind3 = c.nonzero().squeeze() + ind32 = (~c).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb = lb.long() + counter2 = 0 + + if c_l.nelement != 0: + lmbd_opt = (torch.max((b[c_l] - sb[c_l, -1]) / (-s[c_l, -1]), + torch.zeros(sb[c_l, -1].shape) + .to(self.device))).unsqueeze(-1) + d[c_l] = (2 * a[c_l] - 1) * lmbd_opt + + lmbd_opt = (torch.max((b[c2] - sb[c2, lb]) / (-s[c2, lb]), + torch.zeros(sb[c2, lb].shape) + .to(self.device))).unsqueeze(-1) + d[c2] = torch.min(lmbd_opt, d[c2]) * c5[c2]\ + + torch.max(-lmbd_opt, d[c2]) * (1 - c5[c2]) + + return d * (w != 0).float() + + def projection_l2(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + c = (w * t).sum(1) - b + ind2 = (c < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + c[ind2] *= -1 + + u = torch.arange(0, w.shape[0]).unsqueeze(1) + + r = torch.max(t / w, (t - 1) / w) + u2 = torch.ones(r.shape).to(self.device) + r = torch.min(r, 1e12 * u2) + r = torch.max(r, -1e12 * u2) + r[w.abs() < 1e-8] = 1e12 + r[r == -1e12] = -r[r == -1e12] + rs, indr = torch.sort(r, dim=1) + rs2 = torch.cat((rs[:, 1:], + torch.zeros(rs.shape[0], 1).to(self.device)), 1) + rs[rs == 1e12] = 0 + rs2[rs2 == 1e12] = 0 + + w3 = w ** 2 + w3s = w3[u, indr] + w5 = w3s.sum(dim=1, keepdim=True) + ws = w5 - torch.cumsum(w3s, dim=1) + d = -(r * w).clone() + d = d * (w.abs() > 1e-8).float() + s = torch.cat(((-w5.squeeze() * rs[:, 0]).unsqueeze(1), + torch.cumsum((-rs2 + rs) * ws, dim=1) - + w5 * rs[:, 0].unsqueeze(-1)), 1) + + c4 = (s[:, 0] + c < 0) + c3 = ((d * w).sum(dim=1) + c > 0) + c6 = c4.nonzero().squeeze() + c2 = ((1 - c4.float()) * (1 - c3.float())).nonzero().squeeze() + c6 = self.check_shape(c6) + c2 = self.check_shape(c2) + + counter = 0 + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(w.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + c3 = s[c2, counter2] + c[c2] > 0 + ind3 = c3.nonzero().squeeze() + ind32 = (~c3).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb = lb.long() + alpha = torch.zeros([1]) + + if c6.nelement() != 0: + alpha = c[c6] / w5[c6].squeeze(-1) + d[c6] = -alpha.unsqueeze(-1) * w[c6] + + if c2.nelement() != 0: + alpha = (s[c2, lb] + c[c2]) / ws[c2, lb] + rs[c2, lb] + if torch.sum(ws[c2, lb] == 0) > 0: + ind = (ws[c2, lb] == 0).nonzero().squeeze().long() + ind = self.check_shape(ind) + alpha[ind] = 0 + c5 = (alpha.unsqueeze(-1) > r[c2]).float() + d[c2] = d[c2] * c5 - alpha.unsqueeze(-1) * w[c2] * (1 - c5) + + return d * (w.abs() > 1e-8).float() + + def projection_l1(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + c = (w * t).sum(1) - b + ind2 = (c < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + c[ind2] *= -1 + + r = torch.max(1 / w, -1 / w) + r = torch.min(r, 1e12 * torch.ones(r.shape).to(self.device)) + rs, indr = torch.sort(r, dim=1) + _, indr_rev = torch.sort(indr) + + u = torch.arange(0, w.shape[0]).unsqueeze(1) + u2 = torch.arange(0, w.shape[1]).repeat(w.shape[0], 1) + c6 = (w < 0).float() + d = (-t + c6) * (w != 0).float() + d2 = torch.min(-w * t, w * (1 - t)) + ds = d2[u, indr] + ds2 = torch.cat((c.unsqueeze(-1), ds), 1) + s = torch.cumsum(ds2, dim=1) + + c4 = s[:, -1] < 0 + c2 = c4.nonzero().squeeze(-1) + c2 = self.check_shape(c2) + + counter = 0 + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (s.shape[1]) + nitermax = torch.ceil(torch.log2(torch.tensor(s.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + c3 = s[c2, counter2] > 0 + ind3 = c3.nonzero().squeeze() + ind32 = (~c3).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb2 = lb.long() + + if c2.nelement() != 0: + alpha = -s[c2, lb2] / w[c2, indr[c2, lb2]] + c5 = u2[c2].float() < lb.unsqueeze(-1).float() + u3 = c5[u[:c5.shape[0]], indr_rev[c2]] + d[c2] = d[c2] * u3.float().to(self.device) + d[c2, indr[c2, lb2]] = alpha + + return d * (w.abs() > 1e-8).float() + + def attack_single_run(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + #self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + #assert next(self.predict.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print('Clean accuracy: {:.2%}'.format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == 'Linf': + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.reshape([t.shape[0], -1]).abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1]*self.ndims])) * .5 + elif self.norm == 'L2': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / ((t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1]*self.ndims)) * .5 + elif self.norm == 'L1': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.abs().view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1]*self.ndims)) / 2 + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.n_iter: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch(x1, la2) + if self.norm == 'Linf': + dist1 = df.abs() / (1e-12 + + dg.abs() + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1)) + elif self.norm == 'L2': + dist1 = df.abs() / (1e-12 + (dg ** 2) + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1).sqrt()) + elif self.norm == 'L1': + dist1 = df.abs() / (1e-12 + dg.abs().reshape( + [df.shape[0], df.shape[1], -1]).max(dim=2)[0]) + else: + raise ValueError('norm not supported') + ind = dist1.min(dim=1)[1] + dg2 = dg[u1, ind] + b = (- df[u1, ind] + (dg2 * x1).view(x1.shape[0], -1) + .sum(dim=-1)) + w = dg2.reshape([bs, -1]) + + if self.norm == 'Linf': + d3 = self.projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L2': + d3 = self.projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L1': + d3 = self.projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == 'Linf': + a0 = d3.abs().max(dim=1, keepdim=True)[0]\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L2': + a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt()\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L1': + a0 = d3.abs().sum(dim=1, keepdim=True)\ + .view(-1, *[1]*self.ndims) + a0 = torch.max(a0, 1e-8 * torch.ones( + a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min(torch.max(a1 / (a1 + a2), + torch.zeros(a1.shape) + .to(self.device))[0], + self.alpha_max * torch.ones(a1.shape) + .to(self.device)) + x1 = ((x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == 'Linf': + t = (x1[ind_adv] - im2[ind_adv]).reshape( + [ind_adv.shape[0], -1]).abs().max(dim=1)[0] + elif self.norm == 'L2': + t = ((x1[ind_adv] - im2[ind_adv]) ** 2)\ + .view(ind_adv.shape[0], -1).sum(dim=-1).sqrt() + elif self.norm == 'L1': + t = (x1[ind_adv] - im2[ind_adv])\ + .abs().view(ind_adv.shape[0], -1).sum(dim=-1) + adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]).\ + float().reshape([-1, *[1]*self.ndims]) + adv[ind_adv]\ + * (t >= res2[ind_adv]).float().reshape( + [-1, *[1]*self.ndims]) + res2[ind_adv] = t * (t < res2[ind_adv]).float()\ + + res2[ind_adv] * (t >= res2[ind_adv]).float() + x1[ind_adv] = im2[ind_adv] + ( + x1[ind_adv] - im2[ind_adv]) * self.beta + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print('success rate: {:.0f}/{:.0f}' + .format(ind_succ.float().sum(), corr_classified) + + ' (on correctly classified points) in {:.1f} s' + .format(time.time() - startt)) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def attack_single_run_targeted(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + if self.device is None: + self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + #assert next(self.predict.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print('Clean accuracy: {:.2%}'.format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + output = self.predict(x) + la_target = output.sort(dim=-1)[1][:, -self.target_class] + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + la_target2 = la_target[pred].detach().clone() + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == 'Linf': + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.reshape([t.shape[0], -1]).abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1]*self.ndims])) * .5 + elif self.norm == 'L2': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / ((t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1]*self.ndims)) * .5 + elif self.norm == 'L1': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.abs().view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1]*self.ndims)) / 2 + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.n_iter: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch_targeted(x1, la2, la_target2) + if self.norm == 'Linf': + dist1 = df.abs() / (1e-12 + + dg.abs() + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1)) + elif self.norm == 'L2': + dist1 = df.abs() / (1e-12 + (dg ** 2) + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1).sqrt()) + elif self.norm == 'L1': + dist1 = df.abs() / (1e-12 + dg.abs().reshape( + [df.shape[0], df.shape[1], -1]).max(dim=2)[0]) + else: + raise ValueError('norm not supported') + ind = dist1.min(dim=1)[1] + #print(ind) + dg2 = dg[u1, ind] + b = (- df[u1, ind] + (dg2 * x1).view(x1.shape[0], -1) + .sum(dim=-1)) + w = dg2.reshape([bs, -1]) + + if self.norm == 'Linf': + d3 = self.projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L2': + d3 = self.projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L1': + d3 = self.projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == 'Linf': + a0 = d3.abs().max(dim=1, keepdim=True)[0]\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L2': + a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt()\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L1': + a0 = d3.abs().sum(dim=1, keepdim=True)\ + .view(-1, *[1]*self.ndims) + a0 = torch.max(a0, 1e-8 * torch.ones( + a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min(torch.max(a1 / (a1 + a2), + torch.zeros(a1.shape) + .to(self.device))[0], + self.alpha_max * torch.ones(a1.shape) + .to(self.device)) + x1 = ((x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == 'Linf': + t = (x1[ind_adv] - im2[ind_adv]).reshape( + [ind_adv.shape[0], -1]).abs().max(dim=1)[0] + elif self.norm == 'L2': + t = ((x1[ind_adv] - im2[ind_adv]) ** 2)\ + .view(ind_adv.shape[0], -1).sum(dim=-1).sqrt() + elif self.norm == 'L1': + t = (x1[ind_adv] - im2[ind_adv])\ + .abs().view(ind_adv.shape[0], -1).sum(dim=-1) + adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]).\ + float().reshape([-1, *[1]*self.ndims]) + adv[ind_adv]\ + * (t >= res2[ind_adv]).float().reshape( + [-1, *[1]*self.ndims]) + res2[ind_adv] = t * (t < res2[ind_adv]).float()\ + + res2[ind_adv] * (t >= res2[ind_adv]).float() + x1[ind_adv] = im2[ind_adv] + ( + x1[ind_adv] - im2[ind_adv]) * self.beta + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print('success rate: {:.0f}/{:.0f}' + .format(ind_succ.float().sum(), corr_classified) + + ' (on correctly classified points) in {:.1f} s' + .format(time.time() - startt)) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def perturb(self, x, y): + adv = x.clone() + with torch.no_grad(): + acc = self.predict(x).max(1)[1] == y + + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not self.targeted: + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + adv_curr = self.attack_single_run(x_to_fool, y_to_fool, use_rand_start=(counter > 0)) + + acc_curr = self.predict(adv_curr).max(1)[1] == y_to_fool + if self.norm == 'Linf': + res = (x_to_fool - adv_curr).abs().view(x_to_fool.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_to_fool - adv_curr) ** 2).view(x_to_fool.shape[0], -1).sum(dim=-1).sqrt() + acc_curr = torch.max(acc_curr, res > self.eps) + + ind_curr = (acc_curr == 0).nonzero().squeeze() + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + + if self.verbose: + print('restart {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, acc.float().mean(), self.eps, time.time() - startt)) + + else: + for target_class in range(2, self.n_target_classes + 2): + self.target_class = target_class + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + adv_curr = self.attack_single_run_targeted(x_to_fool, y_to_fool, use_rand_start=(counter > 0)) + + acc_curr = self.predict(adv_curr).max(1)[1] == y_to_fool + if self.norm == 'Linf': + res = (x_to_fool - adv_curr).abs().view(x_to_fool.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_to_fool - adv_curr) ** 2).view(x_to_fool.shape[0], -1).sum(dim=-1).sqrt() + acc_curr = torch.max(acc_curr, res > self.eps) + + ind_curr = (acc_curr == 0).nonzero().squeeze() + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + + if self.verbose: + print('restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, self.target_class, acc.float().mean(), self.eps, time.time() - startt)) + + return adv + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/fab_tf.py b/evaluation/GAIRAT/GAIR_RST/autoattack/fab_tf.py new file mode 100644 index 0000000..8498f21 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/fab_tf.py @@ -0,0 +1,745 @@ +# Copyright (c) 2019-present, Francesco Croce +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import torch +from torch.autograd.gradcheck import zero_gradients +import time + +import tensorflow as tf + +#from advertorch.utils import replicate_input + +#from .base import Attack +#from .base import LabelMixin + +DEFAULT_EPS_DICT_BY_NORM = {'Linf': .3, 'L2': 1., 'L1': 5.0} + + +class FABAttack(): + """ + Fast Adaptive Boundary Attack (Linf, L2, L1) + https://arxiv.org/abs/1907.02044 + + :param predict: forward pass function + :param norm: Lp-norm to minimize ('Linf', 'L2', 'L1' supported) + :param n_restarts: number of random restarts + :param n_iter: number of iterations + :param eps: epsilon for the random restarts + :param alpha_max: alpha_max + :param eta: overshooting + :param beta: backward step + """ + + def __init__( + self, + model, + norm='Linf', + n_restarts=1, + n_iter=100, + eps=None, + alpha_max=0.1, + eta=1.05, + beta=0.9, + loss_fn=None, + verbose=False, + seed=0, + targeted=False, + device=None, + n_target_classes=9): + """ FAB-attack implementation in pytorch """ + + self.model = model + self.norm = norm + self.n_restarts = n_restarts + self.n_iter = n_iter + self.eps = eps if eps is not None else DEFAULT_EPS_DICT_BY_NORM[norm] + self.alpha_max = alpha_max + self.eta = eta + self.beta = beta + self.targeted = False + self.verbose = verbose + self.seed = seed + self.target_class = None + self.device = device + self.n_target_classes = n_target_classes + + def _get_predicted_label(self, x): + with torch.no_grad(): + outputs = self.model.predict(x) + _, y = torch.max(outputs, dim=1) + return y + + def check_shape(self, x): + return x if len(x.shape) > 0 else x.unsqueeze(0) + + def get_diff_logits_grads_batch(self, imgs, la): + ### TODO: get both gradients and values with a single pass + g2 = self.model.grad_logits(imgs) + y2 = self.model.predict(imgs) + df = y2 - y2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + dg = g2 - g2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + df[torch.arange(imgs.shape[0]), la] = 1e10 + + return df, dg + + def get_diff_logits_grads_batch_targeted(self, imgs, la, la_target): + df, dg = self.model.get_grad_diff_logits_target(imgs, la, la_target) + + return df, dg + + + def projection_linf(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + ind2 = ((w * t).sum(1) - b < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + b[ind2] *= -1 + + c5 = (w < 0).float() + a = torch.ones(t.shape).to(self.device) + d = (a * c5 - t) * (w != 0).float() + a -= a * (1 - c5) + + p = torch.ones(t.shape).to(self.device) * c5 - t * (2 * c5 - 1) + indp = torch.argsort(p, dim=1) + + b = b - (w * t).sum(1) + b0 = (w * d).sum(1) + b1 = b0.clone() + + counter = 0 + indp2 = indp.unsqueeze(-1).flip(dims=(1, 2)).squeeze() + u = torch.arange(0, w.shape[0]) + ws = w[u.unsqueeze(1), indp2] + bs2 = - ws * d[u.unsqueeze(1), indp2] + + s = torch.cumsum(ws.abs(), dim=1) + sb = torch.cumsum(bs2, dim=1) + b0.unsqueeze(1) + + c = b - b1 > 0 + b2 = sb[u, -1] - s[u, -1] * p[u, indp[u, 0]] + c_l = (b - b2 > 0).nonzero().squeeze() + c2 = ((b - b1 > 0) * (b - b2 <= 0)).nonzero().squeeze() + c_l = self.check_shape(c_l) + c2 = self.check_shape(c2) + + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(w.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + indcurr = indp[c2, -counter2 - 1] + b2 = sb[c2, counter2] - s[c2, counter2] * p[c2, indcurr] + c = b[c2] - b2 > 0 + ind3 = c.nonzero().squeeze() + ind32 = (~c).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb = lb.long() + counter2 = 0 + + if c_l.nelement != 0: + lmbd_opt = (torch.max((b[c_l] - sb[c_l, -1]) / (-s[c_l, -1]), + torch.zeros(sb[c_l, -1].shape) + .to(self.device))).unsqueeze(-1) + d[c_l] = (2 * a[c_l] - 1) * lmbd_opt + + lmbd_opt = (torch.max((b[c2] - sb[c2, lb]) / (-s[c2, lb]), + torch.zeros(sb[c2, lb].shape) + .to(self.device))).unsqueeze(-1) + d[c2] = torch.min(lmbd_opt, d[c2]) * c5[c2]\ + + torch.max(-lmbd_opt, d[c2]) * (1 - c5[c2]) + + return d * (w != 0).float() + + def projection_l2(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + c = (w * t).sum(1) - b + ind2 = (c < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + c[ind2] *= -1 + + u = torch.arange(0, w.shape[0]).unsqueeze(1) + + r = torch.max(t / w, (t - 1) / w) + u2 = torch.ones(r.shape).to(self.device) + r = torch.min(r, 1e12 * u2) + r = torch.max(r, -1e12 * u2) + r[w.abs() < 1e-8] = 1e12 + r[r == -1e12] = -r[r == -1e12] + rs, indr = torch.sort(r, dim=1) + rs2 = torch.cat((rs[:, 1:], + torch.zeros(rs.shape[0], 1).to(self.device)), 1) + rs[rs == 1e12] = 0 + rs2[rs2 == 1e12] = 0 + + w3 = w ** 2 + w3s = w3[u, indr] + w5 = w3s.sum(dim=1, keepdim=True) + ws = w5 - torch.cumsum(w3s, dim=1) + d = -(r * w).clone() + d = d * (w.abs() > 1e-8).float() + s = torch.cat(((-w5.squeeze() * rs[:, 0]).unsqueeze(1), + torch.cumsum((-rs2 + rs) * ws, dim=1) - + w5 * rs[:, 0].unsqueeze(-1)), 1) + + c4 = (s[:, 0] + c < 0) + c3 = ((d * w).sum(dim=1) + c > 0) + c6 = c4.nonzero().squeeze() + c2 = ((1 - c4.float()) * (1 - c3.float())).nonzero().squeeze() + c6 = self.check_shape(c6) + c2 = self.check_shape(c2) + + counter = 0 + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(w.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + c3 = s[c2, counter2] + c[c2] > 0 + ind3 = c3.nonzero().squeeze() + ind32 = (~c3).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb = lb.long() + alpha = torch.zeros([1]) + + if c6.nelement() != 0: + alpha = c[c6] / w5[c6].squeeze(-1) + d[c6] = -alpha.unsqueeze(-1) * w[c6] + + if c2.nelement() != 0: + alpha = (s[c2, lb] + c[c2]) / ws[c2, lb] + rs[c2, lb] + if torch.sum(ws[c2, lb] == 0) > 0: + ind = (ws[c2, lb] == 0).nonzero().squeeze().long() + ind = self.check_shape(ind) + alpha[ind] = 0 + c5 = (alpha.unsqueeze(-1) > r[c2]).float() + d[c2] = d[c2] * c5 - alpha.unsqueeze(-1) * w[c2] * (1 - c5) + + return d * (w.abs() > 1e-8).float() + + def projection_l1(self, points_to_project, w_hyperplane, b_hyperplane): + t = points_to_project.clone() + w = w_hyperplane.clone() + b = b_hyperplane.clone() + + c = (w * t).sum(1) - b + ind2 = (c < 0).nonzero().squeeze() + ind2 = self.check_shape(ind2) + w[ind2] *= -1 + c[ind2] *= -1 + + r = torch.max(1 / w, -1 / w) + r = torch.min(r, 1e12 * torch.ones(r.shape).to(self.device)) + rs, indr = torch.sort(r, dim=1) + _, indr_rev = torch.sort(indr) + + u = torch.arange(0, w.shape[0]).unsqueeze(1) + u2 = torch.arange(0, w.shape[1]).repeat(w.shape[0], 1) + c6 = (w < 0).float() + d = (-t + c6) * (w != 0).float() + d2 = torch.min(-w * t, w * (1 - t)) + ds = d2[u, indr] + ds2 = torch.cat((c.unsqueeze(-1), ds), 1) + s = torch.cumsum(ds2, dim=1) + + c4 = s[:, -1] < 0 + c2 = c4.nonzero().squeeze(-1) + c2 = self.check_shape(c2) + + counter = 0 + lb = torch.zeros(c2.shape[0]) + ub = torch.ones(c2.shape[0]) * (s.shape[1]) + nitermax = torch.ceil(torch.log2(torch.tensor(s.shape[1]).float())) + counter2 = torch.zeros(lb.shape).long() + + while counter < nitermax: + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long() + c3 = s[c2, counter2] > 0 + ind3 = c3.nonzero().squeeze() + ind32 = (~c3).nonzero().squeeze() + ind3 = self.check_shape(ind3) + ind32 = self.check_shape(ind32) + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + counter += 1 + + lb2 = lb.long() + + if c2.nelement() != 0: + alpha = -s[c2, lb2] / w[c2, indr[c2, lb2]] + c5 = u2[c2].float() < lb.unsqueeze(-1).float() + u3 = c5[u[:c5.shape[0]], indr_rev[c2]] + d[c2] = d[c2] * u3.float().to(self.device) + d[c2, indr[c2, lb2]] = alpha + + return d * (w.abs() > 1e-8).float() + + def attack_single_run(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + if self.device is None: + self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + #assert next(self.predict.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print('Clean accuracy: {:.2%}'.format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == 'Linf': + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.reshape([t.shape[0], -1]).abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1]*self.ndims])) * .5 + elif self.norm == 'L2': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / ((t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1]*self.ndims)) * .5 + elif self.norm == 'L1': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.abs().view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1]*self.ndims)) / 2 + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.n_iter: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch(x1, la2) + if self.norm == 'Linf': + dist1 = df.abs() / (1e-12 + + dg.abs() + .reshape(dg.shape[0], dg.shape[1], -1) # view(...) + .sum(dim=-1)) + elif self.norm == 'L2': + dist1 = df.abs() / (1e-12 + (dg ** 2) + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1).sqrt()) + elif self.norm == 'L1': + dist1 = df.abs() / (1e-12 + dg.abs().reshape( + [df.shape[0], df.shape[1], -1]).max(dim=2)[0]) + else: + raise ValueError('norm not supported') + ind = dist1.min(dim=1)[1] + dg2 = dg[u1, ind] + b = (- df[u1, ind] + (dg2 * x1).reshape(x1.shape[0], -1) # view(...) + .sum(dim=-1)) + w = dg2.reshape([bs, -1]) + + if self.norm == 'Linf': + d3 = self.projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L2': + d3 = self.projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L1': + d3 = self.projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == 'Linf': + a0 = d3.abs().max(dim=1, keepdim=True)[0]\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L2': + a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt()\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L1': + a0 = d3.abs().sum(dim=1, keepdim=True)\ + .view(-1, *[1]*self.ndims) + a0 = torch.max(a0, 1e-8 * torch.ones( + a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min(torch.max(a1 / (a1 + a2), + torch.zeros(a1.shape) + .to(self.device))[0], + self.alpha_max * torch.ones(a1.shape) + .to(self.device)) + x1 = ((x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == 'Linf': + t = (x1[ind_adv] - im2[ind_adv]).reshape( + [ind_adv.shape[0], -1]).abs().max(dim=1)[0] + elif self.norm == 'L2': + t = ((x1[ind_adv] - im2[ind_adv]) ** 2)\ + .view(ind_adv.shape[0], -1).sum(dim=-1).sqrt() + elif self.norm == 'L1': + t = (x1[ind_adv] - im2[ind_adv])\ + .abs().view(ind_adv.shape[0], -1).sum(dim=-1) + adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]).\ + float().reshape([-1, *[1]*self.ndims]) + adv[ind_adv]\ + * (t >= res2[ind_adv]).float().reshape( + [-1, *[1]*self.ndims]) + res2[ind_adv] = t * (t < res2[ind_adv]).float()\ + + res2[ind_adv] * (t >= res2[ind_adv]).float() + x1[ind_adv] = im2[ind_adv] + ( + x1[ind_adv] - im2[ind_adv]) * self.beta + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print('success rate: {:.0f}/{:.0f}' + .format(ind_succ.float().sum(), corr_classified) + + ' (on correctly classified points) in {:.1f} s' + .format(time.time() - startt)) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def attack_single_run_targeted(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + #assert next(self.predict.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print('Clean accuracy: {:.2%}'.format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + output = self.model.predict(x) + la_target = output.sort(dim=-1)[1][:, -self.target_class] + + # set target class + #self.model.set_target_class(y, la_target) + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + la_target2 = la_target[pred].detach().clone() + + # set target class for correcty classified points + #self.model.set_target_class(la2, la_target2) + + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == 'Linf': + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.reshape([t.shape[0], -1]).abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1]*self.ndims])) * .5 + elif self.norm == 'L2': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / ((t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1]*self.ndims)) * .5 + elif self.norm == 'L1': + t = torch.randn(x1.shape).to(self.device) + x1 = im2 + (torch.min(res2, + self.eps * torch.ones(res2.shape) + .to(self.device) + ).reshape([-1, *[1]*self.ndims]) + ) * t / (t.abs().view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1]*self.ndims)) / 2 + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.n_iter: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch_targeted(x1, la2, la_target2) + if len(df.shape) == 1: + df.unsqueeze_(1) + dg.unsqueeze_(1) + if self.norm == 'Linf': + dist1 = df.abs() / (1e-12 + + dg.abs() + .reshape(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1)) + elif self.norm == 'L2': + dist1 = df.abs() / (1e-12 + (dg ** 2) + .reshape(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1).sqrt()) + elif self.norm == 'L1': + dist1 = df.abs() / (1e-12 + dg.abs().reshape( + [df.shape[0], df.shape[1], -1]).max(dim=2)[0]) + else: + raise ValueError('norm not supported') + ind = dist1.min(dim=1)[1] + #print(ind) + dg2 = dg[u1, ind] + b = (- df[u1, ind] + (dg2 * x1).reshape(x1.shape[0], -1) + .sum(dim=-1)) + w = dg2.reshape([bs, -1]) + + if self.norm == 'Linf': + d3 = self.projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L2': + d3 = self.projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + elif self.norm == 'L1': + d3 = self.projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0)) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == 'Linf': + a0 = d3.abs().max(dim=1, keepdim=True)[0]\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L2': + a0 = (d3 ** 2).sum(dim=1, keepdim=True).sqrt()\ + .view(-1, *[1]*self.ndims) + elif self.norm == 'L1': + a0 = d3.abs().sum(dim=1, keepdim=True)\ + .view(-1, *[1]*self.ndims) + a0 = torch.max(a0, 1e-8 * torch.ones( + a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min(torch.max(a1 / (a1 + a2), + torch.zeros(a1.shape) + .to(self.device))[0], + self.alpha_max * torch.ones(a1.shape) + .to(self.device)) + x1 = ((x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == 'Linf': + t = (x1[ind_adv] - im2[ind_adv]).reshape( + [ind_adv.shape[0], -1]).abs().max(dim=1)[0] + elif self.norm == 'L2': + t = ((x1[ind_adv] - im2[ind_adv]) ** 2)\ + .reshape(ind_adv.shape[0], -1).sum(dim=-1).sqrt() + elif self.norm == 'L1': + t = (x1[ind_adv] - im2[ind_adv])\ + .abs().reshape(ind_adv.shape[0], -1).sum(dim=-1) + adv[ind_adv] = x1[ind_adv] * (t < res2[ind_adv]).\ + float().reshape([-1, *[1]*self.ndims]) + adv[ind_adv]\ + * (t >= res2[ind_adv]).float().reshape( + [-1, *[1]*self.ndims]) + res2[ind_adv] = t * (t < res2[ind_adv]).float()\ + + res2[ind_adv] * (t >= res2[ind_adv]).float() + x1[ind_adv] = im2[ind_adv] + ( + x1[ind_adv] - im2[ind_adv]) * self.beta + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print('success rate: {:.0f}/{:.0f}' + .format(ind_succ.float().sum(), corr_classified) + + ' (on correctly classified points) in {:.1f} s' + .format(time.time() - startt)) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def perturb(self, x, y): + if self.device is None: + self.device = x.device + adv = x.clone() + with torch.no_grad(): + acc = self.model.predict(x).max(1)[1] == y + + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + if not self.targeted: + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + adv_curr = self.attack_single_run(x_to_fool, y_to_fool, use_rand_start=(counter > 0)) + + acc_curr = self.model.predict(adv_curr).max(1)[1] == y_to_fool + if self.norm == 'Linf': + res = (x_to_fool - adv_curr).abs().reshape(x_to_fool.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_to_fool - adv_curr) ** 2).reshape(x_to_fool.shape[0], -1).sum(dim=-1).sqrt() + acc_curr = torch.max(acc_curr, res > self.eps) + + ind_curr = (acc_curr == 0).nonzero().squeeze() + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + + if self.verbose: + print('restart {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, acc.float().mean(), self.eps, time.time() - startt)) + + else: + for target_class in range(2, self.n_target_classes + 2): + self.target_class = target_class + # + + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = x[ind_to_fool].clone(), y[ind_to_fool].clone() + adv_curr = self.attack_single_run_targeted(x_to_fool, y_to_fool, use_rand_start=(counter > 0)) + + acc_curr = self.model.predict(adv_curr).max(1)[1] == y_to_fool + if self.norm == 'Linf': + res = (x_to_fool - adv_curr).abs().reshape(x_to_fool.shape[0], -1).max(1)[0] + elif self.norm == 'L2': + res = ((x_to_fool - adv_curr) ** 2).reshape(x_to_fool.shape[0], -1).sum(dim=-1).sqrt() + acc_curr = torch.max(acc_curr, res > self.eps) + + ind_curr = (acc_curr == 0).nonzero().squeeze() + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + + if self.verbose: + print('restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s'.format( + counter, self.target_class, acc.float().mean(), self.eps, time.time() - startt)) + + return adv + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/other_utils.py b/evaluation/GAIRAT/GAIR_RST/autoattack/other_utils.py new file mode 100644 index 0000000..98bdb81 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/other_utils.py @@ -0,0 +1,13 @@ +import os + +class Logger(): + def __init__(self, log_path): + self.log_path = log_path + + def log(self, str_to_log): + print(str_to_log) + if not self.log_path is None: + with open(self.log_path, 'a') as f: + f.write(str_to_log + '\n') + f.flush() + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/square.py b/evaluation/GAIRAT/GAIR_RST/autoattack/square.py new file mode 100644 index 0000000..1f32bba --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/square.py @@ -0,0 +1,441 @@ +# Copyright (c) 2020-present, Francesco Croce +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import torch +import time +import math +import torch.nn.functional as F + + +class SquareAttack(): + """ + Square Attack + https://arxiv.org/abs/1912.00049 + + :param predict: forward pass function + :param norm: Lp-norm of the attack ('Linf', 'L2' supported) + :param n_restarts: number of random restarts + :param n_queries: max number of queries (each restart) + :param eps: bound on the norm of perturbations + :param seed: random seed for the starting point + :param p_init: parameter to control size of squares + :param loss: loss function optimized ('margin', 'ce' supported) + :param resc_schedule adapt schedule of p to n_queries + """ + + def __init__( + self, + predict, + norm='Linf', + n_queries=5000, + eps=None, + p_init=.8, + n_restarts=1, + seed=0, + verbose=False, + targeted=False, + loss='margin', + resc_schedule=True, + device=None): + """ + Square Attack implementation in PyTorch + """ + + self.predict = predict + self.norm = norm + self.n_queries = n_queries + self.eps = eps + self.p_init = p_init + self.n_restarts = n_restarts + self.seed = seed + self.verbose = verbose + self.targeted = targeted + self.loss = loss + self.rescale_schedule = resc_schedule + self.device = device + + def margin_and_loss(self, x, y): + """ + :param y: correct labels if untargeted else target labels + """ + + logits = self.predict(x) + xent = F.cross_entropy(logits, y, reduction='none') + u = torch.arange(x.shape[0]) + y_corr = logits[u, y].clone() + logits[u, y] = -float('inf') + y_others = logits.max(dim=-1)[0] + + if not self.targeted: + if self.loss == 'ce': + return y_corr - y_others, -1. * xent + elif self.loss == 'margin': + return y_corr - y_others, y_corr - y_others + else: + return y_others - y_corr, xent + + def init_hyperparam(self, x): + assert self.norm in ['Linf', 'L2'] + assert not self.eps is None + assert self.loss in ['ce', 'margin'] + + if self.device is None: + self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + if self.seed is None: + self.seed = time.time() + + def random_target_classes(self, y_pred, n_classes): + y = torch.zeros_like(y_pred) + for counter in range(y_pred.shape[0]): + l = list(range(n_classes)) + l.remove(y_pred[counter]) + t = self.random_int(0, len(l)) + y[counter] = l[t] + + return y.long().to(self.device) + + def check_shape(self, x): + return x if len(x.shape) == (self.ndims + 1) else x.unsqueeze(0) + + def random_choice(self, shape): + t = 2 * torch.rand(shape).to(self.device) - 1 + return torch.sign(t) + + def random_int(self, low=0, high=1, shape=[1]): + t = low + (high - low) * torch.rand(shape).to(self.device) + return t.long() + + def normalize(self, x): + if self.norm == 'Linf': + t = x.abs().view(x.shape[0], -1).max(1)[0] + return x / (t.view(-1, *([1] * self.ndims)) + 1e-12) + + elif self.norm == 'L2': + t = (x ** 2).view(x.shape[0], -1).sum(-1).sqrt() + return x / (t.view(-1, *([1] * self.ndims)) + 1e-12) + + def lp_norm(self, x): + if self.norm == 'L2': + t = (x ** 2).view(x.shape[0], -1).sum(-1).sqrt() + return t.view(-1, *([1] * self.ndims)) + + def eta_rectangles(self, x, y): + delta = torch.zeros([x, y]).to(self.device) + x_c, y_c = x // 2 + 1, y // 2 + 1 + + counter2 = [x_c - 1, y_c - 1] + for counter in range(0, max(x_c, y_c)): + delta[max(counter2[0], 0):min(counter2[0] + (2*counter + 1), x), + max(0, counter2[1]):min(counter2[1] + (2*counter + 1), y) + ] += 1.0/(torch.Tensor([counter + 1]).view(1, 1).to( + self.device) ** 2) + counter2[0] -= 1 + counter2[1] -= 1 + + delta /= (delta ** 2).sum(dim=(0,1), keepdim=True).sqrt() + + return delta + + def eta(self, s): + delta = torch.zeros([s, s]).to(self.device) + delta[:s // 2] = self.eta_rectangles(s // 2, s) + delta[s // 2:] = -1. * self.eta_rectangles(s - s // 2, s) + delta /= (delta ** 2).sum(dim=(0, 1), keepdim=True).sqrt() + if torch.rand([1]) > 0.5: + delta = delta.permute([1, 0]) + + return delta + + def p_selection(self, it): + """ schedule to decrease the parameter p """ + + if self.rescale_schedule: + it = int(it / self.n_queries * 10000) + + if 10 < it <= 50: + p = self.p_init / 2 + elif 50 < it <= 200: + p = self.p_init / 4 + elif 200 < it <= 500: + p = self.p_init / 8 + elif 500 < it <= 1000: + p = self.p_init / 16 + elif 1000 < it <= 2000: + p = self.p_init / 32 + elif 2000 < it <= 4000: + p = self.p_init / 64 + elif 4000 < it <= 6000: + p = self.p_init / 128 + elif 6000 < it <= 8000: + p = self.p_init / 256 + elif 8000 < it: + p = self.p_init / 512 + else: + p = self.p_init + + return p + + def attack_single_run(self, x, y): + with torch.no_grad(): + adv = x.clone() + c, h, w = x.shape[1:] + n_features = c * h * w + n_ex_total = x.shape[0] + + if self.norm == 'Linf': + x_best = torch.clamp(x + self.eps * self.random_choice( + [x.shape[0], c, 1, w]), 0., 1.) + margin_min, loss_min = self.margin_and_loss(x_best, y) + n_queries = torch.ones(x.shape[0]).to(self.device) + s_init = int(math.sqrt(self.p_init * n_features / c)) + + for i_iter in range(self.n_queries): + idx_to_fool = (margin_min > 0.0).nonzero().squeeze() + + x_curr = self.check_shape(x[idx_to_fool]) + x_best_curr = self.check_shape(x_best[idx_to_fool]) + y_curr = y[idx_to_fool] + if len(y_curr.shape) == 0: + y_curr = y_curr.unsqueeze(0) + margin_min_curr = margin_min[idx_to_fool] + loss_min_curr = loss_min[idx_to_fool] + + p = self.p_selection(i_iter) + s = max(int(round(math.sqrt(p * n_features / c))), 1) + vh = self.random_int(0, h - s) + vw = self.random_int(0, w - s) + new_deltas = torch.zeros([c, h, w]).to(self.device) + new_deltas[:, vh:vh + s, vw:vw + s + ] = 2. * self.eps * self.random_choice([c, 1, 1]) + + x_new = x_best_curr + new_deltas + x_new = torch.min(torch.max(x_new, x_curr - self.eps), + x_curr + self.eps) + x_new = torch.clamp(x_new, 0., 1.) + x_new = self.check_shape(x_new) + + margin, loss = self.margin_and_loss(x_new, y_curr) + + # update loss if new loss is better + idx_improved = (loss < loss_min_curr).float() + + loss_min[idx_to_fool] = idx_improved * loss + ( + 1. - idx_improved) * loss_min_curr + + # update margin and x_best if new loss is better + # or misclassification + idx_miscl = (margin <= 0.).float() + idx_improved = torch.max(idx_improved, idx_miscl) + + margin_min[idx_to_fool] = idx_improved * margin + ( + 1. - idx_improved) * margin_min_curr + idx_improved = idx_improved.reshape([-1, + *[1]*len(x.shape[:-1])]) + x_best[idx_to_fool] = idx_improved * x_new + ( + 1. - idx_improved) * x_best_curr + n_queries[idx_to_fool] += 1. + + ind_succ = (margin_min <= 0.).nonzero().squeeze() + if self.verbose and ind_succ.numel() != 0: + print('{}'.format(i_iter + 1), + '- success rate={}/{} ({:.2%})'.format( + ind_succ.numel(), n_ex_total, + float(ind_succ.numel()) / n_ex_total), + '- avg # queries={:.1f}'.format( + n_queries[ind_succ].mean().item()), + '- med # queries={:.1f}'.format( + n_queries[ind_succ].median().item()), + '- loss={:.3f}'.format(loss_min.mean())) + + if ind_succ.numel() == n_ex_total: + break + + elif self.norm == 'L2': + delta_init = torch.zeros_like(x) + s = h // 5 + sp_init = (h - s * 5) // 2 + vh = sp_init + 0 + for _ in range(h // s): + vw = sp_init + 0 + for _ in range(w // s): + delta_init[:, :, vh:vh + s, vw:vw + s] += self.eta( + s).view(1, 1, s, s) * self.random_choice( + [x.shape[0], c, 1, 1]) + vw += s + vh += s + + x_best = torch.clamp(x + self.normalize(delta_init + ) * self.eps, 0., 1.) + margin_min, loss_min = self.margin_and_loss(x_best, y) + n_queries = torch.ones(x.shape[0]).to(self.device) + s_init = int(math.sqrt(self.p_init * n_features / c)) + + for i_iter in range(self.n_queries): + idx_to_fool = (margin_min > 0.0).nonzero().squeeze() + + x_curr = self.check_shape(x[idx_to_fool]) + x_best_curr = self.check_shape(x_best[idx_to_fool]) + y_curr = y[idx_to_fool] + if len(y_curr.shape) == 0: + y_curr = y_curr.unsqueeze(0) + margin_min_curr = margin_min[idx_to_fool] + loss_min_curr = loss_min[idx_to_fool] + + delta_curr = x_best_curr - x_curr + p = self.p_selection(i_iter) + s = max(int(round(math.sqrt(p * n_features / c))), 3) + if s % 2 == 0: + s += 1 + + vh = self.random_int(0, h - s) + vw = self.random_int(0, w - s) + new_deltas_mask = torch.zeros_like(x_curr) + new_deltas_mask[:, :, vh:vh + s, vw:vw + s] = 1.0 + norms_window_1 = (delta_curr[:, :, vh:vh + s, vw:vw + s + ] ** 2).sum(dim=(-2, -1), keepdim=True).sqrt() + + vh2 = self.random_int(0, h - s) + vw2 = self.random_int(0, w - s) + new_deltas_mask_2 = torch.zeros_like(x_curr) + new_deltas_mask_2[:, :, vh2:vh2 + s, vw2:vw2 + s] = 1. + + norms_image = self.lp_norm(x_best_curr - x_curr) + mask_image = torch.max(new_deltas_mask, new_deltas_mask_2) + norms_windows = self.lp_norm(delta_curr * mask_image) + + new_deltas = torch.ones([x_curr.shape[0], c, s, s] + ).to(self.device) + new_deltas *= (self.eta(s).view(1, 1, s, s) * + self.random_choice([x_curr.shape[0], c, 1, 1])) + old_deltas = delta_curr[:, :, vh:vh + s, vw:vw + s] / ( + 1e-12 + norms_window_1) + new_deltas += old_deltas + new_deltas = new_deltas / (1e-12 + (new_deltas ** 2).sum( + dim=(-2, -1), keepdim=True).sqrt()) * (torch.max( + (self.eps * torch.ones_like(new_deltas)) ** 2 - + norms_image ** 2, torch.zeros_like(new_deltas)) / + c + norms_windows ** 2).sqrt() + delta_curr[:, :, vh2:vh2 + s, vw2:vw2 + s] = 0. + delta_curr[:, :, vh:vh + s, vw:vw + s] = new_deltas + 0 + + x_new = torch.clamp(x_curr + self.normalize(delta_curr + ) * self.eps, 0. ,1.) + x_new = self.check_shape(x_new) + norms_image = self.lp_norm(x_new - x_curr) + + margin, loss = self.margin_and_loss(x_new, y_curr) + + # update loss if new loss is better + idx_improved = (loss < loss_min_curr).float() + + loss_min[idx_to_fool] = idx_improved * loss + ( + 1. - idx_improved) * loss_min_curr + + # update margin and x_best if new loss is better + # or misclassification + idx_miscl = (margin <= 0.).float() + idx_improved = torch.max(idx_improved, idx_miscl) + + margin_min[idx_to_fool] = idx_improved * margin + ( + 1. - idx_improved) * margin_min_curr + idx_improved = idx_improved.reshape([-1, + *[1]*len(x.shape[:-1])]) + x_best[idx_to_fool] = idx_improved * x_new + ( + 1. - idx_improved) * x_best_curr + n_queries[idx_to_fool] += 1. + + ind_succ = (margin_min <= 0.).nonzero().squeeze() + if self.verbose and ind_succ.numel() != 0: + print('{}'.format(i_iter + 1), + '- success rate={}/{} ({:.2%})'.format( + ind_succ.numel(), n_ex_total, float( + ind_succ.numel()) / n_ex_total), + '- avg # queries={:.1f}'.format( + n_queries[ind_succ].mean().item()), + '- med # queries={:.1f}'.format( + n_queries[ind_succ].median().item()), + '- loss={:.3f}'.format(loss_min.mean())) + + assert (x_new != x_new).sum() == 0 + assert (x_best != x_best).sum() == 0 + + if ind_succ.numel() == n_ex_total: + break + + return n_queries, x_best + + def perturb(self, x, y=None): + """ + :param x: clean images + :param y: untargeted attack -> clean labels, + if None we use the predicted labels + targeted attack -> target labels, if None random classes, + different from the predicted ones, are sampled + """ + + self.init_hyperparam(x) + + adv = x.clone() + if y is None: + if not self.targeted: + with torch.no_grad(): + output = self.predict(x) + y_pred = output.max(1)[1] + y = y_pred.detach().clone().long().to(self.device) + else: + with torch.no_grad(): + output = self.predict(x) + n_classes = output.shape[-1] + y_pred = output.max(1)[1] + y = self.random_target_classes(y_pred, n_classes) + else: + y = y.detach().clone().long().to(self.device) + + if not self.targeted: + acc = self.predict(x).max(1)[1] == y + else: + acc = self.predict(x).max(1)[1] != y + + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: + ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool = x[ind_to_fool].clone() + y_to_fool = y[ind_to_fool].clone() + + _, adv_curr = self.attack_single_run(x_to_fool, y_to_fool) + + output_curr = self.predict(adv_curr) + if not self.targeted: + acc_curr = output_curr.max(1)[1] == y_to_fool + else: + acc_curr = output_curr.max(1)[1] != y_to_fool + ind_curr = (acc_curr == 0).nonzero().squeeze() + + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + if self.verbose: + print('restart {} - robust accuracy: {:.2%}'.format( + counter, acc.float().mean()), + '- cum. time: {:.1f} s'.format( + time.time() - startt)) + + return adv + diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/utils_tf.py b/evaluation/GAIRAT/GAIR_RST/autoattack/utils_tf.py new file mode 100644 index 0000000..f359068 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/utils_tf.py @@ -0,0 +1,104 @@ +import tensorflow as tf +import numpy as np +import torch + +class ModelAdapter(): + def __init__(self, logits, x, y, sess, num_classes=10): + self.logits = logits + self.sess = sess + self.x_input = x + self.y_input = y + self.num_classes = num_classes + + # gradients of logits + if num_classes <= 10: + self.grads = [None] * num_classes + for cl in range(num_classes): + self.grads[cl] = tf.gradients(self.logits[:, cl], self.x_input)[0] + + # cross-entropy loss + self.xent = tf.nn.sparse_softmax_cross_entropy_with_logits( + logits=self.logits, labels=self.y_input) + self.grad_xent = tf.gradients(self.xent, self.x_input)[0] + + # dlr loss + self.dlr = dlr_loss(self.logits, self.y_input, num_classes=self.num_classes) + self.grad_dlr = tf.gradients(self.dlr, self.x_input)[0] + + # targeted dlr loss + self.y_target = tf.placeholder(tf.int64, shape=[None]) + self.dlr_target = dlr_loss_targeted(self.logits, self.y_input, self.y_target, num_classes=self.num_classes) + self.grad_target = tf.gradients(self.dlr_target, self.x_input)[0] + + self.la = tf.placeholder(tf.int64, shape=[None]) + self.la_target = tf.placeholder(tf.int64, shape=[None]) + la_mask = tf.one_hot(self.la, self.num_classes) + la_target_mask = tf.one_hot(self.la_target, self.num_classes) + la_logit = tf.reduce_sum(la_mask * self.logits, axis=1) + la_target_logit = tf.reduce_sum(la_target_mask * self.logits, axis=1) + self.diff_logits = la_target_logit - la_logit + self.grad_diff_logits = tf.gradients(self.diff_logits, self.x_input)[0] + + def predict(self, x): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + y = self.sess.run(self.logits, {self.x_input: x2}) + + return torch.from_numpy(y).cuda() + + def grad_logits(self, x): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + g2 = self.sess.run(self.grads, {self.x_input: x2}) + g2 = np.moveaxis(np.array(g2), 0, 1) + g2 = np.transpose(g2, (0, 1, 4, 2, 3)) + + return torch.from_numpy(g2).cuda() + + def get_grad_diff_logits_target(self, x, y=None, y_target=None): + la = y.cpu().numpy() + la_target = y_target.cpu().numpy() + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + dl, g2 = self.sess.run([self.diff_logits, self.grad_diff_logits], {self.x_input: x2, self.la: la, self.la_target: la_target}) + g2 = np.transpose(np.array(g2), (0, 3, 1, 2)) + + return torch.from_numpy(dl).cuda(), torch.from_numpy(g2).cuda() + + def get_logits_loss_grad_xent(self, x, y): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + y2 = y.clone().cpu().numpy() + logits_val, loss_indiv_val, grad_val = self.sess.run([self.logits, self.xent, self.grad_xent], {self.x_input: x2, self.y_input: y2}) + grad_val = np.moveaxis(grad_val, 3, 1) + + return torch.from_numpy(logits_val).cuda(), torch.from_numpy(loss_indiv_val).cuda(), torch.from_numpy(grad_val).cuda() + + def get_logits_loss_grad_dlr(self, x, y): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + y2 = y.clone().cpu().numpy() + logits_val, loss_indiv_val, grad_val = self.sess.run([self.logits, self.dlr, self.grad_dlr], {self.x_input: x2, self.y_input: y2}) + grad_val = np.moveaxis(grad_val, 3, 1) + + return torch.from_numpy(logits_val).cuda(), torch.from_numpy(loss_indiv_val).cuda(), torch.from_numpy(grad_val).cuda() + + def get_logits_loss_grad_target(self, x, y, y_target): + x2 = np.moveaxis(x.cpu().numpy(), 1, 3) + y2 = y.clone().cpu().numpy() + y_targ = y_target.clone().cpu().numpy() + logits_val, loss_indiv_val, grad_val = self.sess.run([self.logits, self.dlr_target, self.grad_target], {self.x_input: x2, self.y_input: y2, self.y_target: y_targ}) + grad_val = np.moveaxis(grad_val, 3, 1) + + return torch.from_numpy(logits_val).cuda(), torch.from_numpy(loss_indiv_val).cuda(), torch.from_numpy(grad_val).cuda() + +def dlr_loss(x, y, num_classes=10): + x_sort = tf.contrib.framework.sort(x, axis=1) + y_onehot = tf.one_hot(y, num_classes) + ### TODO: adapt to the case when the point is already misclassified + loss = -(x_sort[:, -1] - x_sort[:, -2]) / (x_sort[:, -1] - x_sort[:, -3] + 1e-12) + + return loss + +def dlr_loss_targeted(x, y, y_target, num_classes=10): + x_sort = tf.contrib.framework.sort(x, axis=1) + y_onehot = tf.one_hot(y, num_classes) + y_target_onehot = tf.one_hot(y_target, num_classes) + loss = -(tf.reduce_sum(x * y_onehot, axis=1) - tf.reduce_sum(x * y_target_onehot, axis=1)) / (x_sort[:, -1] - .5 * x_sort[:, -3] - .5 * x_sort[:, -4] + 1e-12) + + return loss diff --git a/evaluation/GAIRAT/GAIR_RST/autoattack/utils_tf2.py b/evaluation/GAIRAT/GAIR_RST/autoattack/utils_tf2.py new file mode 100644 index 0000000..74473f8 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoattack/utils_tf2.py @@ -0,0 +1,210 @@ +import tensorflow as tf +import numpy as np +import torch + +class ModelAdapter(): + def __init__(self, model, num_classes=10): + """ + Please note that model should be tf.keras model without activation function 'softmax' + """ + self.num_classes = num_classes + self.tf_model = model + self.__check_channel_ordering() + + def __check_channel_ordering(self): + + for L in self.tf_model.layers: + if isinstance(L, tf.keras.layers.Conv2D): + print("[INFO] set data_format = '{:s}'".format(L.data_format)) + self.data_format = L.data_format + return + + print("[INFO] Can not find Conv2D layer") + input_shape = self.tf_model.input_shape + + if input_shape[3] == 3: + print("[INFO] Because detecting input_shape[3] == 3, set data_format = 'channels_last'") + self.data_format = 'channels_last' + + elif input_shape[3] == 1: + print("[INFO] Because detecting input_shape[3] == 1, set data_format = 'channels_last'") + self.data_format = 'channels_last' + + else: + print("[INFO] set data_format = 'channels_first'") + self.data_format = 'channels_first' + + def __get_logits(self, x_input): + logits = self.tf_model(x_input, training=False) + return logits + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_jacobian(self, x_input): + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x_input) + logits = self.__get_logits(x_input) + + jacobian = g.batch_jacobian(logits, x_input) + + if self.data_format == 'channels_last': + jacobian = tf.transpose(jacobian, perm=[0,1,4,2,3]) + + return jacobian + + def __get_xent(self, logits, y_input): + xent = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y_input) + return xent + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_grad_xent(self, x_input, y_input): + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x_input) + logits = self.__get_logits(x_input) + xent = self.__get_xent(logits, y_input) + + grad_xent = g.gradient(xent, x_input) + + return logits, xent, grad_xent + + def __get_dlr(self, logits, y_input): + val_dlr = dlr_loss(logits, y_input, num_classes=self.num_classes) + return val_dlr + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_grad_dlr(self, x_input, y_input): + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x_input) + logits = self.__get_logits(x_input) + val_dlr = self.__get_dlr(logits, y_input) + + grad_dlr = g.gradient(val_dlr, x_input) + + return logits, val_dlr, grad_dlr + + def __get_dlr_target(self, logits, y_input, y_target): + dlr_target = dlr_loss_targeted(logits, y_input, y_target, num_classes=self.num_classes) + return dlr_target + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_grad_dlr_target(self, x_input, y_input, y_target): + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x_input) + logits = self.__get_logits(x_input) + dlr_target = self.__get_dlr_target(logits, y_input, y_target) + + grad_target = g.gradient(dlr_target, x_input) + + return logits, dlr_target, grad_target + + @tf.function + @tf.autograph.experimental.do_not_convert + def __get_grad_diff_logits_target(self, x, la, la_target): + la_mask = tf.one_hot(la, self.num_classes) + la_target_mask = tf.one_hot(la_target, self.num_classes) + + with tf.GradientTape(watch_accessed_variables=False) as g: + g.watch(x) + logits = self.__get_logits(x) + difflogits = tf.reduce_sum((la_target_mask - la_mask) * logits, axis=1) + + g2 = g.gradient(difflogits, x) + + return difflogits, g2 + + def predict(self, x): + + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + + y = self.__get_logits(x2).numpy() + + return torch.from_numpy(y).cuda() + + def grad_logits(self, x): + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + g2 = self.__get_jacobian(x2) + + return torch.from_numpy(g2.numpy()).cuda() + + def set_target_class(self, y, y_target): + pass + + def get_grad_diff_logits_target(self, x, y, y_target): + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + la = y.cpu().numpy() + la_target = y_target.cpu().numpy() + + difflogits, g2 = self.__get_grad_diff_logits_target(x2, la, la_target) + + if self.data_format == 'channels_last': + g2 = tf.transpose(g2, perm=[0, 3, 1, 2]) + + return torch.from_numpy(difflogits.numpy()).cuda(), torch.from_numpy(g2.numpy()).cuda() + + def get_logits_loss_grad_xent(self, x, y): + + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + y2 = tf.convert_to_tensor(y.clone().cpu().numpy(), dtype=tf.int32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + + logits_val, loss_indiv_val, grad_val = self.__get_grad_xent(x2, y2) + + if self.data_format == 'channels_last': + grad_val = tf.transpose(grad_val, perm=[0,3,1,2]) + + return torch.from_numpy(logits_val.numpy()).cuda(), torch.from_numpy(loss_indiv_val.numpy()).cuda(), torch.from_numpy(grad_val.numpy()).cuda() + + def get_logits_loss_grad_dlr(self, x, y): + + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + y2 = tf.convert_to_tensor(y.clone().cpu().numpy(), dtype=tf.int32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + + logits_val, loss_indiv_val, grad_val = self.__get_grad_dlr(x2, y2) + + if self.data_format == 'channels_last': + grad_val = tf.transpose(grad_val, perm=[0,3,1,2]) + + return torch.from_numpy(logits_val.numpy()).cuda(), torch.from_numpy(loss_indiv_val.numpy()).cuda(), torch.from_numpy(grad_val.numpy()).cuda() + + def get_logits_loss_grad_target(self, x, y, y_target): + + x2 = tf.convert_to_tensor(x.cpu().numpy(), dtype=tf.float32) + y2 = tf.convert_to_tensor(y.clone().cpu().numpy(), dtype=tf.int32) + y_targ = tf.convert_to_tensor(y_target.clone().cpu().numpy(), dtype=tf.int32) + if self.data_format == 'channels_last': + x2 = tf.transpose(x2, perm=[0,2,3,1]) + + logits_val, loss_indiv_val, grad_val = self.__get_grad_dlr_target(x2, y2, y_targ) + + if self.data_format == 'channels_last': + grad_val = tf.transpose(grad_val, perm=[0,3,1,2]) + + return torch.from_numpy(logits_val.numpy()).cuda(), torch.from_numpy(loss_indiv_val.numpy()).cuda(), torch.from_numpy(grad_val.numpy()).cuda() + +def dlr_loss(x, y, num_classes=10): + x_sort = tf.sort(x, axis=1) + y_onehot = tf.one_hot(y, num_classes) + ### TODO: adapt to the case when the point is already misclassified + loss = -(x_sort[:, -1] - x_sort[:, -2]) / (x_sort[:, -1] - x_sort[:, -3] + 1e-12) + + return loss + +def dlr_loss_targeted(x, y, y_target, num_classes=10): + x_sort = tf.sort(x, axis=1) + y_onehot = tf.one_hot(y, num_classes) + y_target_onehot = tf.one_hot(y_target, num_classes) + loss = -(tf.reduce_sum(x * y_onehot, axis=1) - tf.reduce_sum(x * y_target_onehot, axis=1)) / (x_sort[:, -1] - .5 * x_sort[:, -3] - .5 * x_sort[:, -4] + 1e-12) + + return loss diff --git a/evaluation/GAIRAT/GAIR_RST/autoaugment.py b/evaluation/GAIRAT/GAIR_RST/autoaugment.py new file mode 100644 index 0000000..ff5f01a --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/autoaugment.py @@ -0,0 +1,243 @@ +""" +AutoAugment implementation from https://github.com/DeepVoltaire/AutoAugment +""" + +from PIL import Image, ImageEnhance, ImageOps +import numpy as np +import random + +# TODO: add credit + + +class ImageNetPolicy(object): + """ Randomly choose one of the best 24 Sub-policies on ImageNet. + + Example: + >>> policy = ImageNetPolicy() + >>> transformed = policy(image) + + Example as a PyTorch Transform: + >>> transform=transforms.Compose([ + >>> transforms.Resize(256), + >>> ImageNetPolicy(), + >>> transforms.ToTensor()]) + """ + def __init__(self, fillcolor=(128, 128, 128)): + self.policies = [ + SubPolicy(0.4, "posterize", 8, 0.6, "rotate", 9, fillcolor), + SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), + SubPolicy(0.8, "equalize", 8, 0.6, "equalize", 3, fillcolor), + SubPolicy(0.6, "posterize", 7, 0.6, "posterize", 6, fillcolor), + SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), + + SubPolicy(0.4, "equalize", 4, 0.8, "rotate", 8, fillcolor), + SubPolicy(0.6, "solarize", 3, 0.6, "equalize", 7, fillcolor), + SubPolicy(0.8, "posterize", 5, 1.0, "equalize", 2, fillcolor), + SubPolicy(0.2, "rotate", 3, 0.6, "solarize", 8, fillcolor), + SubPolicy(0.6, "equalize", 8, 0.4, "posterize", 6, fillcolor), + + SubPolicy(0.8, "rotate", 8, 0.4, "color", 0, fillcolor), + SubPolicy(0.4, "rotate", 9, 0.6, "equalize", 2, fillcolor), + SubPolicy(0.0, "equalize", 7, 0.8, "equalize", 8, fillcolor), + SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), + SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor), + + SubPolicy(0.8, "rotate", 8, 1.0, "color", 2, fillcolor), + SubPolicy(0.8, "color", 8, 0.8, "solarize", 7, fillcolor), + SubPolicy(0.4, "sharpness", 7, 0.6, "invert", 8, fillcolor), + SubPolicy(0.6, "shearX", 5, 1.0, "equalize", 9, fillcolor), + SubPolicy(0.4, "color", 0, 0.6, "equalize", 3, fillcolor), + + SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), + SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), + SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), + SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor) + ] + + + def __call__(self, img): + policy_idx = random.randint(0, len(self.policies) - 1) + return self.policies[policy_idx](img) + + def __repr__(self): + return "AutoAugment ImageNet Policy" + + +class CIFAR10Policy(object): + """ Randomly choose one of the best 25 Sub-policies on CIFAR10. + + Example: + >>> policy = CIFAR10Policy() + >>> transformed = policy(image) + + Example as a PyTorch Transform: + >>> transform=transforms.Compose([ + >>> transforms.Resize(256), + >>> CIFAR10Policy(), + >>> transforms.ToTensor()]) + """ + def __init__(self, fillcolor=(128, 128, 128)): + self.policies = [ + SubPolicy(0.1, "invert", 7, 0.2, "contrast", 6, fillcolor), + SubPolicy(0.7, "rotate", 2, 0.3, "translateX", 9, fillcolor), + SubPolicy(0.8, "sharpness", 1, 0.9, "sharpness", 3, fillcolor), + SubPolicy(0.5, "shearY", 8, 0.7, "translateY", 9, fillcolor), + SubPolicy(0.5, "autocontrast", 8, 0.9, "equalize", 2, fillcolor), + + SubPolicy(0.2, "shearY", 7, 0.3, "posterize", 7, fillcolor), + SubPolicy(0.4, "color", 3, 0.6, "brightness", 7, fillcolor), + SubPolicy(0.3, "sharpness", 9, 0.7, "brightness", 9, fillcolor), + SubPolicy(0.6, "equalize", 5, 0.5, "equalize", 1, fillcolor), + SubPolicy(0.6, "contrast", 7, 0.6, "sharpness", 5, fillcolor), + + SubPolicy(0.7, "color", 7, 0.5, "translateX", 8, fillcolor), + SubPolicy(0.3, "equalize", 7, 0.4, "autocontrast", 8, fillcolor), + SubPolicy(0.4, "translateY", 3, 0.2, "sharpness", 6, fillcolor), + SubPolicy(0.9, "brightness", 6, 0.2, "color", 8, fillcolor), + SubPolicy(0.5, "solarize", 2, 0.0, "invert", 3, fillcolor), + + SubPolicy(0.2, "equalize", 0, 0.6, "autocontrast", 0, fillcolor), + SubPolicy(0.2, "equalize", 8, 0.8, "equalize", 4, fillcolor), + SubPolicy(0.9, "color", 9, 0.6, "equalize", 6, fillcolor), + SubPolicy(0.8, "autocontrast", 4, 0.2, "solarize", 8, fillcolor), + SubPolicy(0.1, "brightness", 3, 0.7, "color", 0, fillcolor), + + SubPolicy(0.4, "solarize", 5, 0.9, "autocontrast", 3, fillcolor), + SubPolicy(0.9, "translateY", 9, 0.7, "translateY", 9, fillcolor), + SubPolicy(0.9, "autocontrast", 2, 0.8, "solarize", 3, fillcolor), + SubPolicy(0.8, "equalize", 8, 0.1, "invert", 3, fillcolor), + SubPolicy(0.7, "translateY", 9, 0.9, "autocontrast", 1, fillcolor) + ] + + + def __call__(self, img): + policy_idx = random.randint(0, len(self.policies) - 1) + return self.policies[policy_idx](img) + + def __repr__(self): + return "AutoAugment CIFAR10 Policy" + + +class SVHNPolicy(object): + """ Randomly choose one of the best 25 Sub-policies on SVHN. + + Example: + >>> policy = SVHNPolicy() + >>> transformed = policy(image) + + Example as a PyTorch Transform: + >>> transform=transforms.Compose([ + >>> transforms.Resize(256), + >>> SVHNPolicy(), + >>> transforms.ToTensor()]) + """ + def __init__(self, fillcolor=(128, 128, 128)): + self.policies = [ + SubPolicy(0.9, "shearX", 4, 0.2, "invert", 3, fillcolor), + SubPolicy(0.9, "shearY", 8, 0.7, "invert", 5, fillcolor), + SubPolicy(0.6, "equalize", 5, 0.6, "solarize", 6, fillcolor), + SubPolicy(0.9, "invert", 3, 0.6, "equalize", 3, fillcolor), + SubPolicy(0.6, "equalize", 1, 0.9, "rotate", 3, fillcolor), + + SubPolicy(0.9, "shearX", 4, 0.8, "autocontrast", 3, fillcolor), + SubPolicy(0.9, "shearY", 8, 0.4, "invert", 5, fillcolor), + SubPolicy(0.9, "shearY", 5, 0.2, "solarize", 6, fillcolor), + SubPolicy(0.9, "invert", 6, 0.8, "autocontrast", 1, fillcolor), + SubPolicy(0.6, "equalize", 3, 0.9, "rotate", 3, fillcolor), + + SubPolicy(0.9, "shearX", 4, 0.3, "solarize", 3, fillcolor), + SubPolicy(0.8, "shearY", 8, 0.7, "invert", 4, fillcolor), + SubPolicy(0.9, "equalize", 5, 0.6, "translateY", 6, fillcolor), + SubPolicy(0.9, "invert", 4, 0.6, "equalize", 7, fillcolor), + SubPolicy(0.3, "contrast", 3, 0.8, "rotate", 4, fillcolor), + + SubPolicy(0.8, "invert", 5, 0.0, "translateY", 2, fillcolor), + SubPolicy(0.7, "shearY", 6, 0.4, "solarize", 8, fillcolor), + SubPolicy(0.6, "invert", 4, 0.8, "rotate", 4, fillcolor), + SubPolicy(0.3, "shearY", 7, 0.9, "translateX", 3, fillcolor), + SubPolicy(0.1, "shearX", 6, 0.6, "invert", 5, fillcolor), + + SubPolicy(0.7, "solarize", 2, 0.6, "translateY", 7, fillcolor), + SubPolicy(0.8, "shearY", 4, 0.8, "invert", 8, fillcolor), + SubPolicy(0.7, "shearX", 9, 0.8, "translateY", 3, fillcolor), + SubPolicy(0.8, "shearY", 5, 0.7, "autocontrast", 3, fillcolor), + SubPolicy(0.7, "shearX", 2, 0.1, "invert", 5, fillcolor) + ] + + + def __call__(self, img): + policy_idx = random.randint(0, len(self.policies) - 1) + return self.policies[policy_idx](img) + + def __repr__(self): + return "AutoAugment SVHN Policy" + + +class SubPolicy(object): + def __init__(self, p1, operation1, magnitude_idx1, p2, operation2, magnitude_idx2, fillcolor=(128, 128, 128)): + ranges = { + "shearX": np.linspace(0, 0.3, 10), + "shearY": np.linspace(0, 0.3, 10), + "translateX": np.linspace(0, 150 / 331, 10), + "translateY": np.linspace(0, 150 / 331, 10), + "rotate": np.linspace(0, 30, 10), + "color": np.linspace(0.0, 0.9, 10), + "posterize": np.round(np.linspace(8, 4, 10), 0).astype(np.int), + "solarize": np.linspace(256, 0, 10), + "contrast": np.linspace(0.0, 0.9, 10), + "sharpness": np.linspace(0.0, 0.9, 10), + "brightness": np.linspace(0.0, 0.9, 10), + "autocontrast": [0] * 10, + "equalize": [0] * 10, + "invert": [0] * 10 + } + + # from https://stackoverflow.com/questions/5252170/specify-image-filling-color-when-rotating-in-python-with-pil-and-setting-expand + def rotate_with_fill(img, magnitude): + rot = img.convert("RGBA").rotate(magnitude) + return Image.composite(rot, Image.new("RGBA", rot.size, (128,) * 4), rot).convert(img.mode) + + func = { + "shearX": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0), + Image.BICUBIC, fillcolor=fillcolor), + "shearY": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0), + Image.BICUBIC, fillcolor=fillcolor), + "translateX": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, 0, magnitude * img.size[0] * random.choice([-1, 1]), 0, 1, 0), + fillcolor=fillcolor), + "translateY": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * img.size[1] * random.choice([-1, 1])), + fillcolor=fillcolor), + "rotate": lambda img, magnitude: rotate_with_fill(img, magnitude), + # "rotate": lambda img, magnitude: img.rotate(magnitude * random.choice([-1, 1])), + "color": lambda img, magnitude: ImageEnhance.Color(img).enhance(1 + magnitude * random.choice([-1, 1])), + "posterize": lambda img, magnitude: ImageOps.posterize(img, magnitude), + "solarize": lambda img, magnitude: ImageOps.solarize(img, magnitude), + "contrast": lambda img, magnitude: ImageEnhance.Contrast(img).enhance( + 1 + magnitude * random.choice([-1, 1])), + "sharpness": lambda img, magnitude: ImageEnhance.Sharpness(img).enhance( + 1 + magnitude * random.choice([-1, 1])), + "brightness": lambda img, magnitude: ImageEnhance.Brightness(img).enhance( + 1 + magnitude * random.choice([-1, 1])), + "autocontrast": lambda img, magnitude: ImageOps.autocontrast(img), + "equalize": lambda img, magnitude: ImageOps.equalize(img), + "invert": lambda img, magnitude: ImageOps.invert(img) + } + + # self.name = "{}_{:.2f}_and_{}_{:.2f}".format( + # operation1, ranges[operation1][magnitude_idx1], + # operation2, ranges[operation2][magnitude_idx2]) + self.p1 = p1 + self.operation1 = func[operation1] + self.magnitude1 = ranges[operation1][magnitude_idx1] + self.p2 = p2 + self.operation2 = func[operation2] + self.magnitude2 = ranges[operation2][magnitude_idx2] + + + def __call__(self, img): + if random.random() < self.p1: img = self.operation1(img, self.magnitude1) + if random.random() < self.p2: img = self.operation2(img, self.magnitude2) + return img \ No newline at end of file diff --git a/evaluation/GAIRAT/GAIR_RST/cutout.py b/evaluation/GAIRAT/GAIR_RST/cutout.py new file mode 100644 index 0000000..2606903 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/cutout.py @@ -0,0 +1,50 @@ +""" +Cutout augmentation implementation. Code taken from +https://github.com/uoguelph-mlrg/Cutout/blob/master/util/cutout.py +""" + +import torch +import numpy as np + +# TODO: add credit + + +class Cutout(object): + """Randomly mask out one or more patches from an image. + + Args: + n_holes (int): Number of patches to cut out of each image. + length (int): The length (in pixels) of each square patch. + """ + def __init__(self, n_holes, length): + self.n_holes = n_holes + self.length = length + + def __call__(self, img): + """ + Args: + img (Tensor): Tensor image of size (C, H, W). + Returns: + Tensor: Image with n_holes of dimension length x length cut out of it. + """ + h = img.size(1) + w = img.size(2) + + mask = np.ones((h, w), np.float32) + + for n in range(self.n_holes): + y = np.random.randint(h) + x = np.random.randint(w) + + y1 = np.clip(y - self.length // 2, 0, h) + y2 = np.clip(y + self.length // 2, 0, h) + x1 = np.clip(x - self.length // 2, 0, w) + x2 = np.clip(x + self.length // 2, 0, w) + + mask[y1: y2, x1: x2] = 0. + + mask = torch.from_numpy(mask) + mask = mask.expand_as(img) + img = img * mask + + return img diff --git a/evaluation/GAIRAT/GAIR_RST/datasets.py b/evaluation/GAIRAT/GAIR_RST/datasets.py new file mode 100644 index 0000000..7ca7f7a --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/datasets.py @@ -0,0 +1,212 @@ +""" +Datasets with unlabeled (or pseudo-labeled) data +""" + +from torchvision.datasets import CIFAR10, SVHN +from torch.utils.data import Sampler, Dataset +import torch +import numpy as np + +import os +import pickle + +import logging + +DATASETS = ['cifar10', 'svhn'] + + +class SemiSupervisedDataset(Dataset): + def __init__(self, + base_dataset='cifar10', + take_amount=None, + take_amount_seed=13, + add_svhn_extra=False, + aux_data_filename=None, + add_aux_labels=False, + aux_take_amount=None, + train=False, + **kwargs): + """A dataset with auxiliary pseudo-labeled data""" + + if base_dataset == 'cifar10': + self.dataset = CIFAR10(train=train, **kwargs) + elif base_dataset == 'svhn': + if train: + self.dataset = SVHN(split='train', **kwargs) + else: + self.dataset = SVHN(split='test', **kwargs) + # because torchvision is annoying + self.dataset.targets = self.dataset.labels + self.targets = list(self.targets) + + if train and add_svhn_extra: + svhn_extra = SVHN(split='extra', **kwargs) + self.data = np.concatenate([self.data, svhn_extra.data]) + self.targets.extend(svhn_extra.labels) + else: + raise ValueError('Dataset %s not supported' % base_dataset) + self.base_dataset = base_dataset + self.train = train + + if self.train: + if take_amount is not None: + rng_state = np.random.get_state() + np.random.seed(take_amount_seed) + take_inds = np.random.choice(len(self.sup_indices), + take_amount, replace=False) + np.random.set_state(rng_state) + + logger = logging.getLogger() + logger.info('Randomly taking only %d/%d examples from training' + ' set, seed=%d, indices=%s', + take_amount, len(self.sup_indices), + take_amount_seed, take_inds) + self.targets = self.targets[take_inds] + self.data = self.data[take_inds] + + self.sup_indices = list(range(len(self.targets))) + self.unsup_indices = [] + + if aux_data_filename is not None: + aux_path = os.path.join(kwargs['root'], aux_data_filename) + aux_path = os.path.expanduser(aux_path) + print("Loading data from %s" % aux_path) + with open(aux_path, 'rb') as f: + aux = pickle.load(f) + aux_data = aux['data'] + aux_targets = aux['extrapolated_targets'] + orig_len = len(self.data) + + if aux_take_amount is not None: + rng_state = np.random.get_state() + np.random.seed(take_amount_seed) + take_inds = np.random.choice(len(aux_data), + aux_take_amount, replace=False) + np.random.set_state(rng_state) + + logger = logging.getLogger() + logger.info( + 'Randomly taking only %d/%d examples from aux data' + ' set, seed=%d, indices=%s', + aux_take_amount, len(aux_data), + take_amount_seed, take_inds) + aux_data = aux_data[take_inds] + aux_targets = aux_targets[take_inds] + + self.data = np.concatenate((self.data, aux_data), axis=0) + + if not add_aux_labels: + self.targets.extend([-1] * len(aux_data)) + else: + self.targets.extend(aux_targets) + # note that we use unsup indices to track the labeled datapoints + # whose labels are "fake" + self.unsup_indices.extend( + range(orig_len, orig_len+len(aux_data))) + + logger = logging.getLogger() + logger.info("Training set") + logger.info("Number of training samples: %d", len(self.targets)) + logger.info("Number of supervised samples: %d", + len(self.sup_indices)) + logger.info("Number of unsup samples: %d", len(self.unsup_indices)) + logger.info("Label (and pseudo-label) histogram: %s", + tuple( + zip(*np.unique(self.targets, return_counts=True)))) + logger.info("Shape of training data: %s", np.shape(self.data)) + + # Test set + else: + self.sup_indices = list(range(len(self.targets))) + self.unsup_indices = [] + + logger = logging.getLogger() + logger.info("Test set") + logger.info("Number of samples: %d", len(self.targets)) + logger.info("Label histogram: %s", + tuple( + zip(*np.unique(self.targets, return_counts=True)))) + logger.info("Shape of data: %s", np.shape(self.data)) + + @property + def data(self): + return self.dataset.data + + @data.setter + def data(self, value): + self.dataset.data = value + + @property + def targets(self): + return self.dataset.targets + + @targets.setter + def targets(self, value): + self.dataset.targets = value + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, item): + self.dataset.labels = self.targets # because torchvision is annoying + return self.dataset[item] + + def __repr__(self): + fmt_str = 'Semisupervised Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Training: {}\n'.format(self.train) + fmt_str += ' Root Location: {}\n'.format(self.dataset.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.dataset.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.dataset.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + +class SemiSupervisedSampler(Sampler): + """Balanced sampling from the labeled and unlabeled data""" + def __init__(self, sup_inds, unsup_inds, batch_size, unsup_fraction=0.5, + num_batches=None): + if unsup_fraction is None or unsup_fraction < 0: + self.sup_inds = sup_inds + unsup_inds + unsup_fraction = 0.0 + else: + self.sup_inds = sup_inds + self.unsup_inds = unsup_inds + + self.batch_size = batch_size + unsup_batch_size = int(batch_size * unsup_fraction) + self.sup_batch_size = batch_size - unsup_batch_size + + if num_batches is not None: + self.num_batches = num_batches + else: + self.num_batches = int( + np.ceil(len(self.sup_inds) / self.sup_batch_size)) + + super().__init__(None) + + def __iter__(self): + batch_counter = 0 + while batch_counter < self.num_batches: + sup_inds_shuffled = [self.sup_inds[i] + for i in torch.randperm(len(self.sup_inds))] + for sup_k in range(0, len(self.sup_inds), self.sup_batch_size): + if batch_counter == self.num_batches: + break + batch = sup_inds_shuffled[sup_k:(sup_k + self.sup_batch_size)] + if self.sup_batch_size < self.batch_size: + batch.extend([self.unsup_inds[i] for i in + torch.randint(high=len(self.unsup_inds), + size=( + self.batch_size - len( + batch),), + dtype=torch.int64)]) + # this shuffle operation is very important, without it + # batch-norm / DataParallel hell ensues + np.random.shuffle(batch) + yield batch + batch_counter += 1 + + def __len__(self): + return self.num_batches diff --git a/evaluation/GAIRAT/GAIR_RST/losses.py b/evaluation/GAIRAT/GAIR_RST/losses.py new file mode 100644 index 0000000..f1cbb8e --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/losses.py @@ -0,0 +1,262 @@ +""" +Robust training losses. Based on code from +https://github.com/yaodongyu/TRADES +""" + +import contextlib +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +import numpy as np + +import pdb + + +def entropy_loss(unlabeled_logits): + unlabeled_probs = F.softmax(unlabeled_logits, dim=1) + return -(unlabeled_probs * F.log_softmax(unlabeled_logits, dim=1)).sum( + dim=1).mean(dim=0) + +def cwloss(output, target,confidence=50, num_classes=10): + # Compute the probability of the label class versus the maximum other + # The same implementation as in repo CAT https://github.com/sunblaze-ucb/curriculum-adversarial-training-CAT + target = target.data + target_onehot = torch.zeros(target.size() + (num_classes,)) + target_onehot = target_onehot.cuda() + target_onehot.scatter_(1, target.unsqueeze(1), 1.) + target_var = Variable(target_onehot, requires_grad=False) + real = (target_var * output).sum(1) + other = ((1. - target_var) * output - target_var * 10000.).max(1)[0] + loss = -torch.clamp(real - other + confidence, min=0.) # equiv to max(..., 0.) + loss = torch.sum(loss) + return loss + +def pgd_generate(model, data, target, epsilon, step_size, num_steps,loss_fn,category,rand_init): + model.eval() + pgd_steps = torch.zeros(len(data)) + if category == "trades": + x_adv = data.detach() + 0.001 * torch.randn(data.shape).cuda().detach() if rand_init else data.detach() + nat_output = model(data) + if category == "Madry": + x_adv = data.detach() + torch.from_numpy(np.random.uniform(-epsilon, epsilon, data.shape)).float().cuda() if rand_init else data.detach() + x_adv = torch.clamp(x_adv, 0.0, 1.0) + for k in range(num_steps): + x_adv.requires_grad_() + output = model(x_adv) + #nat_output = model(data) + predict = output.max(1, keepdim=True)[1] + for p in range(len(x_adv)): + if predict[p] == target[p]: + pgd_steps[p] += 1 + model.zero_grad() + with torch.enable_grad(): + if loss_fn == "cent": + loss_adv = nn.CrossEntropyLoss(reduction="mean")(output, target) + if loss_fn == "cw": + loss_adv = cwloss(output,target) + if loss_fn == "kl": + criterion_kl = nn.KLDivLoss(size_average=False).cuda() + loss_adv = criterion_kl(F.log_softmax(output, dim=1),F.softmax(nat_output, dim=1)) + loss_adv.backward() + eta = step_size * x_adv.grad.sign() + x_adv = x_adv.detach() + eta + x_adv = torch.min(torch.max(x_adv, data - epsilon), data + epsilon) + x_adv = torch.clamp(x_adv, 0.0, 1.0) + x_adv = Variable(x_adv, requires_grad=False) + return x_adv, pgd_steps + + +def trades_loss(model, + x_natural, + y, + optimizer, + step_size=0.003, + epsilon=0.031, + perturb_steps=10, + beta=1.0, + adversarial=True, + distance='inf', + entropy_weight=0): + """The TRADES KL-robustness regularization term proposed by + Zhang et al., with added support for stability training and entropy + regularization""" + if beta == 0: + logits = model(x_natural) + loss = F.cross_entropy(logits, y) + inf = torch.Tensor([np.inf]) + zero = torch.Tensor([0.]) + return loss, loss, inf, zero + + # define KL-loss + criterion_kl = nn.KLDivLoss(reduction='sum') + model.eval() # moving to eval mode to freeze batchnorm stats + batch_size = len(x_natural) + # generate adversarial example + x_adv = x_natural.detach() + 0. # the + 0. is for copying the tensor + if adversarial: + if distance == 'l_inf': + x_adv += 0.001 * torch.randn(x_natural.shape).cuda().detach() + + for _ in range(perturb_steps): + x_adv.requires_grad_() + with torch.enable_grad(): + loss_kl = criterion_kl(F.log_softmax(model(x_adv), dim=1), + F.softmax(model(x_natural), dim=1)) + grad = torch.autograd.grad(loss_kl, [x_adv])[0] + x_adv = x_adv.detach() + step_size * torch.sign(grad.detach()) + x_adv = torch.min(torch.max(x_adv, x_natural - epsilon), + x_natural + epsilon) + x_adv = torch.clamp(x_adv, 0.0, 1.0) + else: + raise ValueError('No support for distance %s in adversarial ' + 'training' % distance) + else: + if distance == 'l_2': + x_adv = x_adv + epsilon * torch.randn_like(x_adv) + else: + raise ValueError('No support for distance %s in stability ' + 'training' % distance) + + model.train() # moving to train mode to update batchnorm stats + + # zero gradient + optimizer.zero_grad() + + x_adv = Variable(torch.clamp(x_adv, 0.0, 1.0), requires_grad=False) + logits_adv = F.log_softmax(model(x_adv), dim=1) + logits = model(x_natural) + + loss_natural = F.cross_entropy(logits, y, ignore_index=-1) + p_natural = F.softmax(logits, dim=1) + loss_robust = criterion_kl( + logits_adv, p_natural) / batch_size + + loss = loss_natural + beta * loss_robust + + is_unlabeled = (y == -1) + if torch.sum(is_unlabeled) > 0: + logits_unlabeled = logits[is_unlabeled] + loss_entropy_unlabeled = entropy_loss(logits_unlabeled) + loss = loss + entropy_weight * loss_entropy_unlabeled + else: + loss_entropy_unlabeled = torch.tensor(0) + + return loss, loss_natural, loss_robust, loss_entropy_unlabeled + +def GAIR_trades_loss(model, + x_natural, + y, + optimizer, + step_size=0.003, + epsilon=0.031, + perturb_steps=10, + beta=1.0, + adversarial=True, + distance='inf', + entropy_weight=0, + Lambda=-1.0): + """The TRADES KL-robustness regularization term proposed by + Zhang et al., with added support for stability training and entropy + regularization""" + Kappa = torch.zeros(len(x_natural)) + + if beta == 0: + logits = model(x_natural) + loss = F.cross_entropy(logits, y) + inf = torch.Tensor([np.inf]) + zero = torch.Tensor([0.]) + return loss, loss, inf, zero + + # define KL-loss + criterion_kl = nn.KLDivLoss(reduction="sum") + model.eval() # moving to eval mode to freeze batchnorm stats + batch_size = len(x_natural) + # generate adversarial example + x_adv = x_natural.detach() + 0. # the + 0. is for copying the tensor + if adversarial: + if distance == 'l_inf': + x_adv += 0.001 * torch.randn(x_natural.shape).cuda().detach() + + for _ in range(perturb_steps): + x_adv.requires_grad_() + mid_logit = model(x_adv) + mid_predict = mid_logit.max(1, keepdim=True)[1] + # Update Kappa + for p in range(len(x_adv)): + if mid_predict[p] == y[p]: + Kappa[p] += 1 + with torch.enable_grad(): + loss_kl = criterion_kl(F.log_softmax(model(x_adv), dim=1), + F.softmax(model(x_natural), dim=1)) + grad = torch.autograd.grad(loss_kl, [x_adv])[0] + x_adv = x_adv.detach() + step_size * torch.sign(grad.detach()) + x_adv = torch.min(torch.max(x_adv, x_natural - epsilon), + x_natural + epsilon) + x_adv = torch.clamp(x_adv, 0.0, 1.0) + else: + raise ValueError('No support for distance %s in adversarial ' + 'training' % distance) + else: + if distance == 'l_2': + x_adv = x_adv + epsilon * torch.randn_like(x_adv) + else: + raise ValueError('No support for distance %s in stability ' + 'training' % distance) + + if Lambda <= 10.0: + cw_adv, _ = pgd_generate(model,x_natural, y, epsilon, step_size, perturb_steps, loss_fn="cw",category="trades", rand_init=True) + + + Kappa = Kappa.cuda() + model.train() # moving to train mode to update batchnorm stats + + # zero gradient + optimizer.zero_grad() + + x_adv = Variable(torch.clamp(x_adv, 0.0, 1.0), requires_grad=False) + logits_adv = F.log_softmax(model(x_adv), dim=1) + if Lambda <= 10.0: + logits_cw = F.log_softmax(model(cw_adv),dim=1) + logits = model(x_natural) + + loss_natural = F.cross_entropy(logits, y, ignore_index=-1, reduction='none') + + loss_natural = loss_natural.mean() + p_natural = F.softmax(logits, dim=1) + + if Lambda <= 10.0: + criterion_kl = nn.KLDivLoss(reduce=False).cuda() + loss_robust = (criterion_kl(logits_adv, p_natural).sum(dim=1).mul(((Lambda+((perturb_steps/2)-Kappa)).tanh()+1)/2)+criterion_kl(logits_cw, p_natural).sum(dim=1).mul(1-((Lambda+((perturb_steps/2)-Kappa)).tanh()+1)/2)).sum() + loss_robust = loss_robust / batch_size + else: + loss_robust = criterion_kl( + logits_adv, p_natural) / batch_size + + loss = loss_natural + beta * loss_robust + + is_unlabeled = (y == -1) + if torch.sum(is_unlabeled) > 0: + logits_unlabeled = logits[is_unlabeled] + loss_entropy_unlabeled = entropy_loss(logits_unlabeled) + loss = loss + entropy_weight * loss_entropy_unlabeled + else: + loss_entropy_unlabeled = torch.tensor(0) + + return loss, loss_natural, loss_robust, loss_entropy_unlabeled + +def noise_loss(model, + x_natural, + y, + epsilon=0.25, + clamp_x=True): + """Augmenting the input with random noise as in Cohen et al.""" + # logits_natural = model(x_natural) + x_noise = x_natural + epsilon * torch.randn_like(x_natural) + if clamp_x: + x_noise = x_noise.clamp(0.0, 1.0) + logits_noise = model(x_noise) + loss = F.cross_entropy(logits_noise, y, ignore_index=-1) + return loss + diff --git a/evaluation/GAIRAT/GAIR_RST/models/__init__.py b/evaluation/GAIRAT/GAIR_RST/models/__init__.py new file mode 100644 index 0000000..289333a --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/models/__init__.py @@ -0,0 +1,8 @@ +from . import cifar_resnet, wideresnet, shake_shake, resnet + +__all__ = [ + 'cifar_resnet', + 'wideresnet', + 'shake_shake', + 'resnet', +] \ No newline at end of file diff --git a/evaluation/GAIRAT/GAIR_RST/models/cifar_resnet.py b/evaluation/GAIRAT/GAIR_RST/models/cifar_resnet.py new file mode 100644 index 0000000..e3de21d --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/models/cifar_resnet.py @@ -0,0 +1,163 @@ +from __future__ import absolute_import + +''' +This file is from: https://raw.githubusercontent.com/bearpaw/pytorch-classification/master/models/cifar/resnet.py +by Wei Yang +''' +import torch.nn as nn +import math + + +# __all__ = ['resnet'] + +def conv3x3(in_planes, out_planes, stride=1): + "3x3 convolution with padding" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * 4) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, depth, num_classes=1000, block_name='BasicBlock'): + super(ResNet, self).__init__() + # Model type specifies number of layers for CIFAR-10 model + if block_name.lower() == 'basicblock': + assert (depth - 2) % 6 == 0, 'When use basicblock, depth should be 6n+2, e.g. 20, 32, 44, 56, 110, 1202' + n = (depth - 2) // 6 + block = BasicBlock + elif block_name.lower() == 'bottleneck': + assert (depth - 2) % 9 == 0, 'When use bottleneck, depth should be 9n+2, e.g. 20, 29, 47, 56, 110, 1199' + n = (depth - 2) // 9 + block = Bottleneck + else: + raise ValueError('block_name shoule be Basicblock or Bottleneck') + + + self.inplanes = 16 + self.conv1 = nn.Conv2d(3, 16, kernel_size=3, padding=1, + bias=False) + self.bn1 = nn.BatchNorm2d(16) + self.relu = nn.ReLU(inplace=True) + self.layer1 = self._make_layer(block, 16, n) + self.layer2 = self._make_layer(block, 32, n, stride=2) + self.layer3 = self._make_layer(block, 64, n, stride=2) + self.avgpool = nn.AvgPool2d(8) + self.fc = nn.Linear(64 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, + kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) # 32x32 + + x = self.layer1(x) # 32x32 + x = self.layer2(x) # 16x16 + x = self.layer3(x) # 8x8 + + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + + return x + + +# def resnet(**kwargs): +# """ +# Constructs a ResNet model. +# """ +# return ResNet(**kwargs) \ No newline at end of file diff --git a/evaluation/GAIRAT/GAIR_RST/models/resnet.py b/evaluation/GAIRAT/GAIR_RST/models/resnet.py new file mode 100644 index 0000000..b143d3c --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/models/resnet.py @@ -0,0 +1,123 @@ +'''ResNet in PyTorch. + +For Pre-activation ResNet, see 'preact_resnet.py'. + +Reference: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 +''' +import torch +import torch.nn as nn +import torch.nn.functional as F + +from torch.autograd import Variable + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion*planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512*block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def ResNet18(): + return ResNet(BasicBlock, [2,2,2,2]) + +def ResNet34(): + return ResNet(BasicBlock, [3,4,6,3]) + +def ResNet50(): + return ResNet(Bottleneck, [3,4,6,3]) + +def ResNet101(): + return ResNet(Bottleneck, [3,4,23,3]) + +def ResNet152(): + return ResNet(Bottleneck, [3,8,36,3]) + + +def test(): + net = ResNet18() + y = net(Variable(torch.randn(1,3,32,32))) + print(y.size()) + print(net) +# test() \ No newline at end of file diff --git a/evaluation/GAIRAT/GAIR_RST/models/shake_shake.py b/evaluation/GAIRAT/GAIR_RST/models/shake_shake.py new file mode 100644 index 0000000..df3490a --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/models/shake_shake.py @@ -0,0 +1,200 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .shake_shake_function import get_alpha_beta, shake_function + +""" +Based on code from https://github.com/hysts/pytorch_shake_shake +""" + +def initialize_weights(module): + if isinstance(module, nn.Conv2d): + nn.init.kaiming_normal_(module.weight.data, mode='fan_out') + elif isinstance(module, nn.BatchNorm2d): + module.weight.data.fill_(1) + module.bias.data.zero_() + elif isinstance(module, nn.Linear): + module.bias.data.zero_() + + +class ResidualPath(nn.Module): + def __init__(self, in_channels, out_channels, stride): + super(ResidualPath, self).__init__() + + self.conv1 = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=False, + ) + self.bn1 = nn.BatchNorm2d(out_channels) + self.conv2 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.bn2 = nn.BatchNorm2d(out_channels) + + def forward(self, x): + x = F.relu(x, inplace=False) + x = F.relu(self.bn1(self.conv1(x)), inplace=False) + x = self.bn2(self.conv2(x)) + return x + + +class DownsamplingShortcut(nn.Module): + def __init__(self, in_channels): + super(DownsamplingShortcut, self).__init__() + self.conv1 = nn.Conv2d( + in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.conv2 = nn.Conv2d( + in_channels, + in_channels, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn = nn.BatchNorm2d(in_channels * 2) + + def forward(self, x): + x = F.relu(x, inplace=False) + y1 = F.avg_pool2d(x, kernel_size=1, stride=2, padding=0) + y1 = self.conv1(y1) + + y2 = F.pad(x[:, :, 1:, 1:], (0, 1, 0, 1)) + y2 = F.avg_pool2d(y2, kernel_size=1, stride=2, padding=0) + y2 = self.conv2(y2) + + z = torch.cat([y1, y2], dim=1) + z = self.bn(z) + + return z + + +class BasicBlock(nn.Module): + def __init__(self, in_channels, out_channels, stride, shake_config): + super(BasicBlock, self).__init__() + + self.shake_config = shake_config + + self.residual_path1 = ResidualPath(in_channels, out_channels, stride) + self.residual_path2 = ResidualPath(in_channels, out_channels, stride) + + self.shortcut = nn.Sequential() + if in_channels != out_channels: + self.shortcut.add_module('downsample', + DownsamplingShortcut(in_channels)) + + def forward(self, x): + x1 = self.residual_path1(x) + x2 = self.residual_path2(x) + + if self.training: + shake_config = self.shake_config + else: + shake_config = (False, False, False) + + alpha, beta = get_alpha_beta(x.size(0), shake_config, x.device) + y = shake_function(x1, x2, alpha, beta) + + return self.shortcut(x) + y + + +class ShakeNet(nn.Module): + def __init__(self, config): + super().__init__() + + input_shape = config['input_shape'] + n_classes = config['n_classes'] + + base_channels = config['base_channels'] + depth = config['depth'] + self.shake_config = (config['shake_forward'], config['shake_backward'], + config['shake_image']) + + block = BasicBlock + n_blocks_per_stage = (depth - 2) // 6 + assert n_blocks_per_stage * 6 + 2 == depth + + n_channels = [base_channels, base_channels * 2, base_channels * 4] + + self.conv = nn.Conv2d( + input_shape[1], + n_channels[0], + kernel_size=3, + stride=1, + padding=1, + bias=False) + self.bn = nn.BatchNorm2d(base_channels) + + self.stage1 = self._make_stage( + n_channels[0], n_channels[0], n_blocks_per_stage, block, stride=1) + self.stage2 = self._make_stage( + n_channels[0], n_channels[1], n_blocks_per_stage, block, stride=2) + self.stage3 = self._make_stage( + n_channels[1], n_channels[2], n_blocks_per_stage, block, stride=2) + + # compute conv feature size + with torch.no_grad(): + self.feature_size = self._forward_conv( + torch.zeros(*input_shape)).view(-1).shape[0] + + self.fc = nn.Linear(self.feature_size, n_classes) + + # initialize weights + self.apply(initialize_weights) + + def _make_stage(self, in_channels, out_channels, n_blocks, block, stride): + stage = nn.Sequential() + for index in range(n_blocks): + block_name = 'block{}'.format(index + 1) + if index == 0: + stage.add_module( + block_name, + block( + in_channels, + out_channels, + stride=stride, + shake_config=self.shake_config)) + else: + stage.add_module( + block_name, + block( + out_channels, + out_channels, + stride=1, + shake_config=self.shake_config)) + return stage + + def _forward_conv(self, x): + x = F.relu(self.bn(self.conv(x)), inplace=True) + x = self.stage1(x) + x = self.stage2(x) + x = self.stage3(x) + x = F.adaptive_avg_pool2d(x, output_size=1) + return x + + # def forward(self, x): + # x = self._forward_conv(x) + # x = x.view(x.size(0), -1) + # x = self.fc(x) + # return x + + def forward(self, x, return_prelogit=False): + prelogit = self._forward_conv(x) + prelogit = prelogit.view(x.size(0), -1) + out = self.fc(prelogit) + if return_prelogit: + return out, prelogit + else: + return out diff --git a/evaluation/GAIRAT/GAIR_RST/models/shake_shake_function.py b/evaluation/GAIRAT/GAIR_RST/models/shake_shake_function.py new file mode 100644 index 0000000..9b8b5b1 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/models/shake_shake_function.py @@ -0,0 +1,53 @@ +""" +Based on code from https://github.com/hysts/pytorch_shake_shake +""" + + +import torch +from torch.autograd import Function + +class ShakeFunction(Function): + @staticmethod + def forward(ctx, x1, x2, alpha, beta): + ctx.save_for_backward(x1, x2, alpha, beta) + + y = x1 * alpha + x2 * (1 - alpha) + return y + + @staticmethod + def backward(ctx, grad_output): + x1, x2, alpha, beta = ctx.saved_variables + grad_x1 = grad_x2 = grad_alpha = grad_beta = None + + if ctx.needs_input_grad[0]: + grad_x1 = grad_output * beta + if ctx.needs_input_grad[1]: + grad_x2 = grad_output * (1 - beta) + + return grad_x1, grad_x2, grad_alpha, grad_beta + + +shake_function = ShakeFunction.apply + + +def get_alpha_beta(batch_size, shake_config, device): + forward_shake, backward_shake, shake_image = shake_config + + if forward_shake and not shake_image: + alpha = torch.rand(1) + elif forward_shake and shake_image: + alpha = torch.rand(batch_size).view(batch_size, 1, 1, 1) + else: + alpha = torch.FloatTensor([0.5]) + + if backward_shake and not shake_image: + beta = torch.rand(1) + elif backward_shake and shake_image: + beta = torch.rand(batch_size).view(batch_size, 1, 1, 1) + else: + beta = torch.FloatTensor([0.5]) + + alpha = alpha.to(device) + beta = beta.to(device) + + return alpha, beta diff --git a/evaluation/GAIRAT/GAIR_RST/models/wideresnet.py b/evaluation/GAIRAT/GAIR_RST/models/wideresnet.py new file mode 100644 index 0000000..47b6e6d --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/models/wideresnet.py @@ -0,0 +1,98 @@ +"""Based on code from https://github.com/yaodongyu/TRADES""" + +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class WideResNet(nn.Module): + def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0.0): + super(WideResNet, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 4) % 6 == 0) + n = (depth - 4) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x, return_prelogit=False): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + if return_prelogit: + return self.fc(out), out + else: + return self.fc(out) + diff --git a/evaluation/GAIRAT/GAIR_RST/robust_self_training_GAIR.py b/evaluation/GAIRAT/GAIR_RST/robust_self_training_GAIR.py new file mode 100644 index 0000000..c981fe0 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/robust_self_training_GAIR.py @@ -0,0 +1,520 @@ +""" +Based on code from +https://github.com/yaircarmon/semisup-adv +""" + + +import os +import sys +import argparse + +import torch +import torch.nn.functional as F +import torch.optim as optim +from torchvision import transforms +import torch.backends.cudnn as cudnn +from torch.utils.data import DataLoader + +import pandas as pd +import numpy as np + +from utils import get_model + +from losses import GAIR_trades_loss ,trades_loss, noise_loss +from datasets import SemiSupervisedDataset, SemiSupervisedSampler, DATASETS +from attack_pgd import pgd +#from smoothing import quick_smoothing + +from autoaugment import CIFAR10Policy +from cutout import Cutout + +import logging + + +# ----------------------------- CONFIGURATION ---------------------------------- +parser = argparse.ArgumentParser( + description='PyTorch GAIR-TRADES Adversarial Training') + +# Dataset config +parser.add_argument('--dataset', type=str, default='cifar10', + choices=DATASETS, + help='The dataset to use for training)') +parser.add_argument('--data_dir', default='./data', type=str, + help='Directory where datasets are located') +parser.add_argument('--svhn_extra', action='store_true', default=False, + help='Adds the extra SVHN data') + +# Model config +parser.add_argument('--model', '-m', default='wrn-28-10', type=str, + help='Name of the model (see utils.get_model)') +parser.add_argument('--model_dir', default='./rst-model', + help='Directory of model for saving checkpoint') +parser.add_argument('--overwrite', action='store_true', default=False, + help='Cancels the run if an appropriate checkpoint is found') +parser.add_argument('--normalize_input', action='store_true', default=False, + help='Apply standard CIFAR normalization first thing ' + 'in the network (as part of the model, not in the data' + ' fetching pipline)') + +# Logging and checkpointing +parser.add_argument('--log_interval', type=int, default=40, + help='Number of batches between logging of training status') +parser.add_argument('--save_freq', default=25, type=int, + help='Checkpoint save frequency (in epochs)') + +# Generic training configs +parser.add_argument('--seed', type=int, default=1, + help='Random seed. ' + 'Note: fixing the random seed does not give complete ' + 'reproducibility. See ' + 'https://pytorch.org/docs/stable/notes/randomness.html') + +parser.add_argument('--batch_size', type=int, default=256, metavar='N', + help='Input batch size for training (default: 128)') +parser.add_argument('--test_batch_size', type=int, default=500, metavar='N', + help='Input batch size for testing (default: 128)') +parser.add_argument('--epochs', type=int, default=200, metavar='N', + help='Number of epochs to train. ' + 'Note: we arbitrarily define an epoch as a pass ' + 'through 50K datapoints. This is convenient for ' + 'comparison with standard CIFAR-10 training ' + 'configurations.') + +# Eval config +parser.add_argument('--eval_freq', default=1, type=int, + help='Eval frequency (in epochs)') +parser.add_argument('--train_eval_batches', default=None, type=int, + help='Maximum number for batches in training set eval') +parser.add_argument('--eval_attack_batches', default=1, type=int, + help='Number of eval batches to attack with PGD or certify ' + 'with randomized smoothing') + +# Optimizer config +parser.add_argument('--weight_decay', '--wd', default=5e-4, type=float) +parser.add_argument('--lr', type=float, default=0.1, metavar='LR', + help='Learning rate') +parser.add_argument('--lr_schedule', type=str, default='cosine', + choices=('trades', 'trades_fixed', 'cosine', 'wrn'), + help='Learning rate schedule') +parser.add_argument('--momentum', type=float, default=0.9, metavar='M', + help='SGD momentum') +parser.add_argument('--nesterov', action='store_true', default=True, + help='Use extragrdient steps') + +# Adversarial / stability training config +parser.add_argument('--loss', default='trades', type=str, + choices=('trades', 'noise'), + help='Which loss to use: TRADES-like KL regularization ' + 'or noise augmentation') + +parser.add_argument('--distance', '-d', default='l_2', type=str, + help='Metric for attack model: l_inf uses adversarial ' + 'training and l_2 uses stability training and ' + 'randomized smoothing certification', + choices=['l_inf', 'l_2']) +parser.add_argument('--epsilon', default=0.031, type=float, + help='Adversarial perturbation size (takes the role of' + ' sigma for stability training)') + +parser.add_argument('--pgd_num_steps', default=10, type=int, + help='number of pgd steps in adversarial training') +parser.add_argument('--pgd_step_size', default=0.007, + help='pgd steps size in adversarial training', type=float) +parser.add_argument('--beta', default=6.0, type=float, + help='stability regularization, i.e., 1/lambda in TRADES') + +# Semi-supervised training configuration +parser.add_argument('--aux_data_filename', default=None, type=str, + help='Path to pickle file containing unlabeled data and ' + 'pseudo-labels used for RST') + +parser.add_argument('--unsup_fraction', default=0.5, type=float, + help='Fraction of unlabeled examples in each batch; ' + 'implicitly sets the weight of unlabeled data in the ' + 'loss. If set to -1, batches are sampled from a ' + 'single pool') +parser.add_argument('--aux_take_amount', default=None, type=int, + help='Number of random aux examples to retain. ' + 'None retains all aux data.') + +parser.add_argument('--remove_pseudo_labels', action='store_true', + default=False, + help='Performs training without pseudo-labels (rVAT)') +parser.add_argument('--entropy_weight', type=float, + default=0.0, help='Weight on entropy loss') + +# Additional aggressive data augmentation +parser.add_argument('--autoaugment', action='store_true', default=False, + help='Use autoaugment for data augmentation') +parser.add_argument('--cutout', action='store_true', default=False, + help='Use cutout for data augmentation') + +# GAIR parameters +parser.add_argument('--Lambda', default=0.0, type=float) + +# Resume +parser.add_argument('--resume', default=None ,type=str) + +args = parser.parse_args() + +# ------------------------------ OUTPUT SETUP ---------------------------------- +model_dir = args.model_dir +if not os.path.exists(model_dir): + os.makedirs(model_dir) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(message)s", + handlers=[ + logging.FileHandler(os.path.join(args.model_dir, 'training.log')), + logging.StreamHandler() + ]) +logger = logging.getLogger() + +logging.info('Robust self-training') +logging.info('Args: %s', args) + +if not args.overwrite: + final_checkpoint_path = os.path.join( + model_dir, 'checkpoint-epoch{}.pt'.format(args.epochs)) + if os.path.exists(final_checkpoint_path): + logging.info('Appropriate checkpoint found - quitting!') + sys.exit(0) +# ------------------------------------------------------------------------------ + +# ------------------------------- CUDA SETUP ----------------------------------- +# should provide some improved performance +cudnn.benchmark = True +# useful setting for debugging +# cudnn.benchmark = False +# cudnn.deterministic = True + +use_cuda = torch.cuda.is_available() +torch.manual_seed(args.seed) +device = torch.device('cuda' if use_cuda else 'cpu') +# ------------------------------------------------------------------------------ + +# --------------------------- DATA AUGMENTATION -------------------------------- +if args.dataset == 'cifar10': + transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + ]) +elif args.dataset == 'svhn': + # the WRN paper does no augmentation on SVHN + # obviously flipping is a bad idea, and it makes some sense not to + # crop because there are a lot of distractor digits in the edges of the + # image + transform_train = transforms.ToTensor() + +if args.autoaugment or args.cutout: + assert (args.dataset == 'cifar10') + transform_list = [ + transforms.RandomCrop(32, padding=4, fill=128), + # fill parameter needs torchvision installed from source + transforms.RandomHorizontalFlip()] + if args.autoaugment: + transform_list.append(CIFAR10Policy()) + transform_list.append(transforms.ToTensor()) + if args.cutout: + transform_list.append(Cutout(n_holes=1, length=16)) + + transform_train = transforms.Compose(transform_list) + logger.info('Applying aggressive training augmentation: %s' + % transform_train) + +transform_test = transforms.Compose([ + transforms.ToTensor()]) +# ------------------------------------------------------------------------------ + +# ----------------- DATASET WITH AUX PSEUDO-LABELED DATA ----------------------- +trainset = SemiSupervisedDataset(base_dataset=args.dataset, + add_svhn_extra=args.svhn_extra, + root=args.data_dir, train=True, + download=True, transform=transform_train, + aux_data_filename=args.aux_data_filename, + add_aux_labels=not args.remove_pseudo_labels, + aux_take_amount=args.aux_take_amount) + +# num_batches=50000 enforces the definition of an "epoch" as passing through 50K +# datapoints +# TODO: make sure that this code works also when trainset.unsup_indices=[] +train_batch_sampler = SemiSupervisedSampler( + trainset.sup_indices, trainset.unsup_indices, + args.batch_size, args.unsup_fraction, + num_batches=int(np.ceil(50000 / args.batch_size))) +epoch_size = len(train_batch_sampler) * args.batch_size + +kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} +train_loader = DataLoader(trainset, batch_sampler=train_batch_sampler, **kwargs) + +testset = SemiSupervisedDataset(base_dataset=args.dataset, + root=args.data_dir, train=False, + download=True, + transform=transform_test) +test_loader = DataLoader(testset, batch_size=args.test_batch_size, + shuffle=False, **kwargs) + +trainset_eval = SemiSupervisedDataset( + base_dataset=args.dataset, + add_svhn_extra=args.svhn_extra, + root=args.data_dir, train=True, + download=True, transform=transform_train) + +eval_train_loader = DataLoader(trainset_eval, batch_size=args.test_batch_size, + shuffle=True, **kwargs) + +eval_test_loader = DataLoader(testset, batch_size=args.test_batch_size, + shuffle=False, **kwargs) +# ------------------------------------------------------------------------------ + + +# ----------------------- TRAIN AND EVAL FUNCTIONS ----------------------------- +def train(args, model, device, train_loader, optimizer, epoch, Lam): + model.train() + train_metrics = [] + epsilon = args.epsilon + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + + optimizer.zero_grad() + + # calculate robust loss + if args.loss == 'trades': + # The TRADES KL-robustness regularization term proposed by + # Zhang et al., with some additional features + (loss, natural_loss, robust_loss, + entropy_loss_unlabeled) = GAIR_trades_loss( + model=model, + x_natural=data, + y=target, + optimizer=optimizer, + step_size=args.pgd_step_size, + epsilon=epsilon, + perturb_steps=args.pgd_num_steps, + beta=args.beta, + distance=args.distance, + adversarial=args.distance == 'l_inf', + entropy_weight=args.entropy_weight, + Lambda=Lam) + + elif args.loss == 'noise': + # Augmenting the input with random noise as in Cohen et al. + assert (args.distance == 'l_2') + loss = noise_loss(model=model, x_natural=data, + y=target, clamp_x=True, epsilon=epsilon) + entropy_loss_unlabeled = torch.Tensor([0.]) + natural_loss = robust_loss = loss + + loss.backward() + optimizer.step() + + train_metrics.append(dict( + epoch=epoch, + loss=loss.item(), + natural_loss=natural_loss.item(), + robust_loss=robust_loss.item(), + entropy_loss_unlabeled=entropy_loss_unlabeled.item())) + + # print progress + if batch_idx % args.log_interval == 0: + logging.info( + 'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), epoch_size, + 100. * batch_idx / len(train_loader), loss.item())) + + return train_metrics + + +def eval(args, model, device, eval_set, loader): + loss = 0 + total = 0 + correct = 0 + adv_correct = 0 + adv_correct_clean = 0 + adv_total = 0 + + model.eval() + with torch.no_grad(): + for batch_idx, (data, target) in enumerate(loader): + data, target = data.to(device), target.to(device) + data, target = data[target != -1], target[target != -1] + output = model(data) + loss += F.cross_entropy(output, target, reduction='sum').item() + pred = output.max(1, keepdim=True)[1] + correct += pred.eq(target.view_as(pred)).sum().item() + if batch_idx < args.eval_attack_batches: + if args.distance == 'l_2': + # run coarse certification + #incorrect_clean, incorrect_rob = quick_smoothing( + # model, data, target, + # sigma=args.epsilon, + # eps=args.epsilon, + # num_smooth=100, batch_size=1000) + pass + elif args.distance == 'l_inf': + # run medium-strength gradient attack + is_correct_clean, is_correct_rob = pgd( + model, data, target, + epsilon=args.epsilon, + num_steps=2 * args.pgd_num_steps, + step_size=args.pgd_step_size, + random_start=False) + incorrect_clean = (1-is_correct_clean).sum() + incorrect_rob = (1-np.prod(is_correct_rob, axis=1)).sum() + else: + raise ValueError('No support for distance %s', + args.distance) + adv_correct_clean += (len(data) - int(incorrect_clean)) + adv_correct += (len(data) - int(incorrect_rob)) + adv_total += len(data) + total += len(data) + if ((eval_set == 'train') and + (batch_idx + 1 == args.train_eval_batches)): + break + loss /= total + accuracy = correct / total + if adv_total > 0: + robust_clean_accuracy = adv_correct_clean / adv_total + robust_accuracy = adv_correct / adv_total + else: + robust_accuracy = robust_clean_accuracy = 0. + + eval_data = dict(loss=loss, accuracy=accuracy, + robust_accuracy=robust_accuracy, + robust_clean_accuracy=robust_clean_accuracy) + eval_data = {eval_set + '_' + k: v for k, v in eval_data.items()} + logging.info( + '{}: Clean loss: {:.4f}, ' + 'Clean accuracy: {}/{} ({:.2f}%), ' + '{} clean accuracy: {}/{} ({:.2f}%), ' + 'Robust accuracy {}/{} ({:.2f}%)'.format( + eval_set.upper(), loss, + correct, total, 100.0 * accuracy, + 'Smoothing' if args.distance == 'l_2' else 'PGD', + adv_correct_clean, adv_total, 100.0 * robust_clean_accuracy, + adv_correct, adv_total, 100.0 * robust_accuracy)) + + return eval_data + +def lam(epoch): + Lam = 100.0 + if epoch >= 0.5 * args.epochs: + Lam = args.Lambda + return Lam + + +def adjust_learning_rate(optimizer, epoch): + """decrease the learning rate""" + lr = args.lr + schedule = args.lr_schedule + # schedule from TRADES repo (different from paper due to bug there) + if schedule == 'trades': + if epoch >= 0.75 * args.epochs: + lr = args.lr * 0.1 + # schedule as in TRADES paper + elif schedule == 'trades_fixed': + if epoch >= 0.75 * args.epochs: + lr = args.lr * 0.1 + if epoch >= 0.9 * args.epochs: + lr = args.lr * 0.01 + if epoch >= args.epochs: + lr = args.lr * 0.001 + # cosine schedule + elif schedule == 'cosine': + lr = args.lr * 0.5 * (1 + np.cos((epoch - 1) / args.epochs * np.pi)) + # schedule as in WRN paper + elif schedule == 'wrn': + if epoch >= 0.3 * args.epochs: + lr = args.lr * 0.2 + if epoch >= 0.6 * args.epochs: + lr = args.lr * 0.2 * 0.2 + if epoch >= 0.8 * args.epochs: + lr = args.lr * 0.2 * 0.2 * 0.2 + else: + raise ValueError('Unkown LR schedule %s' % schedule) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr +# ------------------------------------------------------------------------------ + +# ----------------------------- TRAINING LOOP ---------------------------------- +start_epoch = 1 + +num_classes = 10 +model = get_model(args.model, num_classes=num_classes, + normalize_input=args.normalize_input) +if use_cuda: + model = torch.nn.DataParallel(model).cuda() +optimizer = optim.SGD(model.parameters(), lr=args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay, + nesterov=args.nesterov) + +if args.resume: + # resume directly point to checkpoint.pth.tar + print ('==> Resuming from checkpoint ..') + print(args.resume) + assert os.path.isfile(args.resume) + #out_dir = os.path.dirname(resume) + checkpoint = torch.load(args.resume) + start_epoch = checkpoint['save_epoch'] + 1 + num_classes = checkpoint['num_classes'] + #best_acc = checkpoint['test_pgd20_acc'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['opt']) + +def main(): + train_df = pd.DataFrame() + eval_df = pd.DataFrame() + + for epoch in range(start_epoch, args.epochs + 1): + # adjust learning rate for SGD + Lam = lam(epoch) + lr = adjust_learning_rate(optimizer, epoch) + logger.info('Setting learning rate to %g' % lr) + # adversarial training + train_data = train(args, model, device, train_loader, optimizer, epoch, Lam) + train_df = train_df.append(pd.DataFrame(train_data), ignore_index=True) + + # evaluation on natural examples + logging.info(120 * '=') + if epoch % args.eval_freq == 0 or epoch == args.epochs: + eval_data = {'epoch': int(epoch)} + eval_data.update( + eval(args, model, device, 'train', eval_train_loader)) + eval_data.update( + eval(args, model, device, 'test', eval_test_loader)) + eval_df = eval_df.append(pd.Series(eval_data), ignore_index=True) + logging.info(120 * '=') + + # save stats + train_df.to_csv(os.path.join(model_dir, 'stats_train.csv')) + eval_df.to_csv(os.path.join(model_dir, 'stats_eval.csv')) + + # save checkpoint + if epoch % args.save_freq == 0 or epoch == args.epochs: + torch.save(dict(num_classes=num_classes, + state_dict=model.state_dict(), + normalize_input=args.normalize_input, + save_epoch=epoch, + opt=optimizer.state_dict()), + os.path.join(model_dir, + 'checkpoint-epoch{}.pt'.format(epoch))) + torch.save(optimizer.state_dict(), + os.path.join(model_dir, + 'opt-checkpoint_epoch{}.tar'.format(epoch))) + + # save last checkpoint + torch.save(dict(num_classes=num_classes, + state_dict=model.state_dict(), + normalize_input=args.normalize_input, + save_epoch=epoch, + opt=optimizer.state_dict()), + os.path.join(model_dir, + 'lastcheckpoint.pt')) +# ------------------------------------------------------------------------------ + +if __name__ == '__main__': + main() diff --git a/evaluation/GAIRAT/GAIR_RST/run_evaluation.sh b/evaluation/GAIRAT/GAIR_RST/run_evaluation.sh new file mode 100644 index 0000000..45d41a1 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/run_evaluation.sh @@ -0,0 +1 @@ +CUDA_VISIBLE_DEVICES='0' python3 attack_evaluation.py --model_path='./GAIR_RST_check/checkpoint-epoch200.pt' --output_suffix='GAIR_RST_PGD_plus_results' & diff --git a/evaluation/GAIRAT/GAIR_RST/run_training.sh b/evaluation/GAIRAT/GAIR_RST/run_training.sh new file mode 100644 index 0000000..44bc201 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/run_training.sh @@ -0,0 +1 @@ +CUDA_VISIBLE_DEVICES='0,1,2,3' python3 robust_self_training_GAIR.py --aux_data_filename='ti_500K_pseudo_labeled.pickle' --distance='l_inf' --epsilon=0.031 --Lambda=3.0 --model_dir='./GAIR_RST_check' & diff --git a/evaluation/GAIRAT/GAIR_RST/utils.py b/evaluation/GAIRAT/GAIR_RST/utils.py new file mode 100644 index 0000000..e460335 --- /dev/null +++ b/evaluation/GAIRAT/GAIR_RST/utils.py @@ -0,0 +1,177 @@ +""" +Utilities. Partially based on code from +https://github.com/modestyachts/CIFAR-10.1 +""" +import io +import json +import os +import pickle + +import numpy as np +import pathlib + +from models.wideresnet import WideResNet +from models.shake_shake import ShakeNet +from models.cifar_resnet import ResNet + +import torch +from torch.nn import Sequential, Module + +cifar10_label_names = ['airplane', 'automobile', 'bird', + 'cat', 'deer', 'dog', 'frog', 'horse', + 'ship', 'truck'] + + +def get_model(name, num_classes=10, normalize_input=False): + name_parts = name.split('-') + if name_parts[0] == 'wrn': + depth = int(name_parts[1]) + widen = int(name_parts[2]) + model = WideResNet( + depth=depth, num_classes=num_classes, widen_factor=widen) + + elif name_parts[0] == 'ss': + model = ShakeNet(dict(depth=int(name_parts[1]), + base_channels=int(name_parts[2]), + shake_forward=True, shake_backward=True, + shake_image=True, input_shape=(1, 3, 32, 32), + n_classes=num_classes, + )) + elif name_parts[0] == 'resnet': + model = ResNet(num_classes=num_classes, depth=int(name_parts[1])) + else: + raise ValueError('Could not parse model name %s' % name) + + if normalize_input: + model = Sequential(NormalizeInput(), model) + + return model + + +class NormalizeInput(Module): + def __init__(self, mean=(0.4914, 0.4822, 0.4465), + std=(0.2023, 0.1994, 0.2010)): + super().__init__() + + self.register_buffer('mean', torch.Tensor(mean).reshape(1, -1, 1, 1)) + self.register_buffer('std', torch.Tensor(std).reshape(1, -1, 1, 1)) + + def forward(self, x): + return (x - self.mean) / self.std + +# TODO: decide whether to remove all code below this line +# Should we add some description of how these files +# were obtained? +def load_tinyimage_subset(other_data_path, + version_string='v7'): + image_data_filename = 'tinyimage_subset_data' + if version_string != '': + image_data_filename += '_' + version_string + image_data_filename += '.pickle' + image_data_filepath = os.path.abspath(os.path.join(other_data_path, image_data_filename)) + indices_filename = 'tinyimage_subset_indices' + if version_string != '': + indices_filename += '_' + version_string + indices_filename += '.json' + indices_filepath = os.path.abspath(os.path.join(other_data_path, indices_filename)) + print('Loading indices from file {}'.format(indices_filepath)) + assert pathlib.Path(indices_filepath).is_file() + print('Loading image data from file {}'.format(image_data_filepath)) + assert pathlib.Path(image_data_filepath).is_file() + with open(indices_filepath, 'r') as f: + indices = json.load(f) + with open(image_data_filepath, 'rb') as f: + image_data = pickle.load(f) + num_entries = 0 + for kw, kw_indices in indices.items(): + for entry in kw_indices: + assert entry['tinyimage_index'] in image_data + num_entries += 1 + assert num_entries == len(image_data) + return indices, image_data + + +def load_cifar10_by_keyword(unique_keywords=True, version_string='v7'): + cifar10_keywords = load_cifar10_keywords(unique_keywords=unique_keywords, + lists_for_unique=True, + version_string=version_string) + cifar10_by_keyword = {} + for ii, keyword_entries in enumerate(cifar10_keywords): + for entry in keyword_entries: + cur_keyword = entry['nn_keyword'] + if not cur_keyword in cifar10_by_keyword: + cifar10_by_keyword[cur_keyword] = [] + cifar10_by_keyword[cur_keyword].append(ii) + return cifar10_by_keyword + + +def load_cifar10_keywords(other_data_path, + unique_keywords=True, + lists_for_unique=False, + version_string='v7'): + filename = 'cifar10_keywords' + if unique_keywords: + filename += '_unique' + if version_string != '': + filename += '_' + version_string + filename += '.json' + keywords_filepath = os.path.abspath(os.path.join(other_data_path, filename)) + print('Loading keywords from file {}'.format(keywords_filepath)) + assert pathlib.Path(keywords_filepath).is_file() + with open(keywords_filepath, 'r') as f: + cifar10_keywords = json.load(f) + if unique_keywords and lists_for_unique: + result = [] + for entry in cifar10_keywords: + result.append([entry]) + else: + result = cifar10_keywords + assert len(result) == 60000 + return result + + +def load_distances_to_cifar10(version_string='v7'): + data_path = os.path.join(os.path.dirname(__file__), '../data/') + filename = 'tinyimage_cifar10_distances' + if version_string != '': + filename += '_' + version_string + filename += '.json' + filepath = os.path.abspath(os.path.join(data_path, filename)) + print('Loading distances from file {}'.format(filepath)) + assert pathlib.Path(filepath).is_file() + with open(filepath, 'r') as f: + tmp = json.load(f) + if version_string == 'v4': + assert len(tmp) == 372131 + elif version_string == 'v6': + assert len(tmp) == 1646248 + elif version_string == 'v7': + assert len(tmp) == 589711 + result = {} + for k, v in tmp.items(): + result[int(k)] = v + return result + + +def load_new_test_data_indices(version_string='v7'): + data_path = os.path.join(os.path.dirname(__file__), '../data/') + filename = 'cifar10.1' + if version_string == '': + version_string = 'v7' + if version_string in ['v4', 'v6', 'v7']: + filename += '_' + version_string + else: + raise ValueError('Unknown dataset version "{}".'.format(version_string)) + ti_indices_data_path = os.path.join(os.path.dirname(__file__), '../data/') + ti_indices_filename = 'cifar10.1_' + version_string + '_ti_indices.json' + ti_indices_filepath = os.path.abspath(os.path.join(ti_indices_data_path, ti_indices_filename)) + print('Loading Tiny Image indices from file {}'.format(ti_indices_filepath)) + assert pathlib.Path(ti_indices_filepath).is_file() + with open(ti_indices_filepath, 'r') as f: + tinyimage_indices = json.load(f) + assert type(tinyimage_indices) is list + if version_string == 'v6' or version_string == 'v7': + assert len(tinyimage_indices) == 2000 + elif version_string == 'v4': + assert len(tinyimage_indices) == 2021 + return tinyimage_indices diff --git a/evaluation/GAIRAT/__pycache__/GAIR.cpython-36.pyc b/evaluation/GAIRAT/__pycache__/GAIR.cpython-36.pyc new file mode 100644 index 0000000..e0cbdec Binary files /dev/null and b/evaluation/GAIRAT/__pycache__/GAIR.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/__pycache__/GAIR.cpython-37.pyc b/evaluation/GAIRAT/__pycache__/GAIR.cpython-37.pyc new file mode 100644 index 0000000..f91c61c Binary files /dev/null and b/evaluation/GAIRAT/__pycache__/GAIR.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/__pycache__/attack_generator.cpython-36.pyc b/evaluation/GAIRAT/__pycache__/attack_generator.cpython-36.pyc new file mode 100644 index 0000000..900ced1 Binary files /dev/null and b/evaluation/GAIRAT/__pycache__/attack_generator.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/__pycache__/attack_generator.cpython-37.pyc b/evaluation/GAIRAT/__pycache__/attack_generator.cpython-37.pyc new file mode 100644 index 0000000..0f4c7ac Binary files /dev/null and b/evaluation/GAIRAT/__pycache__/attack_generator.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/attack_cw.py b/evaluation/GAIRAT/attack_cw.py new file mode 100644 index 0000000..356a282 --- /dev/null +++ b/evaluation/GAIRAT/attack_cw.py @@ -0,0 +1,368 @@ +""" +Implementation of L_infty Carlini-Wagner attack based on the L2 implementation +in FoolBox v1.9 (with many dependencies on that pakage) +https://github.com/bethgelab/foolbox + +For PGD plus evaluation +Based on https://github.com/yaircarmon/semisup-adv +""" + +try: + from foolbox.models import PyTorchModel + from foolbox.attacks import Attack + from foolbox.attacks.base import call_decorator + from foolbox.utils import onehot_like + from foolbox.distances import Linf + HAVE_FOOLBOX = True +except ImportError: + HAVE_FOOLBOX = False + Attack = object + call_decorator = lambda x: x +import numpy as np +import logging + + +def cw(model, + X, + y, + binary_search_steps=5, + max_iterations=1000, + learning_rate=5E-3, + initial_const=1E-2, + tau_decrease_factor=0.9 + ): + if not HAVE_FOOLBOX: + raise ImportError('Could not import FoolBox') + + foolbox_model = PyTorchModel(model, bounds=(0, 1), num_classes=10) + attack = CarliniWagnerLIAttack(foolbox_model, distance=Linf) + linf_distances = [] + for i in range(len(X)): + logging.info('Example: %g', i) + image = X[i, :].detach().cpu().numpy() + label = y[i].cpu().numpy() + adversarial = attack(image, label, + binary_search_steps=binary_search_steps, + max_iterations=max_iterations, + learning_rate=learning_rate, + initial_const=initial_const, + tau_decrease_factor=tau_decrease_factor) + logging.info('Linf distance: %g', np.max(np.abs(adversarial - image))) + linf_distances.append(np.max(np.abs(adversarial - image))) + return linf_distances + + +class CarliniWagnerLIAttack(Attack): + """The Linf version of the Carlini & Wagner attack. + + This attack is described in [1]_. This implementation + is based on the reference implementation by Carlini [2]_. + For bounds \ne (0, 1), it differs from [2]_ because we + normalize the squared L2 loss with the bounds. + + References + ---------- + .. [1] Nicholas Carlini, David Wagner: "Towards Evaluating the + Robustness of Neural Networks", https://arxiv.org/abs/1608.04644 + .. [2] https://github.com/carlini/nn_robust_attacks + + """ + + @call_decorator + def __call__(self, input_or_adv, label=None, unpack=True, + binary_search_steps=5, + tau_decrease_factor=0.9, + max_iterations=1000, + confidence=0, learning_rate=5e-3, + initial_const=1e-2, abort_early=True): + + """The L_infty version of the Carlini & Wagner attack. + + Parameters + ---------- + input_or_adv : `numpy.ndarray` or :class:`Adversarial` + The original, unperturbed input as a `numpy.ndarray` or + an :class:`Adversarial` instance. + label : int + The reference label of the original input. Must be passed + if `a` is a `numpy.ndarray`, must not be passed if `a` is + an :class:`Adversarial` instance. + unpack : bool + If true, returns the adversarial input, otherwise returns + the Adversarial object. + binary_search_steps : int + The number of steps for the binary search used to + find the optimal tradeoff-constant between distance and confidence. + max_iterations : int + The maximum number of iterations. Larger values are more + accurate; setting it too small will require a large learning rate + and will produce poor results. + confidence : int or float + Confidence of adversarial examples: a higher value produces + adversarials that are further away, but more strongly classified + as adversarial. + learning_rate : float + The learning rate for the attack algorithm. Smaller values + produce better results but take longer to converge. + initial_const : float + The initial tradeoff-constant to use to tune the relative + importance of distance and confidence. If `binary_search_steps` + is large, the initial constant is not important. + abort_early : bool + If True, Adam will be aborted if the loss hasn't decreased + for some time (a tenth of max_iterations). + + """ + + a = input_or_adv + + del input_or_adv + del label + del unpack + + if not a.has_gradient(): + logging.fatal('Applied gradient-based attack to model that ' + 'does not provide gradients.') + return + + min_, max_ = a.bounds() + + def to_attack_space(x): + # map from [min_, max_] to [-1, +1] + a = (min_ + max_) / 2 + b = (max_ - min_) / 2 + x = (x - a) / b + + # from [-1, +1] to approx. (-1, +1) + x = x * 0.999999 + + # from (-1, +1) to (-inf, +inf) + return np.arctanh(x) + + def to_model_space(x): + """Transforms an input from the attack space + to the model space. This transformation and + the returned gradient are elementwise.""" + + # from (-inf, +inf) to (-1, +1) + x = np.tanh(x) + + grad = 1 - np.square(x) + + # map from (-1, +1) to (min_, max_) + a = (min_ + max_) / 2 + b = (max_ - min_) / 2 + x = x * b + a + + grad = grad * b + return x, grad + + # variables representing inputs in attack space will be + # prefixed with att_ + att_original = to_attack_space(a.original_image) + + # will be close but not identical to a.original_image + reconstructed_original, _ = to_model_space(att_original) + + # the binary search finds the smallest const for which we + # find an adversarial + const = initial_const + lower_bound = 0 + upper_bound = np.inf + true_logits, _ = a.predictions(reconstructed_original) + true_label = np.argmax(true_logits) + # Binary search for constant + start_tau = 1.0 + for binary_search_step in range(binary_search_steps): + if binary_search_step == binary_search_steps - 1 and \ + binary_search_steps >= 10: + # in the last binary search step, use the upper_bound instead + # TODO: find out why... it's not obvious why this is useful + const = upper_bound + + logging.info( + 'starting optimization with const = {}, best overall distance = {}'.format( + const, a.distance)) + + # found adv with the current const + + att_warmstart = att_original + tau = start_tau + while tau > 1. / 256: + found_adv = False + att_perturbation = np.zeros_like(att_original) + # create a new optimizer to minimize the perturbation + optimizer = AdamOptimizer(att_perturbation.shape) + loss_at_previous_check = np.inf + # logging.info('Running with const:{}, tau:{}'.format(const, tau)) + for iteration in range(max_iterations): + + x, dxdp = to_model_space(att_warmstart + att_perturbation) + logits, i = a.predictions(x) + false_label = np.argmax(logits) + is_adv = not (false_label == true_label) + + loss, dldx, adv_loss, distance = self.loss_function( + const, tau, a, x, logits, reconstructed_original, + confidence, min_, max_) + + check_loss = logits[true_label] - logits[false_label] + + # logging.info('Iteration:{}, loss: {}; adv_loss:{}; distance:{}'.format( + # iteration, loss, adv_loss, distance)) + # backprop the gradient of the loss w.r.t. x further + # to get the gradient of the loss w.r.t. att_perturbation + assert dldx.shape == x.shape + assert dxdp.shape == x.shape + # we can do a simple elementwise multiplication, because + # grad_x_wrt_p is a matrix of elementwise derivatives + # (i.e. each x[i] w.r.t. p[i] only, for all i) and + # grad_loss_wrt_x is a real gradient reshaped as a matrix + gradient = dldx * dxdp + + att_perturbation += optimizer(gradient, learning_rate) + x, dxdp = to_model_space(att_warmstart + att_perturbation) + + if is_adv: + # Tau + binary search was successful but continuing opt + found_adv = True + + if abort_early and \ + iteration % (np.ceil(max_iterations / 10)) == 0: + # after each tenth of the iterations, check progress + # logging.info('Iteration:{}, loss: {}; best overall distance: {}; is_adv:{}'.format( + # iteration, loss, a.distance, is_adv)) + if not (loss_at_previous_check - loss > 0.0001): + break # stop Adam if there has not been progress + loss_at_previous_check = loss + + if not found_adv: + # This constant is fine, just that we broke out of tau + if tau < start_tau: + found_adv = True + start_tau = tau + break + + else: + actualtau = np.max(np.abs(x - reconstructed_original)) + if actualtau < tau: + tau = actualtau + tau = tau * tau_decrease_factor + att_warmstart = to_attack_space(x) + + if found_adv: + # logging.info('found adversarial with const = {}'.format(const)) + upper_bound = const + else: + # logging.info('failed to find adversarial ' + # 'with const = {}'.format(const)) + lower_bound = const + + if upper_bound == np.inf: + # exponential search + const *= 10 + else: + # binary search + const = (lower_bound + upper_bound) / 2 + + @classmethod + def loss_function(cls, const, tau, a, x, logits, reconstructed_original, + confidence, min_, max_): + """Returns the loss and the gradient of the loss w.r.t. x, + assuming that logits = model(x).""" + + targeted = a.target_class() is not None + if targeted: + c_minimize = cls.best_other_class(logits, a.target_class()) + c_maximize = a.target_class() + else: + c_minimize = a.original_class + c_maximize = cls.best_other_class(logits, a.original_class) + + is_adv_loss = logits[c_minimize] - logits[c_maximize] + + # is_adv is True as soon as the is_adv_loss goes below 0 + # but sometimes we want additional confidence + is_adv_loss += confidence + is_adv_loss = max(0, is_adv_loss) + + s = max_ - min_ + # squared_l2_distance = np.sum((x - reconstructed_original)**2) / s**2 + linf_distance = np.sum( + np.maximum(np.abs(x - reconstructed_original) - tau, 0)) + + total_loss = linf_distance + const * is_adv_loss + + # calculate the gradient of total_loss w.r.t. x + logits_diff_grad = np.zeros_like(logits) + logits_diff_grad[c_minimize] = 1 + logits_diff_grad[c_maximize] = -1 + is_adv_loss_grad = a.backward(logits_diff_grad, x) + assert is_adv_loss >= 0 + if is_adv_loss == 0: + is_adv_loss_grad = 0 + + squared_l2_distance_grad = (2 / s ** 2) * (x - reconstructed_original) + linf_distance_grad = np.sign(x - reconstructed_original) * ( + np.abs(x - reconstructed_original) - tau > 0) + total_loss_grad = linf_distance_grad + const * is_adv_loss_grad + return total_loss, total_loss_grad, is_adv_loss, linf_distance + + @staticmethod + def best_other_class(logits, exclude): + """Returns the index of the largest logit, ignoring the class that + is passed as `exclude`.""" + other_logits = logits - onehot_like(logits, exclude, value=np.inf) + return np.argmax(other_logits) + + +class AdamOptimizer: + """Basic Adam optimizer implementation that can minimize w.r.t. + a single variable. + + Parameters + ---------- + shape : tuple + shape of the variable w.r.t. which the loss should be minimized + + """ + + def __init__(self, shape): + self.m = np.zeros(shape) + self.v = np.zeros(shape) + self.t = 0 + + def __call__(self, gradient, learning_rate, + beta1=0.9, beta2=0.999, epsilon=10e-8): + """Updates internal parameters of the optimizer and returns + the change that should be applied to the variable. + + Parameters + ---------- + gradient : `np.ndarray` + the gradient of the loss w.r.t. to the variable + learning_rate: float + the learning rate in the current iteration + beta1: float + decay rate for calculating the exponentially + decaying average of past gradients + beta2: float + decay rate for calculating the exponentially + decaying average of past squared gradients + epsilon: float + small value to avoid division by zero + + """ + + self.t += 1 + + self.m = beta1 * self.m + (1 - beta1) * gradient + self.v = beta2 * self.v + (1 - beta2) * gradient ** 2 + + bias_correction_1 = 1 - beta1 ** self.t + bias_correction_2 = 1 - beta2 ** self.t + + m_hat = self.m / bias_correction_1 + v_hat = self.v / bias_correction_2 + + return -learning_rate * m_hat / (np.sqrt(v_hat) + epsilon) diff --git a/evaluation/GAIRAT/attack_generator.py b/evaluation/GAIRAT/attack_generator.py new file mode 100644 index 0000000..cd5aefe --- /dev/null +++ b/evaluation/GAIRAT/attack_generator.py @@ -0,0 +1,85 @@ +import numpy as np +from models import * +from torch.autograd import Variable + +def cwloss(output, target,confidence=50, num_classes=10): + # Compute the probability of the label class versus the maximum other + # The same implementation as in repo CAT https://github.com/sunblaze-ucb/curriculum-adversarial-training-CAT + target = target.data + target_onehot = torch.zeros(target.size() + (num_classes,)) + target_onehot = target_onehot.cuda() + target_onehot.scatter_(1, target.unsqueeze(1), 1.) + target_var = Variable(target_onehot, requires_grad=False) + real = (target_var * output).sum(1) + other = ((1. - target_var) * output - target_var * 10000.).max(1)[0] + loss = -torch.clamp(real - other + confidence, min=0.) # equiv to max(..., 0.) + loss = torch.sum(loss) + return loss + +# Geometry-aware projected gradient descent (GA-PGD) +def GA_PGD(model, data, target, epsilon, step_size, num_steps,loss_fn,category,rand_init): + model.eval() + Kappa = torch.zeros(len(data)) + if category == "trades": + x_adv = data.detach() + 0.001 * torch.randn(data.shape).cuda().detach() if rand_init else data.detach() + nat_output = model(data) + if category == "Madry": + x_adv = data.detach() + torch.from_numpy(np.random.uniform(-epsilon, epsilon, data.shape)).float().cuda() if rand_init else data.detach() + x_adv = torch.clamp(x_adv, 0.0, 1.0) + for k in range(num_steps): + x_adv.requires_grad_() + output = model(x_adv) + predict = output.max(1, keepdim=True)[1] + # Update Kappa + for p in range(len(x_adv)): + if predict[p] == target[p]: + Kappa[p] += 1 + model.zero_grad() + with torch.enable_grad(): + if loss_fn == "cent": + loss_adv = nn.CrossEntropyLoss(reduction="mean")(output, target) + if loss_fn == "cw": + loss_adv = cwloss(output,target) + if loss_fn == "kl": + criterion_kl = nn.KLDivLoss(size_average=False).cuda() + loss_adv = criterion_kl(F.log_softmax(output, dim=1),F.softmax(nat_output, dim=1)) + loss_adv.backward() + eta = step_size * x_adv.grad.sign() + # Update adversarial data + x_adv = x_adv.detach() + eta + x_adv = torch.min(torch.max(x_adv, data - epsilon), data + epsilon) + x_adv = torch.clamp(x_adv, 0.0, 1.0) + x_adv = Variable(x_adv, requires_grad=False) + return x_adv, Kappa + +def eval_clean(model, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.cuda(), target.cuda() + output = model(data) + test_loss += F.cross_entropy(output, target, size_average=False).item() + pred = output.max(1, keepdim=True)[1] + correct += pred.eq(target.view_as(pred)).sum().item() + test_loss /= len(test_loader.dataset) + test_accuracy = correct / len(test_loader.dataset) + return test_loss, test_accuracy + +def eval_robust(model, test_loader, perturb_steps, epsilon, step_size, loss_fn, category, random): + model.eval() + test_loss = 0 + correct = 0 + with torch.enable_grad(): + for data, target in test_loader: + data, target = data.cuda(), target.cuda() + x_adv, _ = GA_PGD(model,data,target,epsilon,step_size,perturb_steps,loss_fn,category,rand_init=random) + output = model(x_adv) + test_loss += F.cross_entropy(output, target, size_average=False).item() + pred = output.max(1, keepdim=True)[1] + correct += pred.eq(target.view_as(pred)).sum().item() + test_loss /= len(test_loader.dataset) + test_accuracy = correct / len(test_loader.dataset) + return test_loss, test_accuracy + diff --git a/evaluation/GAIRAT/attack_pgd.py b/evaluation/GAIRAT/attack_pgd.py new file mode 100644 index 0000000..66a36e5 --- /dev/null +++ b/evaluation/GAIRAT/attack_pgd.py @@ -0,0 +1,58 @@ +### PGD implementation + +""" +For PGD plus evaluation +Based on https://github.com/yaircarmon/semisup-adv +""" + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from torch.autograd import Variable +import torch.optim as optim +import logging + + +def pgd(model, + X, + y, + epsilon=8 / 255, + num_steps=20, + step_size=0.01, + random_start=True): + out = model(X) + is_correct_natural = (out.max(1)[1] == y).float().cpu().numpy() + perturbation = torch.zeros_like(X, requires_grad=True) + + if random_start: + perturbation = torch.rand_like(X, requires_grad=True) + perturbation.data = perturbation.data * 2 * epsilon - epsilon + + is_correct_adv = [] + opt = optim.SGD([perturbation], lr=1e-3) # This is just to clear the grad + + for _ in range(num_steps): + opt.zero_grad() + + with torch.enable_grad(): + loss = nn.CrossEntropyLoss()(model(X + perturbation), y) + + loss.backward() + + perturbation.data = ( + perturbation + step_size * perturbation.grad.detach().sign()).clamp( + -epsilon, epsilon) + perturbation.data = torch.min(torch.max(perturbation.detach(), -X), + 1 - X) # clip X+delta to [0,1] + X_pgd = Variable(torch.clamp(X.data + perturbation.data, 0, 1.0), + requires_grad=False) + is_correct_adv.append(np.reshape( + (model(X_pgd).max(1)[1] == y).float().cpu().numpy(), + [-1, 1])) + + is_correct_adv = np.concatenate(is_correct_adv, axis=1) + return is_correct_natural, is_correct_adv + + diff --git a/evaluation/GAIRAT/datasets.py b/evaluation/GAIRAT/datasets.py new file mode 100644 index 0000000..52c6070 --- /dev/null +++ b/evaluation/GAIRAT/datasets.py @@ -0,0 +1,213 @@ +""" +Datasets with unlabeled (or pseudo-labeled) data for PGD plus evaluation +Based on https://github.com/yaircarmon/semisup-adv +""" + +from torchvision.datasets import CIFAR10, SVHN +from torch.utils.data import Sampler, Dataset +import torch +import numpy as np + +import os +import pickle + +import logging + +DATASETS = ['cifar10', 'svhn'] + + +class SemiSupervisedDataset(Dataset): + def __init__(self, + base_dataset='cifar10', + take_amount=None, + take_amount_seed=13, + add_svhn_extra=False, + aux_data_filename=None, + add_aux_labels=False, + aux_take_amount=None, + train=False, + **kwargs): + """A dataset with auxiliary pseudo-labeled data""" + + if base_dataset == 'cifar10': + self.dataset = CIFAR10(train=train, **kwargs) + elif base_dataset == 'svhn': + if train: + self.dataset = SVHN(split='train', **kwargs) + else: + self.dataset = SVHN(split='test', **kwargs) + # because torchvision is annoying + self.dataset.targets = self.dataset.labels + self.targets = list(self.targets) + + if train and add_svhn_extra: + svhn_extra = SVHN(split='extra', **kwargs) + self.data = np.concatenate([self.data, svhn_extra.data]) + self.targets.extend(svhn_extra.labels) + else: + raise ValueError('Dataset %s not supported' % base_dataset) + self.base_dataset = base_dataset + self.train = train + + if self.train: + if take_amount is not None: + rng_state = np.random.get_state() + np.random.seed(take_amount_seed) + take_inds = np.random.choice(len(self.sup_indices), + take_amount, replace=False) + np.random.set_state(rng_state) + + logger = logging.getLogger() + logger.info('Randomly taking only %d/%d examples from training' + ' set, seed=%d, indices=%s', + take_amount, len(self.sup_indices), + take_amount_seed, take_inds) + self.targets = self.targets[take_inds] + self.data = self.data[take_inds] + + self.sup_indices = list(range(len(self.targets))) + self.unsup_indices = [] + + if aux_data_filename is not None: + aux_path = os.path.join(kwargs['root'], aux_data_filename) + aux_path = os.path.expanduser(aux_path) + print("Loading data from %s" % aux_path) + with open(aux_path, 'rb') as f: + aux = pickle.load(f) + aux_data = aux['data'] + aux_targets = aux['extrapolated_targets'] + orig_len = len(self.data) + + if aux_take_amount is not None: + rng_state = np.random.get_state() + np.random.seed(take_amount_seed) + take_inds = np.random.choice(len(aux_data), + aux_take_amount, replace=False) + np.random.set_state(rng_state) + + logger = logging.getLogger() + logger.info( + 'Randomly taking only %d/%d examples from aux data' + ' set, seed=%d, indices=%s', + aux_take_amount, len(aux_data), + take_amount_seed, take_inds) + aux_data = aux_data[take_inds] + aux_targets = aux_targets[take_inds] + + self.data = np.concatenate((self.data, aux_data), axis=0) + + if not add_aux_labels: + self.targets.extend([-1] * len(aux_data)) + else: + self.targets.extend(aux_targets) + # note that we use unsup indices to track the labeled datapoints + # whose labels are "fake" + self.unsup_indices.extend( + range(orig_len, orig_len+len(aux_data))) + + logger = logging.getLogger() + logger.info("Training set") + logger.info("Number of training samples: %d", len(self.targets)) + logger.info("Number of supervised samples: %d", + len(self.sup_indices)) + logger.info("Number of unsup samples: %d", len(self.unsup_indices)) + logger.info("Label (and pseudo-label) histogram: %s", + tuple( + zip(*np.unique(self.targets, return_counts=True)))) + logger.info("Shape of training data: %s", np.shape(self.data)) + + # Test set + else: + self.sup_indices = list(range(len(self.targets))) + self.unsup_indices = [] + + logger = logging.getLogger() + logger.info("Test set") + logger.info("Number of samples: %d", len(self.targets)) + logger.info("Label histogram: %s", + tuple( + zip(*np.unique(self.targets, return_counts=True)))) + logger.info("Shape of data: %s", np.shape(self.data)) + + @property + def data(self): + return self.dataset.data + + @data.setter + def data(self, value): + self.dataset.data = value + + @property + def targets(self): + return self.dataset.targets + + @targets.setter + def targets(self, value): + self.dataset.targets = value + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, item): + self.dataset.labels = self.targets # because torchvision is annoying + return self.dataset[item] + + def __repr__(self): + fmt_str = 'Semisupervised Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Training: {}\n'.format(self.train) + fmt_str += ' Root Location: {}\n'.format(self.dataset.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.dataset.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.dataset.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + +class SemiSupervisedSampler(Sampler): + """Balanced sampling from the labeled and unlabeled data""" + def __init__(self, sup_inds, unsup_inds, batch_size, unsup_fraction=0.5, + num_batches=None): + if unsup_fraction is None or unsup_fraction < 0: + self.sup_inds = sup_inds + unsup_inds + unsup_fraction = 0.0 + else: + self.sup_inds = sup_inds + self.unsup_inds = unsup_inds + + self.batch_size = batch_size + unsup_batch_size = int(batch_size * unsup_fraction) + self.sup_batch_size = batch_size - unsup_batch_size + + if num_batches is not None: + self.num_batches = num_batches + else: + self.num_batches = int( + np.ceil(len(self.sup_inds) / self.sup_batch_size)) + + super().__init__(None) + + def __iter__(self): + batch_counter = 0 + while batch_counter < self.num_batches: + sup_inds_shuffled = [self.sup_inds[i] + for i in torch.randperm(len(self.sup_inds))] + for sup_k in range(0, len(self.sup_inds), self.sup_batch_size): + if batch_counter == self.num_batches: + break + batch = sup_inds_shuffled[sup_k:(sup_k + self.sup_batch_size)] + if self.sup_batch_size < self.batch_size: + batch.extend([self.unsup_inds[i] for i in + torch.randint(high=len(self.unsup_inds), + size=( + self.batch_size - len( + batch),), + dtype=torch.int64)]) + # this shuffle operation is very important, without it + # batch-norm / DataParallel hell ensues + np.random.shuffle(batch) + yield batch + batch_counter += 1 + + def __len__(self): + return self.num_batches diff --git a/evaluation/GAIRAT/earlystop.py b/evaluation/GAIRAT/earlystop.py new file mode 100644 index 0000000..3120a2e --- /dev/null +++ b/evaluation/GAIRAT/earlystop.py @@ -0,0 +1,107 @@ +from models import * +import torch +import numpy as np + +# Geometry-aware early stopped PGD +def GA_earlystop(model, data, target, step_size, epsilon, perturb_steps, tau, type, random, omega): + # Based on code from https://github.com/zjfheart/Friendly-Adversarial-Training + + model.eval() + K = perturb_steps + count = 0 + + output_target = [] + output_adv = [] + output_natural = [] + output_Kappa = [] + + control = torch.zeros(len(target)).cuda() + control += tau + Kappa = torch.zeros(len(data)).cuda() + + if random == False: + iter_adv = data.cuda().detach() + else: + + if type == "fat_for_trades" : + iter_adv = data.detach() + 0.001 * torch.randn(data.shape).cuda().detach() + iter_adv = torch.clamp(iter_adv, 0.0, 1.0) + if type == "fat" or "fat_for_mart": + iter_adv = data.detach() + torch.from_numpy(np.random.uniform(-epsilon, epsilon, data.shape)).float().cuda() + iter_adv = torch.clamp(iter_adv, 0.0, 1.0) + iter_clean_data = data.cuda().detach() + iter_target = target.cuda().detach() + output_iter_clean_data = model(data) + + while K>0: + iter_adv.requires_grad_() + output = model(iter_adv) + pred = output.max(1, keepdim=True)[1] + output_index = [] + iter_index = [] + + for idx in range(len(pred)): + if pred[idx] != target[idx]: + if control[idx]==0: + output_index.append(idx) + else: + control[idx]-=1 + iter_index.append(idx) + else: + # Update Kappa + Kappa[idx] += 1 + iter_index.append(idx) + + if (len(output_index)!=0): + if (len(output_target) == 0): + # incorrect adv data should not keep iterated + output_adv = iter_adv[output_index].reshape(-1, 3, 32, 32).cuda() + output_natural = iter_clean_data[output_index].reshape(-1, 3, 32, 32).cuda() + output_target = iter_target[output_index].reshape(-1).cuda() + output_Kappa = Kappa[output_index].reshape(-1).cuda() + else: + # incorrect adv data should not keep iterated + output_adv = torch.cat((output_adv, iter_adv[output_index].reshape(-1, 3, 32, 32).cuda()), dim=0) + output_natural = torch.cat((output_natural, iter_clean_data[output_index].reshape(-1, 3, 32, 32).cuda()), dim=0) + output_target = torch.cat((output_target, iter_target[output_index].reshape(-1).cuda()), dim=0) + output_Kappa = torch.cat((output_Kappa, Kappa[output_index].reshape(-1).cuda()), dim=0) + + model.zero_grad() + with torch.enable_grad(): + if type == "fat" or type == "fat_for_mart": + loss_adv = nn.CrossEntropyLoss()(output, iter_target) + if type == "fat_for_trades": + criterion_kl = nn.KLDivLoss(size_average=False).cuda() + loss_adv = criterion_kl(F.log_softmax(output, dim=1),F.softmax(output_iter_clean_data, dim=1)) + loss_adv.backward(retain_graph=True) + grad = iter_adv.grad + + if len(iter_index) != 0: + Kappa = Kappa[iter_index] + control = control[iter_index] + iter_adv = iter_adv[iter_index] + iter_clean_data = iter_clean_data[iter_index] + iter_target = iter_target[iter_index] + output_iter_clean_data = output_iter_clean_data[iter_index] + grad = grad[iter_index] + eta = step_size * grad.sign() + iter_adv = iter_adv.detach() + eta + omega * torch.randn(iter_adv.shape).detach().cuda() + iter_adv = torch.min(torch.max(iter_adv, iter_clean_data - epsilon), iter_clean_data + epsilon) + iter_adv = torch.clamp(iter_adv, 0, 1) + count += len(iter_target) + else: + return output_adv, output_target, output_natural, count, output_Kappa + K = K-1 + + if (len(output_target) == 0): + output_target = iter_target.reshape(-1).squeeze().cuda() + output_adv = iter_adv.reshape(-1, 3, 32, 32).cuda() + output_natural = iter_clean_data.reshape(-1, 3, 32, 32).cuda() + output_Kappa = Kappa.reshape(-1).cuda() + else: + output_adv = torch.cat((output_adv, iter_adv.reshape(-1, 3, 32, 32)), dim=0).cuda() + output_target = torch.cat((output_target, iter_target.reshape(-1)), dim=0).squeeze().cuda() + output_natural = torch.cat((output_natural, iter_clean_data.reshape(-1, 3, 32, 32).cuda()),dim=0).cuda() + output_Kappa = torch.cat((output_Kappa, Kappa.reshape(-1)),dim=0).squeeze().cuda() + + return output_adv, output_target, output_natural, count, output_Kappa diff --git a/evaluation/GAIRAT/eval_PGD_plus.py b/evaluation/GAIRAT/eval_PGD_plus.py new file mode 100644 index 0000000..621d506 --- /dev/null +++ b/evaluation/GAIRAT/eval_PGD_plus.py @@ -0,0 +1,246 @@ +""" +Evaluate robustness against specific attack. +Based on code from https://github.com/yaircarmon/semisup-adv +""" + +import os +import json +import numpy as np +import argparse +import logging +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from torch.autograd import Variable +from datasets import SemiSupervisedDataset, DATASETS +from torchvision import transforms +from attack_pgd import pgd +from attack_cw import cw +import torch.backends.cudnn as cudnn +from utils_eval import get_model + + +def eval_adv_test(model, device, test_loader, attack, attack_params, + results_dir, num_eval_batches): + """ + evaluate model by white-box attack + """ + model.eval() + if attack == 'pgd': + restarts_matrices = [] + for restart in range(attack_params['num_restarts']): + is_correct_adv_rows = [] + count = 0 + batch_num = 0 + natural_num_correct = 0 + for data, target in test_loader: + batch_num = batch_num + 1 + if num_eval_batches and batch_num > num_eval_batches: + break + data, target = data.to(device), target.to(device) + count += len(target) + X, y = Variable(data, requires_grad=True), Variable(target) + # is_correct_adv has batch_size*num_iterations dimensions + is_correct_natural, is_correct_adv = pgd( + model, X, y, + epsilon=attack_params['epsilon'], + num_steps=attack_params['num_steps'], + step_size=attack_params['step_size'], + random_start=attack_params['random_start']) + natural_num_correct += is_correct_natural.sum() + is_correct_adv_rows.append(is_correct_adv) + + is_correct_adv_matrix = np.concatenate(is_correct_adv_rows, axis=0) + restarts_matrices.append(is_correct_adv_matrix) + + is_correct_adv_over_restarts = np.stack(restarts_matrices, axis=-1) + num_correct_adv = is_correct_adv_over_restarts.prod( + axis=-1).prod(axis=-1).sum() + + logging.info("Accuracy after %d restarts: %.4g%%" % + (restart + 1, 100 * num_correct_adv / count)) + stats = {'attack': 'pgd', + 'count': count, + 'attack_params': attack_params, + 'natural_accuracy': natural_num_correct / count, + 'is_correct_adv_array': is_correct_adv_over_restarts, + 'robust_accuracy': num_correct_adv / count, + 'restart_num': restart + } + + np.save(os.path.join(results_dir, 'pgd_results.npy'), stats) + + elif attack == 'cw': + all_linf_distances = [] + count = 0 + for data, target in test_loader: + logging.info('Batch: %g', count) + count = count + 1 + if num_eval_batches and count > num_eval_batches: + break + data, target = data.to(device), target.to(device) + X, y = Variable(data, requires_grad=True), Variable(target) + batch_linf_distances = cw(model, X, y, + binary_search_steps=attack_params[ + 'binary_search_steps'], + max_iterations=attack_params[ + 'max_iterations'], + learning_rate=attack_params[ + 'learning_rate'], + initial_const=attack_params[ + 'initial_const'], + tau_decrease_factor=attack_params[ + 'tau_decrease_factor']) + all_linf_distances.append(batch_linf_distances) + stats = {'attack': 'cw', + 'attack_params': attack_params, + 'linf_distances': np.array(all_linf_distances), + } + np.save(os.path.join(results_dir, 'cw_results.npy'), stats) + + else: + raise ValueError('Unknown attack %s' % attack) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='PyTorch CIFAR Attack Evaluation') + parser.add_argument('--dataset', type=str, default='cifar10', + choices=DATASETS, + help='The dataset') + parser.add_argument('--model_path', + help='Model for attack evaluation') + parser.add_argument('--model', '-m', default='wrn-32-10', type=str, + help='Name of the model: wrn-XX-XX, resnet-XX, small-cnn') + parser.add_argument('--output_suffix', default='', type=str, + help='String to add to log filename') + parser.add_argument('--batch_size', type=int, default=128, metavar='N', + help='Input batch size for testing (default: 200)') + parser.add_argument('--no_cuda', action='store_true', default=False, + help='Disables CUDA training') + parser.add_argument('--epsilon', default=0.031, type=float, + help='Attack perturbation magnitude') + parser.add_argument('--attack', default='pgd', type=str, + help='Attack type (CW requires FoolBox)', + choices=('pgd', 'cw')) + parser.add_argument('--num_steps', default=40, type=int, + help='Number of PGD steps') + parser.add_argument('--step_size', default=0.01, type=float, + help='PGD step size') + parser.add_argument('--num_restarts', default=5, type=int, + help='Number of restarts for PGD attack') + parser.add_argument('--no_random_start', dest='random_start', + action='store_false', + help='Disable random PGD initialization') + parser.add_argument('--binary_search_steps', default=5, type=int, + help='Number of binary search steps for CW attack') + parser.add_argument('--max_iterations', default=1000, type=int, + help='Max number of Adam iterations in each CW' + ' optimization') + parser.add_argument('--learning_rate', default=5E-3, type=float, + help='Learning rate for CW attack') + parser.add_argument('--initial_const', default=1E-2, type=float, + help='Initial constant for CW attack') + parser.add_argument('--tau_decrease_factor', default=0.9, type=float, + help='Tau decrease factor for CW attack') + parser.add_argument('--random_seed', default=0, type=int, + help='Random seed for permutation of test instances') + parser.add_argument('--num_eval_batches', default=None, type=int, + help='Number of batches to run evalaution on') + parser.add_argument('--shuffle_testset', action='store_true', default=False, + help='Shuffles the test set') + + args = parser.parse_args() + torch.manual_seed(args.random_seed) + + output_dir, checkpoint_name = os.path.split(args.model_path) + + results_dir = os.path.join('./', args.output_suffix) + if not os.path.isdir(results_dir): + os.mkdir(results_dir) + #epoch = int(re.search('epoch(\d+)', checkpoint_name).group(1)) + epoch = 0 + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s | %(message)s", + handlers=[ + logging.FileHandler(os.path.join(results_dir, + 'attack_epoch%d%s.log' % + (epoch, args.output_suffix))), + logging.StreamHandler() + ]) + logger = logging.getLogger() + + + + logging.info('Attack evaluation') + logging.info('Args: %s' % args) + + # settings + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + dl_kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} + + # set up data loader + transform_test = transforms.Compose([transforms.ToTensor(), ]) + testset = SemiSupervisedDataset(base_dataset=args.dataset, + train=False, root='./data/cifar-10', + download=True, + transform=transform_test) + + if args.shuffle_testset: + np.random.seed(123) + logging.info("Permuting testset") + permutation = np.random.permutation(len(testset)) + testset.data = testset.data[permutation, :] + testset.targets = [testset.targets[i] for i in permutation] + + test_loader = torch.utils.data.DataLoader(testset, + batch_size=args.batch_size, + shuffle=False, **dl_kwargs) + + checkpoint = torch.load(args.model_path) + state_dict = checkpoint.get('state_dict', checkpoint) + num_classes = checkpoint.get('num_classes', 10) + normalize_input = checkpoint.get('normalize_input', False) + model = get_model(args.model, num_classes=num_classes, + normalize_input=normalize_input) + if use_cuda: + model = torch.nn.DataParallel(model).cuda() + cudnn.benchmark = True + if not all([k.startswith('module') for k in state_dict]): + state_dict = {'module.' + k: v for k, v in state_dict.items()} + else: + def strip_data_parallel(s): + if s.startswith('module'): + return s[len('module.'):] + else: + return s + state_dict = {strip_data_parallel(k): v for k, v in state_dict.items()} + model.load_state_dict(state_dict) + + attack_params = { + 'epsilon': args.epsilon, + 'seed': args.random_seed + } + if args.attack == 'pgd': + attack_params.update({ + 'num_restarts': args.num_restarts, + 'step_size': args.step_size, + 'num_steps': args.num_steps, + 'random_start': args.random_start, + }) + elif args.attack == 'cw': + attack_params.update({ + 'binary_search_steps': args.binary_search_steps, + 'max_iterations': args.max_iterations, + 'learning_rate': args.learning_rate, + 'initial_const': args.initial_const, + 'tau_decrease_factor': args.tau_decrease_factor + }) + + logging.info('Running %s' % attack_params) + eval_adv_test(model, device, test_loader, attack=args.attack, + attack_params=attack_params, results_dir=results_dir, + num_eval_batches=args.num_eval_batches) diff --git a/evaluation/GAIRAT/images/GAIRAT_learning_obj.png b/evaluation/GAIRAT/images/GAIRAT_learning_obj.png new file mode 100644 index 0000000..3d60aa1 Binary files /dev/null and b/evaluation/GAIRAT/images/GAIRAT_learning_obj.png differ diff --git a/evaluation/GAIRAT/images/diff_net_error_white.png b/evaluation/GAIRAT/images/diff_net_error_white.png new file mode 100644 index 0000000..8c33a04 Binary files /dev/null and b/evaluation/GAIRAT/images/diff_net_error_white.png differ diff --git a/evaluation/GAIRAT/images/eps_error_white.png b/evaluation/GAIRAT/images/eps_error_white.png new file mode 100644 index 0000000..2ee64a0 Binary files /dev/null and b/evaluation/GAIRAT/images/eps_error_white.png differ diff --git a/evaluation/GAIRAT/images/motivation.png b/evaluation/GAIRAT/images/motivation.png new file mode 100644 index 0000000..c2e64f3 Binary files /dev/null and b/evaluation/GAIRAT/images/motivation.png differ diff --git a/evaluation/GAIRAT/images/pca01.png b/evaluation/GAIRAT/images/pca01.png new file mode 100644 index 0000000..f6b735d Binary files /dev/null and b/evaluation/GAIRAT/images/pca01.png differ diff --git a/evaluation/GAIRAT/models/__init__.py b/evaluation/GAIRAT/models/__init__.py new file mode 100644 index 0000000..3b308d4 --- /dev/null +++ b/evaluation/GAIRAT/models/__init__.py @@ -0,0 +1,5 @@ +from .resnet import * +from .preact_resnet import * +from .wide_resnet import * +from .small_cnn import * +from .wrn_madry import * diff --git a/evaluation/GAIRAT/models/__pycache__/__init__.cpython-36.pyc b/evaluation/GAIRAT/models/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..2fca48d Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/__init__.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/__init__.cpython-37.pyc b/evaluation/GAIRAT/models/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..c724e89 Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/__init__.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/preact_resnet.cpython-36.pyc b/evaluation/GAIRAT/models/__pycache__/preact_resnet.cpython-36.pyc new file mode 100644 index 0000000..7c7e1b1 Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/preact_resnet.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/preact_resnet.cpython-37.pyc b/evaluation/GAIRAT/models/__pycache__/preact_resnet.cpython-37.pyc new file mode 100644 index 0000000..9a32489 Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/preact_resnet.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/resnet.cpython-36.pyc b/evaluation/GAIRAT/models/__pycache__/resnet.cpython-36.pyc new file mode 100644 index 0000000..74a0523 Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/resnet.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/resnet.cpython-37.pyc b/evaluation/GAIRAT/models/__pycache__/resnet.cpython-37.pyc new file mode 100644 index 0000000..3a79b02 Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/resnet.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/small_cnn.cpython-36.pyc b/evaluation/GAIRAT/models/__pycache__/small_cnn.cpython-36.pyc new file mode 100644 index 0000000..3c93a8d Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/small_cnn.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/small_cnn.cpython-37.pyc b/evaluation/GAIRAT/models/__pycache__/small_cnn.cpython-37.pyc new file mode 100644 index 0000000..ee9b5d9 Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/small_cnn.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/wide_resnet.cpython-36.pyc b/evaluation/GAIRAT/models/__pycache__/wide_resnet.cpython-36.pyc new file mode 100644 index 0000000..ec36e90 Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/wide_resnet.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/wide_resnet.cpython-37.pyc b/evaluation/GAIRAT/models/__pycache__/wide_resnet.cpython-37.pyc new file mode 100644 index 0000000..8bfe316 Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/wide_resnet.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/wrn_madry.cpython-36.pyc b/evaluation/GAIRAT/models/__pycache__/wrn_madry.cpython-36.pyc new file mode 100644 index 0000000..234939a Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/wrn_madry.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/models/__pycache__/wrn_madry.cpython-37.pyc b/evaluation/GAIRAT/models/__pycache__/wrn_madry.cpython-37.pyc new file mode 100644 index 0000000..71e1b6e Binary files /dev/null and b/evaluation/GAIRAT/models/__pycache__/wrn_madry.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/models/preact_resnet.py b/evaluation/GAIRAT/models/preact_resnet.py new file mode 100644 index 0000000..aef2116 --- /dev/null +++ b/evaluation/GAIRAT/models/preact_resnet.py @@ -0,0 +1,120 @@ +'''Pre-activation ResNet in PyTorch. + +Reference: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Identity Mappings in Deep Residual Networks. arXiv:1603.05027 +''' +import torch +import torch.nn as nn +import torch.nn.functional as F + +from torch.autograd import Variable + + +class PreActBlock(nn.Module): + '''Pre-activation version of the BasicBlock.''' + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(PreActBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False) + ) + + def forward(self, x): + out = F.relu(self.bn1(x)) + shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x + out = self.conv1(out) + out = self.conv2(F.relu(self.bn2(out))) + out += shortcut + return out + + +class PreActBottleneck(nn.Module): + '''Pre-activation version of the original Bottleneck module.''' + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(PreActBottleneck, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) + + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False) + ) + + def forward(self, x): + out = F.relu(self.bn1(x)) + shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x + out = self.conv1(out) + out = self.conv2(F.relu(self.bn2(out))) + out = self.conv3(F.relu(self.bn3(out))) + out += shortcut + return out + + +class PreActResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(PreActResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512*block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = self.conv1(x) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def PreActResNet18(): + return PreActResNet(PreActBlock, [2,2,2,2]) + +def PreActResNet34(): + return PreActResNet(PreActBlock, [3,4,6,3]) + +def PreActResNet50(): + return PreActResNet(PreActBottleneck, [3,4,6,3]) + +def PreActResNet101(): + return PreActResNet(PreActBottleneck, [3,4,23,3]) + +def PreActResNet152(): + return PreActResNet(PreActBottleneck, [3,8,36,3]) + + +def test(): + net = PreActResNet18() + y = net(Variable(torch.randn(1,3,32,32))) + print(y.size()) + +# test() diff --git a/evaluation/GAIRAT/models/resnet.py b/evaluation/GAIRAT/models/resnet.py new file mode 100644 index 0000000..0dfbfb5 --- /dev/null +++ b/evaluation/GAIRAT/models/resnet.py @@ -0,0 +1,123 @@ +'''ResNet in PyTorch. + +For Pre-activation ResNet, see 'preact_resnet.py'. + +Reference: +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun + Deep Residual Learning for Image Recognition. arXiv:1512.03385 +''' +import torch +import torch.nn as nn +import torch.nn.functional as F + +from torch.autograd import Variable + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion*planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512*block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def ResNet18(num_classes=10): + return ResNet(BasicBlock, [2,2,2,2], num_classes=num_classes) + +def ResNet34(): + return ResNet(BasicBlock, [3,4,6,3]) + +def ResNet50(): + return ResNet(Bottleneck, [3,4,6,3]) + +def ResNet101(): + return ResNet(Bottleneck, [3,4,23,3]) + +def ResNet152(): + return ResNet(Bottleneck, [3,8,36,3]) + + +def test(): + net = ResNet18() + y = net(Variable(torch.randn(1,3,32,32))) + print(y.size()) + print(net) +# test() \ No newline at end of file diff --git a/evaluation/GAIRAT/models/small_cnn.py b/evaluation/GAIRAT/models/small_cnn.py new file mode 100644 index 0000000..c6a500c --- /dev/null +++ b/evaluation/GAIRAT/models/small_cnn.py @@ -0,0 +1,73 @@ +from collections import OrderedDict +import torch.nn as nn +import torch +from torch.autograd import Variable + +class SmallCNN(nn.Module): + def __init__(self, num_classes=10): + super(SmallCNN, self).__init__() + + self.block1_conv1 = nn.Conv2d(3, 64, 3, padding=1) + self.block1_conv2 = nn.Conv2d(64, 64, 3, padding=1) + self.block1_pool1 = nn.MaxPool2d(2, 2) + self.batchnorm1_1 = nn.BatchNorm2d(64) + self.batchnorm1_2 = nn.BatchNorm2d(64) + + self.block2_conv1 = nn.Conv2d(64, 128, 3, padding=1) + self.block2_conv2 = nn.Conv2d(128, 128, 3, padding=1) + self.block2_pool1 = nn.MaxPool2d(2, 2) + self.batchnorm2_1 = nn.BatchNorm2d(128) + self.batchnorm2_2 = nn.BatchNorm2d(128) + + self.block3_conv1 = nn.Conv2d(128, 196, 3, padding=1) + self.block3_conv2 = nn.Conv2d(196, 196, 3, padding=1) + self.block3_pool1 = nn.MaxPool2d(2, 2) + self.batchnorm3_1 = nn.BatchNorm2d(196) + self.batchnorm3_2 = nn.BatchNorm2d(196) + + self.activ = nn.ReLU() + + self.fc1 = nn.Linear(196*4*4,256) + self.fc2 = nn.Linear(256,num_classes) + + def forward(self, x): + #block1 + x = self.block1_conv1(x) + x = self.batchnorm1_1(x) + x = self.activ(x) + x = self.block1_conv2(x) + x = self.batchnorm1_2(x) + x = self.activ(x) + x = self.block1_pool1(x) + + #block2 + x = self.block2_conv1(x) + x = self.batchnorm2_1(x) + x = self.activ(x) + x = self.block2_conv2(x) + x = self.batchnorm2_2(x) + x = self.activ(x) + x = self.block2_pool1(x) + #block3 + x = self.block3_conv1(x) + x = self.batchnorm3_1(x) + x = self.activ(x) + x = self.block3_conv2(x) + x = self.batchnorm3_2(x) + x = self.activ(x) + x = self.block3_pool1(x) + + x = x.view(-1,196*4*4) + x = self.fc1(x) + x = self.activ(x) + x = self.fc2(x) + + return x + +def small_cnn(num_classes=10): + return SmallCNN(num_classes=num_classes) +def test(): + net = small_cnn() + y = net(Variable(torch.randn(1,3,32,32))) + print(y.size()) + print(net) \ No newline at end of file diff --git a/evaluation/GAIRAT/models/wide_resnet.py b/evaluation/GAIRAT/models/wide_resnet.py new file mode 100644 index 0000000..bcad5ae --- /dev/null +++ b/evaluation/GAIRAT/models/wide_resnet.py @@ -0,0 +1,98 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class Wide_ResNet(nn.Module): + def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0): + super(Wide_ResNet, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 4) % 6 == 0) + n = (depth - 4) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + return self.fc(out) +def test(): + net = Wide_ResNet() + y = net(Variable(torch.randn(1, 3, 32, 32))) + #print(y.size()) + print(net) +# test() \ No newline at end of file diff --git a/evaluation/GAIRAT/models/wrn_madry.py b/evaluation/GAIRAT/models/wrn_madry.py new file mode 100644 index 0000000..cbba758 --- /dev/null +++ b/evaluation/GAIRAT/models/wrn_madry.py @@ -0,0 +1,98 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + +class BasicBlock(nn.Module): + def __init__(self, in_planes, out_planes, stride, dropRate=0.0): + super(BasicBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.relu1 = nn.ReLU(inplace=True) + self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(out_planes) + self.relu2 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, + padding=1, bias=False) + self.droprate = dropRate + self.equalInOut = (in_planes == out_planes) + self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, + padding=0, bias=False) or None + + def forward(self, x): + if not self.equalInOut: + x = self.relu1(self.bn1(x)) + else: + out = self.relu1(self.bn1(x)) + out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) + if self.droprate > 0: + out = F.dropout(out, p=self.droprate, training=self.training) + out = self.conv2(out) + return torch.add(x if self.equalInOut else self.convShortcut(x), out) + + +class NetworkBlock(nn.Module): + def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): + super(NetworkBlock, self).__init__() + self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) + + def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): + layers = [] + for i in range(int(nb_layers)): + layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) + return nn.Sequential(*layers) + + def forward(self, x): + return self.layer(x) + + +class Wide_ResNet_Madry(nn.Module): + def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0): + super(Wide_ResNet_Madry, self).__init__() + nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] + assert ((depth - 2) % 6 == 0) + n = (depth - 2) / 6 + block = BasicBlock + # 1st conv before any network block + self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, + padding=1, bias=False) + # 1st block + self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 1st sub-block + # self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) + # 2nd block + self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) + # 3rd block + self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) + # global average pooling and classifier + self.bn1 = nn.BatchNorm2d(nChannels[3]) + self.relu = nn.ReLU(inplace=True) + self.fc = nn.Linear(nChannels[3], num_classes) + self.nChannels = nChannels[3] + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + out = self.conv1(x) + out = self.block1(out) + out = self.block2(out) + out = self.block3(out) + out = self.relu(self.bn1(out)) + out = F.avg_pool2d(out, 8) + out = out.view(-1, self.nChannels) + return self.fc(out) +def test(): + net = Wide_ResNet_Madry() + y = net(Variable(torch.randn(1, 3, 32, 32))) + #print(y.size()) + print(net) +# test() \ No newline at end of file diff --git a/evaluation/GAIRAT/utils/__init__.py b/evaluation/GAIRAT/utils/__init__.py new file mode 100644 index 0000000..bce5561 --- /dev/null +++ b/evaluation/GAIRAT/utils/__init__.py @@ -0,0 +1,10 @@ +"""Useful utils +""" +from .misc import * +from .logger import * +from .eval import * + +# progress bar +import os, sys +sys.path.append(os.path.join(os.path.dirname(__file__), "progress")) +#from progress.bar import Bar as Bar \ No newline at end of file diff --git a/evaluation/GAIRAT/utils/__pycache__/__init__.cpython-36.pyc b/evaluation/GAIRAT/utils/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..8089254 Binary files /dev/null and b/evaluation/GAIRAT/utils/__pycache__/__init__.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/utils/__pycache__/__init__.cpython-37.pyc b/evaluation/GAIRAT/utils/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..ca5c7b9 Binary files /dev/null and b/evaluation/GAIRAT/utils/__pycache__/__init__.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/utils/__pycache__/eval.cpython-36.pyc b/evaluation/GAIRAT/utils/__pycache__/eval.cpython-36.pyc new file mode 100644 index 0000000..ec72ac4 Binary files /dev/null and b/evaluation/GAIRAT/utils/__pycache__/eval.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/utils/__pycache__/eval.cpython-37.pyc b/evaluation/GAIRAT/utils/__pycache__/eval.cpython-37.pyc new file mode 100644 index 0000000..7c08422 Binary files /dev/null and b/evaluation/GAIRAT/utils/__pycache__/eval.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/utils/__pycache__/logger.cpython-36.pyc b/evaluation/GAIRAT/utils/__pycache__/logger.cpython-36.pyc new file mode 100644 index 0000000..d1c7fdb Binary files /dev/null and b/evaluation/GAIRAT/utils/__pycache__/logger.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/utils/__pycache__/logger.cpython-37.pyc b/evaluation/GAIRAT/utils/__pycache__/logger.cpython-37.pyc new file mode 100644 index 0000000..0e2a93a Binary files /dev/null and b/evaluation/GAIRAT/utils/__pycache__/logger.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/utils/__pycache__/misc.cpython-36.pyc b/evaluation/GAIRAT/utils/__pycache__/misc.cpython-36.pyc new file mode 100644 index 0000000..a2e915f Binary files /dev/null and b/evaluation/GAIRAT/utils/__pycache__/misc.cpython-36.pyc differ diff --git a/evaluation/GAIRAT/utils/__pycache__/misc.cpython-37.pyc b/evaluation/GAIRAT/utils/__pycache__/misc.cpython-37.pyc new file mode 100644 index 0000000..26ef7d5 Binary files /dev/null and b/evaluation/GAIRAT/utils/__pycache__/misc.cpython-37.pyc differ diff --git a/evaluation/GAIRAT/utils/eval.py b/evaluation/GAIRAT/utils/eval.py new file mode 100644 index 0000000..5051350 --- /dev/null +++ b/evaluation/GAIRAT/utils/eval.py @@ -0,0 +1,18 @@ +from __future__ import print_function, absolute_import + +__all__ = ['accuracy'] + +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res \ No newline at end of file diff --git a/evaluation/GAIRAT/utils/logger.py b/evaluation/GAIRAT/utils/logger.py new file mode 100644 index 0000000..7eb5c67 --- /dev/null +++ b/evaluation/GAIRAT/utils/logger.py @@ -0,0 +1,127 @@ +# A simple torch style logger +# (C) Wei YANG 2017 +from __future__ import absolute_import +import matplotlib.pyplot as plt +import os +import sys +import numpy as np + +__all__ = ['Logger', 'LoggerMonitor', 'savefig'] + +def savefig(fname, dpi=None): + dpi = 150 if dpi == None else dpi + plt.savefig(fname, dpi=dpi) + +def plot_overlap(logger, names=None): + names = logger.names if names == None else names + numbers = logger.numbers + for _, name in enumerate(names): + x = np.arange(len(numbers[name])) + plt.plot(x, np.asarray(numbers[name])) + return [logger.title + '(' + name + ')' for name in names] + +class Logger(object): + '''Save training process to log file with simple plot function.''' + def __init__(self, fpath, title=None, resume=False): + self.file = None + self.resume = resume + self.title = '' if title == None else title + if fpath is not None: + if resume: + self.file = open(fpath, 'r') + name = self.file.readline() + self.names = name.rstrip().split('\t') + self.numbers = {} + for _, name in enumerate(self.names): + self.numbers[name] = [] + + for numbers in self.file: + numbers = numbers.rstrip().split('\t') + for i in range(0, len(numbers)): + self.numbers[self.names[i]].append(numbers[i]) + self.file.close() + self.file = open(fpath, 'a') + else: + self.file = open(fpath, 'w') + + def set_names(self, names): + if self.resume: + pass + # initialize numbers as empty list + self.numbers = {} + self.names = names + for _, name in enumerate(self.names): + self.file.write(name) + self.file.write('\t') + self.numbers[name] = [] + self.file.write('\n') + self.file.flush() + + + def append(self, numbers): + assert len(self.names) == len(numbers), 'Numbers do not match names' + for index, num in enumerate(numbers): + self.file.write("{0:.6f}".format(num)) + self.file.write('\t') + self.numbers[self.names[index]].append(num) + self.file.write('\n') + self.file.flush() + + def plot(self, names=None): + names = self.names if names == None else names + numbers = self.numbers + for _, name in enumerate(names): + x = np.arange(len(numbers[name])) + plt.plot(x, np.asarray(numbers[name])) + plt.legend([self.title + '(' + name + ')' for name in names]) + plt.grid(True) + + def close(self): + if self.file is not None: + self.file.close() + +class LoggerMonitor(object): + '''Load and visualize multiple logs.''' + def __init__ (self, paths): + '''paths is a distionary with {name:filepath} pair''' + self.loggers = [] + for title, path in paths.items(): + logger = Logger(path, title=title, resume=True) + self.loggers.append(logger) + + def plot(self, names=None): + plt.figure() + plt.subplot(121) + legend_text = [] + for logger in self.loggers: + legend_text += plot_overlap(logger, names) + plt.legend(legend_text, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) + plt.grid(True) + +if __name__ == '__main__': + # # Example + # logger = Logger('test.txt') + # logger.set_names(['Train loss', 'Valid loss','Test loss']) + + # length = 100 + # t = np.arange(length) + # train_loss = np.exp(-t / 10.0) + np.random.rand(length) * 0.1 + # valid_loss = np.exp(-t / 10.0) + np.random.rand(length) * 0.1 + # test_loss = np.exp(-t / 10.0) + np.random.rand(length) * 0.1 + + # for i in range(0, length): + # logger.append([train_loss[i], valid_loss[i], test_loss[i]]) + # logger.plot() + + # Example: logger monitor + paths = { + 'resadvnet20':'/home/wyang/code/pytorch-classification/checkpoint/cifar10/resadvnet20/log.txt', + 'resadvnet32':'/home/wyang/code/pytorch-classification/checkpoint/cifar10/resadvnet32/log.txt', + 'resadvnet44':'/home/wyang/code/pytorch-classification/checkpoint/cifar10/resadvnet44/log.txt', + } + + field = ['Valid Acc.'] + + monitor = LoggerMonitor(paths) + monitor.plot(names=field) + savefig('test.eps') \ No newline at end of file diff --git a/evaluation/GAIRAT/utils/misc.py b/evaluation/GAIRAT/utils/misc.py new file mode 100644 index 0000000..d387f59 --- /dev/null +++ b/evaluation/GAIRAT/utils/misc.py @@ -0,0 +1,76 @@ +'''Some helper functions for PyTorch, including: + - get_mean_and_std: calculate the mean and std value of dataset. + - msr_init: net parameter initialization. + - progress_bar: progress bar mimic xlua.progress. +''' +import errno +import os +import sys +import time +import math + +import torch.nn as nn +import torch.nn.init as init +from torch.autograd import Variable + +__all__ = ['get_mean_and_std', 'init_params', 'mkdir_p', 'AverageMeter'] + + +def get_mean_and_std(dataset): + '''Compute the mean and std value of dataset.''' + dataloader = trainloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=2) + + mean = torch.zeros(3) + std = torch.zeros(3) + print('==> Computing mean and std..') + for inputs, targets in dataloader: + for i in range(3): + mean[i] += inputs[:,i,:,:].mean() + std[i] += inputs[:,i,:,:].std() + mean.div_(len(dataset)) + std.div_(len(dataset)) + return mean, std + +def init_params(net): + '''Init layer parameters.''' + for m in net.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal(m.weight, mode='fan_out') + if m.bias: + init.constant(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant(m.weight, 1) + init.constant(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal(m.weight, std=1e-3) + if m.bias: + init.constant(m.bias, 0) + +def mkdir_p(path): + '''make dir if not exist''' + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + +class AverageMeter(object): + """Computes and stores the average and current value + Imported from https://github.com/pytorch/examples/blob/master/imagenet/main.py#L247-L262 + """ + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count \ No newline at end of file diff --git a/evaluation/GAIRAT/utils_eval.py b/evaluation/GAIRAT/utils_eval.py new file mode 100644 index 0000000..de39a27 --- /dev/null +++ b/evaluation/GAIRAT/utils_eval.py @@ -0,0 +1,47 @@ +""" +Utilities for PGD plus evaluation. +Based on code from https://github.com/yaircarmon/semisup-adv +""" +import os +import numpy as np + +from models.wrn_madry import Wide_ResNet_Madry +from models.resnet import * +from models.small_cnn import SmallCNN + +import torch +from torch.nn import Sequential, Module + + +def get_model(name, num_classes=10, normalize_input=False): + name_parts = name.split('-') + if name_parts[0] == 'wrn': + depth = int(name_parts[1]) + widen = int(name_parts[2]) + model = Wide_ResNet_Madry( + depth=depth, num_classes=num_classes, widen_factor=widen) + + elif name_parts[0] == 'small': + model = SmallCNN() + elif name_parts[0] == 'resnet': + model = ResNet18() + else: + raise ValueError('Could not parse model name %s' % name) + + if normalize_input: + model = Sequential(NormalizeInput(), model) + + return model + + +class NormalizeInput(Module): + def __init__(self, mean=(0.4914, 0.4822, 0.4465), + std=(0.2023, 0.1994, 0.2010)): + super().__init__() + + self.register_buffer('mean', torch.Tensor(mean).reshape(1, -1, 1, 1)) + self.register_buffer('std', torch.Tensor(std).reshape(1, -1, 1, 1)) + + def forward(self, x): + return (x - self.mean) / self.std + diff --git a/evaluation/augmix/LICENSE b/evaluation/augmix/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/evaluation/augmix/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/evaluation/augmix/allconv.py b/evaluation/augmix/allconv.py new file mode 100644 index 0000000..89d8c7b --- /dev/null +++ b/evaluation/augmix/allconv.py @@ -0,0 +1,77 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""AllConv implementation (https://arxiv.org/abs/1412.6806).""" +import math +import torch +import torch.nn as nn + + +class GELU(nn.Module): + + def forward(self, x): + return torch.sigmoid(1.702 * x) * x + + +def make_layers(cfg): + """Create a single layer.""" + layers = [] + in_channels = 3 + for v in cfg: + if v == 'Md': + layers += [nn.MaxPool2d(kernel_size=2, stride=2), nn.Dropout(p=0.5)] + elif v == 'A': + layers += [nn.AvgPool2d(kernel_size=8)] + elif v == 'NIN': + conv2d = nn.Conv2d(in_channels, in_channels, kernel_size=1, padding=1) + layers += [conv2d, nn.BatchNorm2d(in_channels), GELU()] + elif v == 'nopad': + conv2d = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=0) + layers += [conv2d, nn.BatchNorm2d(in_channels), GELU()] + else: + conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) + layers += [conv2d, nn.BatchNorm2d(v), GELU()] + in_channels = v + return nn.Sequential(*layers) + + +class AllConvNet(nn.Module): + """AllConvNet main class.""" + + def __init__(self, num_classes): + super(AllConvNet, self).__init__() + + self.num_classes = num_classes + self.width1, w1 = 96, 96 + self.width2, w2 = 192, 192 + + self.features = make_layers( + [w1, w1, w1, 'Md', w2, w2, w2, 'Md', 'nopad', 'NIN', 'NIN', 'A']) + self.classifier = nn.Linear(self.width2, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) # He initialization + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + x = self.features(x) + x = x.view(x.size(0), -1) + x = self.classifier(x) + return x diff --git a/evaluation/augmix/augment_and_mix.py b/evaluation/augmix/augment_and_mix.py new file mode 100644 index 0000000..c9a1d91 --- /dev/null +++ b/evaluation/augmix/augment_and_mix.py @@ -0,0 +1,70 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Reference implementation of AugMix's data augmentation method in numpy.""" +import augmentations +import numpy as np +from PIL import Image + +# CIFAR-10 constants +MEAN = [0.4914, 0.4822, 0.4465] +STD = [0.2023, 0.1994, 0.2010] + + +def normalize(image): + """Normalize input image channel-wise to zero mean and unit variance.""" + image = image.transpose(2, 0, 1) # Switch to channel-first + mean, std = np.array(MEAN), np.array(STD) + image = (image - mean[:, None, None]) / std[:, None, None] + return image.transpose(1, 2, 0) + + +def apply_op(image, op, severity): + image = np.clip(image * 255., 0, 255).astype(np.uint8) + pil_img = Image.fromarray(image) # Convert to PIL.Image + pil_img = op(pil_img, severity) + return np.asarray(pil_img) / 255. + + +def augment_and_mix(image, severity=3, width=3, depth=-1, alpha=1.): + """Perform AugMix augmentations and compute mixture. + + Args: + image: Raw input image as float32 np.ndarray of shape (h, w, c) + severity: Severity of underlying augmentation operators (between 1 to 10). + width: Width of augmentation chain + depth: Depth of augmentation chain. -1 enables stochastic depth uniformly + from [1, 3] + alpha: Probability coefficient for Beta and Dirichlet distributions. + + Returns: + mixed: Augmented and mixed image. + """ + ws = np.float32( + np.random.dirichlet([alpha] * width)) + m = np.float32(np.random.beta(alpha, alpha)) + + mix = np.zeros_like(image) + for i in range(width): + image_aug = image.copy() + d = depth if depth > 0 else np.random.randint(1, 4) + for _ in range(d): + op = np.random.choice(augmentations.augmentations) + image_aug = apply_op(image_aug, op, severity) + # Preprocessing commutes since all coefficients are convex + mix += ws[i] * normalize(image_aug) + + mixed = (1 - m) * normalize(image) + m * mix + return mixed + diff --git a/evaluation/augmix/augmentations.py b/evaluation/augmix/augmentations.py new file mode 100644 index 0000000..f02ef43 --- /dev/null +++ b/evaluation/augmix/augmentations.py @@ -0,0 +1,152 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Base augmentations operators.""" + +import numpy as np +from PIL import Image, ImageOps, ImageEnhance + +# ImageNet code should change this value +IMAGE_SIZE = 32 + + +def int_parameter(level, maxval): + """Helper function to scale `val` between 0 and maxval . + + Args: + level: Level of the operation that will be between [0, `PARAMETER_MAX`]. + maxval: Maximum value that the operation can have. This will be scaled to + level/PARAMETER_MAX. + + Returns: + An int that results from scaling `maxval` according to `level`. + """ + return int(level * maxval / 10) + + +def float_parameter(level, maxval): + """Helper function to scale `val` between 0 and maxval. + + Args: + level: Level of the operation that will be between [0, `PARAMETER_MAX`]. + maxval: Maximum value that the operation can have. This will be scaled to + level/PARAMETER_MAX. + + Returns: + A float that results from scaling `maxval` according to `level`. + """ + return float(level) * maxval / 10. + + +def sample_level(n): + return np.random.uniform(low=0.1, high=n) + + +def autocontrast(pil_img, _): + return ImageOps.autocontrast(pil_img) + + +def equalize(pil_img, _): + return ImageOps.equalize(pil_img) + + +def posterize(pil_img, level): + level = int_parameter(sample_level(level), 4) + return ImageOps.posterize(pil_img, 4 - level) + + +def rotate(pil_img, level): + degrees = int_parameter(sample_level(level), 30) + if np.random.uniform() > 0.5: + degrees = -degrees + return pil_img.rotate(degrees, resample=Image.BILINEAR) + + +def solarize(pil_img, level): + level = int_parameter(sample_level(level), 256) + return ImageOps.solarize(pil_img, 256 - level) + + +def shear_x(pil_img, level): + level = float_parameter(sample_level(level), 0.3) + if np.random.uniform() > 0.5: + level = -level + return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), + Image.AFFINE, (1, level, 0, 0, 1, 0), + resample=Image.BILINEAR) + + +def shear_y(pil_img, level): + level = float_parameter(sample_level(level), 0.3) + if np.random.uniform() > 0.5: + level = -level + return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), + Image.AFFINE, (1, 0, 0, level, 1, 0), + resample=Image.BILINEAR) + + +def translate_x(pil_img, level): + level = int_parameter(sample_level(level), IMAGE_SIZE / 3) + if np.random.random() > 0.5: + level = -level + return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), + Image.AFFINE, (1, 0, level, 0, 1, 0), + resample=Image.BILINEAR) + + +def translate_y(pil_img, level): + level = int_parameter(sample_level(level), IMAGE_SIZE / 3) + if np.random.random() > 0.5: + level = -level + return pil_img.transform((IMAGE_SIZE, IMAGE_SIZE), + Image.AFFINE, (1, 0, 0, 0, 1, level), + resample=Image.BILINEAR) + + +# operation that overlaps with ImageNet-C's test set +def color(pil_img, level): + level = float_parameter(sample_level(level), 1.8) + 0.1 + return ImageEnhance.Color(pil_img).enhance(level) + + +# operation that overlaps with ImageNet-C's test set +def contrast(pil_img, level): + level = float_parameter(sample_level(level), 1.8) + 0.1 + return ImageEnhance.Contrast(pil_img).enhance(level) + + +# operation that overlaps with ImageNet-C's test set +def brightness(pil_img, level): + level = float_parameter(sample_level(level), 1.8) + 0.1 + return ImageEnhance.Brightness(pil_img).enhance(level) + + +# operation that overlaps with ImageNet-C's test set +def sharpness(pil_img, level): + level = float_parameter(sample_level(level), 1.8) + 0.1 + return ImageEnhance.Sharpness(pil_img).enhance(level) + + +augmentations_none = [] + + +augmentations = [ + autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y, + translate_x, translate_y +] + +augmentations_all = [ + autocontrast, equalize, posterize, rotate, solarize, shear_x, shear_y, + translate_x, translate_y, color, contrast, brightness, sharpness +] diff --git a/evaluation/augmix/cifar.py b/evaluation/augmix/cifar.py new file mode 100644 index 0000000..2fad7af --- /dev/null +++ b/evaluation/augmix/cifar.py @@ -0,0 +1,533 @@ + + +""" +The code is adapted from the google-research's repo: +https://github.com/google-research/augmix/blob/master/cifar.py +""" + + + +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Main script to launch AugMix training on CIFAR-10/100. +Supports WideResNet, AllConv, ResNeXt models on CIFAR-10 and CIFAR-100 as well +as evaluation on CIFAR-10-C and CIFAR-100-C. +Example usage: + `python cifar.py` +""" +from __future__ import print_function + +import argparse +import os +import shutil +import time + +import augmentations +from models.cifar.allconv import AllConvNet +import numpy as np +from third_party.ResNeXt_DenseNet.models.densenet import densenet +from third_party.ResNeXt_DenseNet.models.resnext import resnext29 +from third_party.WideResNet_pytorch.wideresnet import WideResNet + +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F +from torchvision import datasets +from torchvision import transforms + +parser = argparse.ArgumentParser( + description='Trains a CIFAR Classifier', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument( + '--dataset', + type=str, + default='cifar10', + choices=['cifar10', 'cifar100'], + help='Choose between CIFAR-10, CIFAR-100.') +parser.add_argument( + '--model', + '-m', + type=str, + default='wrn', + choices=['wrn', 'allconv', 'densenet', 'resnext'], + help='Choose architecture.') +# Optimization options +parser.add_argument( + '--epochs', '-e', type=int, default=100, help='Number of epochs to train.') +parser.add_argument( + '--learning-rate', + '-lr', + type=float, + default=0.1, + help='Initial learning rate.') +parser.add_argument( + '--batch-size', '-b', type=int, default=128, help='Batch size.') +parser.add_argument('--eval-batch-size', type=int, default=1000) +parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.') +parser.add_argument( + '--decay', + '-wd', + type=float, + default=0.0005, + help='Weight decay (L2 penalty).') +# WRN Architecture options +parser.add_argument( + '--layers', default=40, type=int, help='total number of layers') +parser.add_argument('--widen-factor', default=2, type=int, help='Widen factor') +parser.add_argument( + '--droprate', default=0.0, type=float, help='Dropout probability') +# AugMix options +parser.add_argument( + '--mixture-width', + default=3, + type=int, + help='Number of augmentation chains to mix per augmented example') +parser.add_argument( + '--mixture-depth', + default=-1, + type=int, + help='Depth of augmentation chains. -1 denotes stochastic depth in [1, 3]') +parser.add_argument( + '--aug-severity', + default=3, + type=int, + help='Severity of base augmentation operators') +parser.add_argument( + '--no-jsd', + '-nj', + action='store_true', + help='Turn off JSD consistency loss.') +parser.add_argument( + '--all-ops', + '-all', + action='store_true', + help='Turn on all operations (+brightness,contrast,color,sharpness).') +# Checkpointing options +parser.add_argument( + '--save', + '-s', + type=str, + default='./snapshots', + help='Folder to save checkpoints.') +parser.add_argument( + '--resume', + '-r', + type=str, + default='', + help='Checkpoint path for resume / test.') +parser.add_argument('--evaluate', action='store_true', help='Eval only.') +parser.add_argument( + '--print-freq', + type=int, + default=50, + help='Training loss print frequency (batches).') +# Acceleration +parser.add_argument( + '--num-workers', + type=int, + default=4, + help='Number of pre-fetching threads.') + +######################################################### +parser.add_argument( + '--mix_off', + action='store_true', + help='Turn off augmix') +######################################################### + +args = parser.parse_args() + +CORRUPTIONS = [ + 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', + 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', + 'brightness', 'contrast', 'elastic_transform', 'pixelate', + 'jpeg_compression' +] + + +def get_lr(step, total_steps, lr_max, lr_min): + """Compute learning rate according to cosine annealing schedule.""" + return lr_min + (lr_max - lr_min) * 0.5 * (1 + + np.cos(step / total_steps * np.pi)) + +######################################################################################################################################################################################################################## +def aug(image, preprocess): + """Perform AugMix augmentations and compute mixture. + Args: + image: PIL.Image input image + preprocess: Preprocessing function which should return a torch tensor. + Returns: + mixed: Augmented and mixed image. + """ + aug_list = augmentations.augmentations + if args.all_ops: + aug_list = augmentations.augmentations_all + if args.mix_off: + mixed = preprocess(image) + return mixed + + ws = np.float32(np.random.dirichlet([1] * args.mixture_width)) + m = np.float32(np.random.beta(1, 1)) + + mix = torch.zeros_like(preprocess(image)) + for i in range(args.mixture_width): + image_aug = image.copy() + depth = args.mixture_depth if args.mixture_depth > 0 else np.random.randint( + 1, 4) + for _ in range(depth): + op = np.random.choice(aug_list) + image_aug = op(image_aug, args.aug_severity) + # Preprocessing commutes since all coefficients are convex + mix += ws[i] * preprocess(image_aug) + + mixed = (1 - m) * preprocess(image) + m * mix + return mixed + + +class AugMixDataset(torch.utils.data.Dataset): + """Dataset wrapper to perform AugMix augmentation.""" + + def __init__(self, dataset, preprocess, no_jsd=False): + self.dataset = dataset + self.preprocess = preprocess + self.no_jsd = no_jsd + + def __getitem__(self, i): + x, y = self.dataset[i] + if self.no_jsd: + return aug(x, self.preprocess), y + elif args.mix_off: + return aug(x, self.preprocess), y + else: + im_tuple = (self.preprocess(x), aug(x, self.preprocess), + aug(x, self.preprocess)) + return im_tuple, y + + def __len__(self): + return len(self.dataset) + +######################################################################################################################################################################################################################## + + + +def train(net, train_loader, optimizer, scheduler, location): + """Train for one epoch.""" + net.train() + loss_ema = 0. + for i, (images, targets) in enumerate(train_loader): + optimizer.zero_grad() + + if args.no_jsd: + images = images.cuda() + targets = targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + logits1 = logits.cpu().detach().numpy() + images1 = images.cpu().reshape((len(images.cpu()), np.prod(images.cpu().shape[1:]))) + np.savetxt("./{}/train_corrupted_input.csv".format(location),images1) + np.savetxt("./{}/train_corrupted_labels.csv".format(location),targets.cpu()) + np.savetxt("./{}/train_corrupted_output.csv".format(location),logits1) +#################changed here############### + elif args.mix_off: + images = images.cuda() + targets = targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + logits1 = logits.cpu().detach().numpy() + images1 = images.cpu().reshape((len(images.cpu()), np.prod(images.cpu().shape[1:]))) + np.savetxt("./{}/train_clean_input.csv".format(location),images1) + np.savetxt("./{}/train_clean_labels.csv".format(location),targets.cpu()) + np.savetxt("./{}/train_clean_output.csv".format(location),logits1) +############################################ + else: + images_all = torch.cat(images, 0).cuda() + targets = targets.cuda() + logits_all = net(images_all) + logits_all1 = logits_all.cpu().detach().numpy() + images1 = images_all.cpu().reshape((len(images_all.cpu()), np.prod(images_all.cpu().shape[1:]))) + np.savetxt("./{}/train_corrupted_input.csv".format(location),images1) + np.savetxt("./{}/train_corrupted_labels.csv".format(location),targets.cpu()) + np.savetxt("./{}/train_corrupted_output.csv".format(location),logits_all1) + logits_clean, logits_aug1, logits_aug2 = torch.split( + logits_all, images[0].size(0)) + + # Cross-entropy is only computed on clean images + loss = F.cross_entropy(logits_clean, targets) + + p_clean, p_aug1, p_aug2 = F.softmax( + logits_clean, dim=1), F.softmax( + logits_aug1, dim=1), F.softmax( + logits_aug2, dim=1) + + # Clamp mixture distribution to avoid exploding KL divergence + p_mixture = torch.clamp((p_clean + p_aug1 + p_aug2) / 3., 1e-7, 1).log() + loss += 12 * (F.kl_div(p_mixture, p_clean, reduction='batchmean') + + F.kl_div(p_mixture, p_aug1, reduction='batchmean') + + F.kl_div(p_mixture, p_aug2, reduction='batchmean')) / 3. + + loss.backward() + optimizer.step() + scheduler.step() + loss_ema = loss_ema * 0.9 + float(loss) * 0.1 + if i % args.print_freq == 0: + print('Train Loss {:.3f}'.format(loss_ema)) + + return loss_ema + + +def test(net, test_loader,location): + """Evaluate network on given dataset.""" + net.eval() + total_loss = 0. + total_correct = 0 + with torch.no_grad(): + for images, targets in test_loader: + images, targets = images.cuda(), targets.cuda() + logits = net(images) + images1 = images.cpu().reshape((len(images.cpu()), np.prod(images.cpu().shape[1:]))) + np.savetxt("./{}/test_clean_input.csv".format(location),images1) + np.savetxt("./{}/test_clean_labels.csv".format(location),targets.cpu()) + np.savetxt("./{}/test_clean_output.csv".format(location),logits.cpu()) + loss = F.cross_entropy(logits, targets) + pred = logits.data.max(1)[1] + total_loss += float(loss.data) + total_correct += pred.eq(targets.data).sum().item() + + + return total_loss / len(test_loader.dataset), total_correct / len( + test_loader.dataset) + + +################################################################################# +def my_test(net, test_loader,location, corruption): + """Evaluate network on corrupted dataset.""" + net.eval() + total_loss = 0. + total_correct = 0 + with torch.no_grad(): + for images, targets in test_loader: + images, targets = images.cuda(), targets.cuda() + logits = net(images) + images1 = images.cpu().reshape((len(images.cpu()), np.prod(images.cpu().shape[1:]))) + np.savetxt("./{}/{}_test_corrupted_input.csv".format(location,corruption),images1) + np.savetxt("./{}/{}_test_corrupted_labels.csv".format(location,corruption),targets.cpu()) + np.savetxt("./{}/{}_test_corrupted_output.csv".format(location,corruption),logits.cpu()) + loss = F.cross_entropy(logits, targets) + pred = logits.data.max(1)[1] + total_loss += float(loss.data) + total_correct += pred.eq(targets.data).sum().item() +################################################################################### + + return total_loss / len(test_loader.dataset), total_correct / len( + test_loader.dataset) + + + +def test_c(net, test_data, base_path): + """Evaluate network on given corrupted dataset.""" + corruption_accs = [] + for corruption in CORRUPTIONS: + # Reference to original data is mutated + test_data.data = np.load(base_path + corruption + '.npy') + test_data.targets = torch.LongTensor(np.load(base_path + 'labels.npy')) + + test_loader = torch.utils.data.DataLoader( + test_data, + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + test_loss, test_acc = my_test(net, test_loader, args.save, corruption) + + + + + + corruption_accs.append(test_acc) + print('{}\n\tTest Loss {:.3f} | Test Error {:.3f}'.format( + corruption, test_loss, 100 - 100. * test_acc)) + + return np.mean(corruption_accs) + + +def main(): + torch.manual_seed(1) + np.random.seed(1) + + # Load datasets + train_transform = transforms.Compose( + [transforms.RandomHorizontalFlip(), + transforms.RandomCrop(32, padding=4)]) + + ###################################related to augmixdataset################################ + preprocess = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize([0.5] * 3, [0.5] * 3)]) + ########################################################################################### + + + + test_transform = preprocess + + if args.dataset == 'cifar10': + train_data = datasets.CIFAR10( + './data/cifar', train=True, transform=train_transform, download=True) + test_data = datasets.CIFAR10( + './data/cifar', train=False, transform=test_transform, download=True) + base_c_path = './data/cifar/CIFAR-10-C/' + num_classes = 10 + else: + train_data = datasets.CIFAR100( + './data/cifar', train=True, transform=train_transform, download=True) + test_data = datasets.CIFAR100( + './data/cifar', train=False, transform=test_transform, download=True) + base_c_path = './data/cifar/CIFAR-100-C/' + num_classes = 100 + + + + + + + ###################################change here########################################## + train_data = AugMixDataset(train_data, preprocess, args.no_jsd) + ######################################################################################## + + + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=args.batch_size, + shuffle=True, + num_workers=args.num_workers, + pin_memory=True) + + test_loader = torch.utils.data.DataLoader( + test_data, + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + # Create model + if args.model == 'densenet': + net = densenet(num_classes=num_classes) + elif args.model == 'wrn': + net = WideResNet(args.layers, num_classes, args.widen_factor, args.droprate) + elif args.model == 'allconv': + net = AllConvNet(num_classes) + elif args.model == 'resnext': + net = resnext29(num_classes=num_classes) + + optimizer = torch.optim.SGD( + net.parameters(), + args.learning_rate, + momentum=args.momentum, + weight_decay=args.decay, + nesterov=True) + + # Distribute model across all visible GPUs + net = torch.nn.DataParallel(net).cuda() + cudnn.benchmark = True + + start_epoch = 0 + + if args.resume: + if os.path.isfile(args.resume): + checkpoint = torch.load(args.resume) + start_epoch = checkpoint['epoch'] + 1 + best_acc = checkpoint['best_acc'] + net.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print('Model restored from epoch:', start_epoch) + + if args.evaluate: + # Evaluate clean accuracy first because test_c mutates underlying data + test_loss, test_acc = test(net, test_loader, args.save) + print('Clean\n\tTest Loss {:.3f} | Test Error {:.2f}'.format( + test_loss, 100 - 100. * test_acc)) + + #test_c_acc = test_c(net, test_data, base_c_path) + #print('Mean Corruption Error: {:.3f}'.format(100 - 100. * test_c_acc)) + return + + scheduler = torch.optim.lr_scheduler.LambdaLR( + optimizer, + lr_lambda=lambda step: get_lr( # pylint: disable=g-long-lambda + step, + args.epochs * len(train_loader), + 1, # lr_lambda computes multiplicative factor + 1e-6 / args.learning_rate)) + + if not os.path.exists(args.save): + os.makedirs(args.save) + if not os.path.isdir(args.save): + raise Exception('%s is not a dir' % args.save) + + log_path = os.path.join(args.save, + args.dataset + '_' + args.model + '_training_log.csv') + with open(log_path, 'w') as f: + f.write('epoch,time(s),train_loss,test_loss,test_error(%)\n') + + best_acc = 0 + print('Beginning training from epoch:', start_epoch + 1) + for epoch in range(start_epoch, args.epochs): + begin_time = time.time() + + train_loss_ema = train(net, train_loader, optimizer, scheduler, args.save) + test_loss, test_acc = test(net, test_loader, args.save) + + is_best = test_acc > best_acc + best_acc = max(test_acc, best_acc) + checkpoint = { + 'epoch': epoch, + 'dataset': args.dataset, + 'model': args.model, + 'state_dict': net.state_dict(), + 'best_acc': best_acc, + 'optimizer': optimizer.state_dict(), + } + + save_path = os.path.join(args.save, 'checkpoint.pth.tar') + torch.save(checkpoint, save_path) + if is_best: + shutil.copyfile(save_path, os.path.join(args.save, 'model_best.pth.tar')) + + with open(log_path, 'a') as f: + f.write('%03d,%05d,%0.6f,%0.5f,%0.2f\n' % ( + (epoch + 1), + time.time() - begin_time, + train_loss_ema, + test_loss, + 100 - 100. * test_acc, + )) + + print( + 'Epoch {0:3d} | Time {1:5d} | Train Loss {2:.4f} | Test Loss {3:.3f} |' + ' Test Error {4:.2f}' + .format((epoch + 1), int(time.time() - begin_time), train_loss_ema, + test_loss, 100 - 100. * test_acc)) + + test_c_acc = test_c(net, test_data, base_c_path) + print('Mean Corruption Error: {:.3f}'.format(100 - 100. * test_c_acc)) + + with open(log_path, 'a') as f: + f.write('%03d,%05d,%0.6f,%0.5f,%0.2f\n' % + (args.epochs + 1, 0, 0, 0, 100 - 100 * test_c_acc)) + + +if __name__ == '__main__': + main() + diff --git a/evaluation/augmix/imagenet.py b/evaluation/augmix/imagenet.py new file mode 100644 index 0000000..d88431a --- /dev/null +++ b/evaluation/augmix/imagenet.py @@ -0,0 +1,493 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Main script to launch AugMix training on ImageNet. + +Currently only supports ResNet-50 training. + +Example usage: + `python imagenet.py ` +""" +from __future__ import print_function + +import argparse +import os +import shutil +import time + +import augmentations + +import numpy as np +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F +from torchvision import datasets +from torchvision import models +from torchvision import transforms + +augmentations.IMAGE_SIZE = 224 + +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith('__') and + callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='Trains an ImageNet Classifier') +parser.add_argument( + 'clean_data', metavar='DIR', help='path to clean ImageNet dataset') +parser.add_argument( + 'corrupted_data', metavar='DIR_C', help='path to ImageNet-C dataset') +parser.add_argument( + '--model', + '-m', + default='resnet50', + choices=model_names, + help='model architecture: ' + ' | '.join(model_names) + + ' (default: resnet50)') +# Optimization options +parser.add_argument( + '--epochs', '-e', type=int, default=90, help='Number of epochs to train.') +parser.add_argument( + '--learning-rate', + '-lr', + type=float, + default=0.1, + help='Initial learning rate.') +parser.add_argument( + '--batch-size', '-b', type=int, default=256, help='Batch size.') +parser.add_argument('--eval-batch-size', type=int, default=1000) +parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.') +parser.add_argument( + '--decay', + '-wd', + type=float, + default=0.0001, + help='Weight decay (L2 penalty).') +# AugMix options +parser.add_argument( + '--mixture-width', + default=3, + type=int, + help='Number of augmentation chains to mix per augmented example') +parser.add_argument( + '--mixture-depth', + default=-1, + type=int, + help='Depth of augmentation chains. -1 denotes stochastic depth in [1, 3]') +parser.add_argument( + '--aug-severity', + default=1, + type=int, + help='Severity of base augmentation operators') +parser.add_argument( + '--aug-prob-coeff', + default=1., + type=float, + help='Probability distribution coefficients') +parser.add_argument( + '--no-jsd', + '-nj', + action='store_true', + help='Turn off JSD consistency loss.') +parser.add_argument( + '--all-ops', + '-all', + action='store_true', + help='Turn on all operations (+brightness,contrast,color,sharpness).') +# Checkpointing options +parser.add_argument( + '--save', + '-s', + type=str, + default='./snapshots', + help='Folder to save checkpoints.') +parser.add_argument( + '--resume', + '-r', + type=str, + default='', + help='Checkpoint path for resume / test.') +parser.add_argument('--evaluate', action='store_true', help='Eval only.') +parser.add_argument( + '--print-freq', + type=int, + default=10, + help='Training loss print frequency (batches).') +parser.add_argument( + '--pretrained', + dest='pretrained', + action='store_true', + help='use pre-trained model') +# Acceleration +parser.add_argument( + '--num-workers', + type=int, + default=4, + help='Number of pre-fetching threads.') + +args = parser.parse_args() + +CORRUPTIONS = [ + 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', + 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', + 'brightness', 'contrast', 'elastic_transform', 'pixelate', + 'jpeg_compression' +] + +# Raw AlexNet errors taken from https://github.com/hendrycks/robustness +ALEXNET_ERR = [ + 0.886428, 0.894468, 0.922640, 0.819880, 0.826268, 0.785948, 0.798360, + 0.866816, 0.826572, 0.819324, 0.564592, 0.853204, 0.646056, 0.717840, + 0.606500 +] + + +def adjust_learning_rate(optimizer, epoch): + """Sets the learning rate to the initial LR (linearly scaled to batch size) decayed by 10 every n / 3 epochs.""" + b = args.batch_size / 256. + k = args.epochs // 3 + if epoch < k: + m = 1 + elif epoch < 2 * k: + m = 0.1 + else: + m = 0.01 + lr = args.learning_rate * m * b + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k.""" + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def compute_mce(corruption_accs): + """Compute mCE (mean Corruption Error) normalized by AlexNet performance.""" + mce = 0. + for i in range(len(CORRUPTIONS)): + avg_err = 1 - np.mean(corruption_accs[CORRUPTIONS[i]]) + ce = 100 * avg_err / ALEXNET_ERR[i] + mce += ce / 15 + return mce + + +def aug(image, preprocess): + """Perform AugMix augmentations and compute mixture. + + Args: + image: PIL.Image input image + preprocess: Preprocessing function which should return a torch tensor. + + Returns: + mixed: Augmented and mixed image. + """ + aug_list = augmentations.augmentations + if args.all_ops: + aug_list = augmentations.augmentations_all + + ws = np.float32( + np.random.dirichlet([args.aug_prob_coeff] * args.mixture_width)) + m = np.float32(np.random.beta(args.aug_prob_coeff, args.aug_prob_coeff)) + + mix = torch.zeros_like(preprocess(image)) + for i in range(args.mixture_width): + image_aug = image.copy() + depth = args.mixture_depth if args.mixture_depth > 0 else np.random.randint( + 1, 4) + for _ in range(depth): + op = np.random.choice(aug_list) + image_aug = op(image_aug, args.aug_severity) + # Preprocessing commutes since all coefficients are convex + mix += ws[i] * preprocess(image_aug) + + mixed = (1 - m) * preprocess(image) + m * mix + return mixed + + +class AugMixDataset(torch.utils.data.Dataset): + """Dataset wrapper to perform AugMix augmentation.""" + + def __init__(self, dataset, preprocess, no_jsd=False): + self.dataset = dataset + self.preprocess = preprocess + self.no_jsd = no_jsd + + def __getitem__(self, i): + x, y = self.dataset[i] + if self.no_jsd: + return aug(x, self.preprocess), y + else: + im_tuple = (self.preprocess(x), aug(x, self.preprocess), + aug(x, self.preprocess)) + return im_tuple, y + + def __len__(self): + return len(self.dataset) + + +def train(net, train_loader, optimizer): + """Train for one epoch.""" + net.train() + data_ema = 0. + batch_ema = 0. + loss_ema = 0. + acc1_ema = 0. + acc5_ema = 0. + + end = time.time() + for i, (images, targets) in enumerate(train_loader): + # Compute data loading time + data_time = time.time() - end + optimizer.zero_grad() + + if args.no_jsd: + images = images.cuda() + targets = targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + acc1, acc5 = accuracy(logits, targets, topk=(1, 5)) # pylint: disable=unbalanced-tuple-unpacking + else: + images_all = torch.cat(images, 0).cuda() + targets = targets.cuda() + logits_all = net(images_all) + logits_clean, logits_aug1, logits_aug2 = torch.split( + logits_all, images[0].size(0)) + + # Cross-entropy is only computed on clean images + loss = F.cross_entropy(logits_clean, targets) + + p_clean, p_aug1, p_aug2 = F.softmax( + logits_clean, dim=1), F.softmax( + logits_aug1, dim=1), F.softmax( + logits_aug2, dim=1) + + # Clamp mixture distribution to avoid exploding KL divergence + p_mixture = torch.clamp((p_clean + p_aug1 + p_aug2) / 3., 1e-7, 1).log() + loss += 12 * (F.kl_div(p_mixture, p_clean, reduction='batchmean') + + F.kl_div(p_mixture, p_aug1, reduction='batchmean') + + F.kl_div(p_mixture, p_aug2, reduction='batchmean')) / 3. + acc1, acc5 = accuracy(logits_clean, targets, topk=(1, 5)) # pylint: disable=unbalanced-tuple-unpacking + + loss.backward() + optimizer.step() + + # Compute batch computation time and update moving averages. + batch_time = time.time() - end + end = time.time() + + data_ema = data_ema * 0.1 + float(data_time) * 0.9 + batch_ema = batch_ema * 0.1 + float(batch_time) * 0.9 + loss_ema = loss_ema * 0.1 + float(loss) * 0.9 + acc1_ema = acc1_ema * 0.1 + float(acc1) * 0.9 + acc5_ema = acc5_ema * 0.1 + float(acc5) * 0.9 + + if i % args.print_freq == 0: + print( + 'Batch {}/{}: Data Time {:.3f} | Batch Time {:.3f} | Train Loss {:.3f} | Train Acc1 ' + '{:.3f} | Train Acc5 {:.3f}'.format(i, len(train_loader), data_ema, + batch_ema, loss_ema, acc1_ema, + acc5_ema)) + + return loss_ema, acc1_ema, batch_ema + + +def test(net, test_loader): + """Evaluate network on given dataset.""" + net.eval() + total_loss = 0. + total_correct = 0 + with torch.no_grad(): + for images, targets in test_loader: + images, targets = images.cuda(), targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + pred = logits.data.max(1)[1] + total_loss += float(loss.data) + total_correct += pred.eq(targets.data).sum().item() + + return total_loss / len(test_loader.dataset), total_correct / len( + test_loader.dataset) + + +def test_c(net, test_transform): + """Evaluate network on given corrupted dataset.""" + corruption_accs = {} + for c in CORRUPTIONS: + print(c) + for s in range(1, 6): + valdir = os.path.join(args.corrupted_data, c, str(s)) + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, test_transform), + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + loss, acc1 = test(net, val_loader) + if c in corruption_accs: + corruption_accs[c].append(acc1) + else: + corruption_accs[c] = [acc1] + + print('\ts={}: Test Loss {:.3f} | Test Acc1 {:.3f}'.format( + s, loss, 100. * acc1)) + + return corruption_accs + + +def main(): + torch.manual_seed(1) + np.random.seed(1) + + # Load datasets + mean = [0.485, 0.456, 0.406] + std = [0.229, 0.224, 0.225] + train_transform = transforms.Compose( + [transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip()]) + preprocess = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize(mean, std)]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + preprocess, + ]) + + traindir = os.path.join(args.clean_data, 'train') + valdir = os.path.join(args.clean_data, 'val') + train_dataset = datasets.ImageFolder(traindir, train_transform) + train_dataset = AugMixDataset(train_dataset, preprocess) + train_loader = torch.utils.data.DataLoader( + train_dataset, + batch_size=args.batch_size, + shuffle=True, + num_workers=args.num_workers) + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, test_transform), + batch_size=args.batch_size, + shuffle=False, + num_workers=args.num_workers) + + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.model)) + net = models.__dict__[args.model](pretrained=True) + else: + print("=> creating model '{}'".format(args.model)) + net = models.__dict__[args.model]() + + optimizer = torch.optim.SGD( + net.parameters(), + args.learning_rate, + momentum=args.momentum, + weight_decay=args.decay) + + # Distribute model across all visible GPUs + net = torch.nn.DataParallel(net).cuda() + cudnn.benchmark = True + + start_epoch = 0 + + if args.resume: + if os.path.isfile(args.resume): + checkpoint = torch.load(args.resume) + start_epoch = checkpoint['epoch'] + 1 + best_acc1 = checkpoint['best_acc1'] + net.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print('Model restored from epoch:', start_epoch) + + if args.evaluate: + test_loss, test_acc1 = test(net, val_loader) + print('Clean\n\tTest Loss {:.3f} | Test Acc1 {:.3f}'.format( + test_loss, 100 * test_acc1)) + + corruption_accs = test_c(net, test_transform) + for c in CORRUPTIONS: + print('\t'.join([c] + map(str, corruption_accs[c]))) + + print('mCE (normalized by AlexNet): ', compute_mce(corruption_accs)) + return + + if not os.path.exists(args.save): + os.makedirs(args.save) + if not os.path.isdir(args.save): + raise Exception('%s is not a dir' % args.save) + + log_path = os.path.join(args.save, + 'imagenet_{}_training_log.csv'.format(args.model)) + with open(log_path, 'w') as f: + f.write( + 'epoch,batch_time,train_loss,train_acc1(%),test_loss,test_acc1(%)\n') + + best_acc1 = 0 + print('Beginning training from epoch:', start_epoch + 1) + for epoch in range(start_epoch, args.epochs): + adjust_learning_rate(optimizer, epoch) + + train_loss_ema, train_acc1_ema, batch_ema = train(net, train_loader, + optimizer) + test_loss, test_acc1 = test(net, val_loader) + + is_best = test_acc1 > best_acc1 + best_acc1 = max(test_acc1, best_acc1) + checkpoint = { + 'epoch': epoch, + 'model': args.model, + 'state_dict': net.state_dict(), + 'best_acc1': best_acc1, + 'optimizer': optimizer.state_dict(), + } + + save_path = os.path.join(args.save, 'checkpoint.pth.tar') + torch.save(checkpoint, save_path) + if is_best: + shutil.copyfile(save_path, os.path.join(args.save, 'model_best.pth.tar')) + + with open(log_path, 'a') as f: + f.write('%03d,%0.3f,%0.6f,%0.2f,%0.5f,%0.2f\n' % ( + (epoch + 1), + batch_ema, + train_loss_ema, + 100. * train_acc1_ema, + test_loss, + 100. * test_acc1, + )) + + print( + 'Epoch {:3d} | Train Loss {:.4f} | Test Loss {:.3f} | Test Acc1 ' + '{:.2f}' + .format((epoch + 1), train_loss_ema, test_loss, 100. * test_acc1)) + + corruption_accs = test_c(net, test_transform) + for c in CORRUPTIONS: + print('\t'.join(map(str, [c] + corruption_accs[c]))) + + print('mCE (normalized by AlexNet):', compute_mce(corruption_accs)) + + +if __name__ == '__main__': + main() diff --git a/evaluation/augmix/models/cifar/__pycache__/allconv.cpython-37.pyc b/evaluation/augmix/models/cifar/__pycache__/allconv.cpython-37.pyc new file mode 100644 index 0000000..0173016 Binary files /dev/null and b/evaluation/augmix/models/cifar/__pycache__/allconv.cpython-37.pyc differ diff --git a/evaluation/augmix/models/cifar/allconv.py b/evaluation/augmix/models/cifar/allconv.py new file mode 100644 index 0000000..89d8c7b --- /dev/null +++ b/evaluation/augmix/models/cifar/allconv.py @@ -0,0 +1,77 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""AllConv implementation (https://arxiv.org/abs/1412.6806).""" +import math +import torch +import torch.nn as nn + + +class GELU(nn.Module): + + def forward(self, x): + return torch.sigmoid(1.702 * x) * x + + +def make_layers(cfg): + """Create a single layer.""" + layers = [] + in_channels = 3 + for v in cfg: + if v == 'Md': + layers += [nn.MaxPool2d(kernel_size=2, stride=2), nn.Dropout(p=0.5)] + elif v == 'A': + layers += [nn.AvgPool2d(kernel_size=8)] + elif v == 'NIN': + conv2d = nn.Conv2d(in_channels, in_channels, kernel_size=1, padding=1) + layers += [conv2d, nn.BatchNorm2d(in_channels), GELU()] + elif v == 'nopad': + conv2d = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=0) + layers += [conv2d, nn.BatchNorm2d(in_channels), GELU()] + else: + conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1) + layers += [conv2d, nn.BatchNorm2d(v), GELU()] + in_channels = v + return nn.Sequential(*layers) + + +class AllConvNet(nn.Module): + """AllConvNet main class.""" + + def __init__(self, num_classes): + super(AllConvNet, self).__init__() + + self.num_classes = num_classes + self.width1, w1 = 96, 96 + self.width2, w2 = 192, 192 + + self.features = make_layers( + [w1, w1, w1, 'Md', w2, w2, w2, 'Md', 'nopad', 'NIN', 'NIN', 'A']) + self.classifier = nn.Linear(self.width2, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) # He initialization + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def forward(self, x): + x = self.features(x) + x = x.view(x.size(0), -1) + x = self.classifier(x) + return x diff --git a/evaluation/augmix/original_cifar.py b/evaluation/augmix/original_cifar.py new file mode 100644 index 0000000..9b316de --- /dev/null +++ b/evaluation/augmix/original_cifar.py @@ -0,0 +1,436 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Main script to launch AugMix training on CIFAR-10/100. +Supports WideResNet, AllConv, ResNeXt models on CIFAR-10 and CIFAR-100 as well +as evaluation on CIFAR-10-C and CIFAR-100-C. +Example usage: + `python cifar.py` +""" +from __future__ import print_function + +import argparse +import os +import shutil +import time + +import augmentations +from models.cifar.allconv import AllConvNet +import numpy as np +from third_party.ResNeXt_DenseNet.models.densenet import densenet +from third_party.ResNeXt_DenseNet.models.resnext import resnext29 +from third_party.WideResNet_pytorch.wideresnet import WideResNet + +import torch +import torch.backends.cudnn as cudnn +import torch.nn.functional as F +from torchvision import datasets +from torchvision import transforms + +parser = argparse.ArgumentParser( + description='Trains a CIFAR Classifier', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument( + '--dataset', + type=str, + default='cifar10', + choices=['cifar10', 'cifar100'], + help='Choose between CIFAR-10, CIFAR-100.') +parser.add_argument( + '--model', + '-m', + type=str, + default='wrn', + choices=['wrn', 'allconv', 'densenet', 'resnext'], + help='Choose architecture.') +# Optimization options +parser.add_argument( + '--epochs', '-e', type=int, default=100, help='Number of epochs to train.') +parser.add_argument( + '--learning-rate', + '-lr', + type=float, + default=0.1, + help='Initial learning rate.') +parser.add_argument( + '--batch-size', '-b', type=int, default=128, help='Batch size.') +parser.add_argument('--eval-batch-size', type=int, default=1000) +parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.') +parser.add_argument( + '--decay', + '-wd', + type=float, + default=0.0005, + help='Weight decay (L2 penalty).') +# WRN Architecture options +parser.add_argument( + '--layers', default=40, type=int, help='total number of layers') +parser.add_argument('--widen-factor', default=2, type=int, help='Widen factor') +parser.add_argument( + '--droprate', default=0.0, type=float, help='Dropout probability') +# AugMix options +parser.add_argument( + '--mixture-width', + default=3, + type=int, + help='Number of augmentation chains to mix per augmented example') +parser.add_argument( + '--mixture-depth', + default=-1, + type=int, + help='Depth of augmentation chains. -1 denotes stochastic depth in [1, 3]') +parser.add_argument( + '--aug-severity', + default=3, + type=int, + help='Severity of base augmentation operators') +parser.add_argument( + '--no-jsd', + '-nj', + action='store_true', + help='Turn off JSD consistency loss.') +parser.add_argument( + '--all-ops', + '-all', + action='store_true', + help='Turn on all operations (+brightness,contrast,color,sharpness).') +# Checkpointing options +parser.add_argument( + '--save', + '-s', + type=str, + default='./snapshots', + help='Folder to save checkpoints.') +parser.add_argument( + '--resume', + '-r', + type=str, + default='', + help='Checkpoint path for resume / test.') +parser.add_argument('--evaluate', action='store_true', help='Eval only.') +parser.add_argument( + '--print-freq', + type=int, + default=50, + help='Training loss print frequency (batches).') +# Acceleration +parser.add_argument( + '--num-workers', + type=int, + default=4, + help='Number of pre-fetching threads.') + +args = parser.parse_args() + +CORRUPTIONS = [ + 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', + 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', + 'brightness', 'contrast', 'elastic_transform', 'pixelate', + 'jpeg_compression' +] + + +def get_lr(step, total_steps, lr_max, lr_min): + """Compute learning rate according to cosine annealing schedule.""" + return lr_min + (lr_max - lr_min) * 0.5 * (1 + + np.cos(step / total_steps * np.pi)) + + +def aug(image, preprocess): + """Perform AugMix augmentations and compute mixture. + Args: + image: PIL.Image input image + preprocess: Preprocessing function which should return a torch tensor. + Returns: + mixed: Augmented and mixed image. + """ + aug_list = augmentations.augmentations + if args.all_ops: + aug_list = augmentations.augmentations_all + + ws = np.float32(np.random.dirichlet([1] * args.mixture_width)) + m = np.float32(np.random.beta(1, 1)) + + mix = torch.zeros_like(preprocess(image)) + for i in range(args.mixture_width): + image_aug = image.copy() + depth = args.mixture_depth if args.mixture_depth > 0 else np.random.randint( + 1, 4) + for _ in range(depth): + op = np.random.choice(aug_list) + image_aug = op(image_aug, args.aug_severity) + # Preprocessing commutes since all coefficients are convex + mix += ws[i] * preprocess(image_aug) + + mixed = (1 - m) * preprocess(image) + m * mix + return mixed + + +class AugMixDataset(torch.utils.data.Dataset): + """Dataset wrapper to perform AugMix augmentation.""" + + def __init__(self, dataset, preprocess, no_jsd=False): + self.dataset = dataset + self.preprocess = preprocess + self.no_jsd = no_jsd + + def __getitem__(self, i): + x, y = self.dataset[i] + if self.no_jsd: + return aug(x, self.preprocess), y + else: + im_tuple = (self.preprocess(x), aug(x, self.preprocess), + aug(x, self.preprocess)) + return im_tuple, y + + def __len__(self): + return len(self.dataset) + + +def train(net, train_loader, optimizer, scheduler): + """Train for one epoch.""" + net.train() + loss_ema = 0. + for i, (images, targets) in enumerate(train_loader): + optimizer.zero_grad() + + if args.no_jsd: + images = images.cuda() + targets = targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + else: + images_all = torch.cat(images, 0).cuda() + targets = targets.cuda() + logits_all = net(images_all) + logits_clean, logits_aug1, logits_aug2 = torch.split( + logits_all, images[0].size(0)) + + # Cross-entropy is only computed on clean images + loss = F.cross_entropy(logits_clean, targets) + + p_clean, p_aug1, p_aug2 = F.softmax( + logits_clean, dim=1), F.softmax( + logits_aug1, dim=1), F.softmax( + logits_aug2, dim=1) + + # Clamp mixture distribution to avoid exploding KL divergence + p_mixture = torch.clamp((p_clean + p_aug1 + p_aug2) / 3., 1e-7, 1).log() + loss += 12 * (F.kl_div(p_mixture, p_clean, reduction='batchmean') + + F.kl_div(p_mixture, p_aug1, reduction='batchmean') + + F.kl_div(p_mixture, p_aug2, reduction='batchmean')) / 3. + + loss.backward() + optimizer.step() + scheduler.step() + loss_ema = loss_ema * 0.9 + float(loss) * 0.1 + if i % args.print_freq == 0: + print('Train Loss {:.3f}'.format(loss_ema)) + + return loss_ema + + +def test(net, test_loader): + """Evaluate network on given dataset.""" + net.eval() + total_loss = 0. + total_correct = 0 + with torch.no_grad(): + for images, targets in test_loader: + images, targets = images.cuda(), targets.cuda() + logits = net(images) + loss = F.cross_entropy(logits, targets) + pred = logits.data.max(1)[1] + total_loss += float(loss.data) + total_correct += pred.eq(targets.data).sum().item() + + return total_loss / len(test_loader.dataset), total_correct / len( + test_loader.dataset) + + +def test_c(net, test_data, base_path): + """Evaluate network on given corrupted dataset.""" + corruption_accs = [] + for corruption in CORRUPTIONS: + # Reference to original data is mutated + test_data.data = np.load(base_path + corruption + '.npy') + test_data.targets = torch.LongTensor(np.load(base_path + 'labels.npy')) + + test_loader = torch.utils.data.DataLoader( + test_data, + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + test_loss, test_acc = test(net, test_loader) + corruption_accs.append(test_acc) + print('{}\n\tTest Loss {:.3f} | Test Error {:.3f}'.format( + corruption, test_loss, 100 - 100. * test_acc)) + + return np.mean(corruption_accs) + + +def main(): + torch.manual_seed(1) + np.random.seed(1) + + # Load datasets + train_transform = transforms.Compose( + [transforms.RandomHorizontalFlip(), + transforms.RandomCrop(32, padding=4)]) + preprocess = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize([0.5] * 3, [0.5] * 3)]) + test_transform = preprocess + + if args.dataset == 'cifar10': + train_data = datasets.CIFAR10( + './data/cifar', train=True, transform=train_transform, download=True) + test_data = datasets.CIFAR10( + './data/cifar', train=False, transform=test_transform, download=True) + base_c_path = './data/cifar/CIFAR-10-C/' + num_classes = 10 + else: + train_data = datasets.CIFAR100( + './data/cifar', train=True, transform=train_transform, download=True) + test_data = datasets.CIFAR100( + './data/cifar', train=False, transform=test_transform, download=True) + base_c_path = './data/cifar/CIFAR-100-C/' + num_classes = 100 + + train_data = AugMixDataset(train_data, preprocess, args.no_jsd) + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=args.batch_size, + shuffle=True, + num_workers=args.num_workers, + pin_memory=True) + + test_loader = torch.utils.data.DataLoader( + test_data, + batch_size=args.eval_batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + # Create model + if args.model == 'densenet': + net = densenet(num_classes=num_classes) + elif args.model == 'wrn': + net = WideResNet(args.layers, num_classes, args.widen_factor, args.droprate) + elif args.model == 'allconv': + net = AllConvNet(num_classes) + elif args.model == 'resnext': + net = resnext29(num_classes=num_classes) + + optimizer = torch.optim.SGD( + net.parameters(), + args.learning_rate, + momentum=args.momentum, + weight_decay=args.decay, + nesterov=True) + + # Distribute model across all visible GPUs + net = torch.nn.DataParallel(net).cuda() + cudnn.benchmark = True + + start_epoch = 0 + + if args.resume: + if os.path.isfile(args.resume): + checkpoint = torch.load(args.resume) + start_epoch = checkpoint['epoch'] + 1 + best_acc = checkpoint['best_acc'] + net.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print('Model restored from epoch:', start_epoch) + + if args.evaluate: + # Evaluate clean accuracy first because test_c mutates underlying data + test_loss, test_acc = test(net, test_loader) + print('Clean\n\tTest Loss {:.3f} | Test Error {:.2f}'.format( + test_loss, 100 - 100. * test_acc)) + + test_c_acc = test_c(net, test_data, base_c_path) + print('Mean Corruption Error: {:.3f}'.format(100 - 100. * test_c_acc)) + return + + scheduler = torch.optim.lr_scheduler.LambdaLR( + optimizer, + lr_lambda=lambda step: get_lr( # pylint: disable=g-long-lambda + step, + args.epochs * len(train_loader), + 1, # lr_lambda computes multiplicative factor + 1e-6 / args.learning_rate)) + + if not os.path.exists(args.save): + os.makedirs(args.save) + if not os.path.isdir(args.save): + raise Exception('%s is not a dir' % args.save) + + log_path = os.path.join(args.save, + args.dataset + '_' + args.model + '_training_log.csv') + with open(log_path, 'w') as f: + f.write('epoch,time(s),train_loss,test_loss,test_error(%)\n') + + best_acc = 0 + print('Beginning training from epoch:', start_epoch + 1) + for epoch in range(start_epoch, args.epochs): + begin_time = time.time() + + train_loss_ema = train(net, train_loader, optimizer, scheduler) + test_loss, test_acc = test(net, test_loader) + + is_best = test_acc > best_acc + best_acc = max(test_acc, best_acc) + checkpoint = { + 'epoch': epoch, + 'dataset': args.dataset, + 'model': args.model, + 'state_dict': net.state_dict(), + 'best_acc': best_acc, + 'optimizer': optimizer.state_dict(), + } + + save_path = os.path.join(args.save, 'checkpoint.pth.tar') + torch.save(checkpoint, save_path) + if is_best: + shutil.copyfile(save_path, os.path.join(args.save, 'model_best.pth.tar')) + + with open(log_path, 'a') as f: + f.write('%03d,%05d,%0.6f,%0.5f,%0.2f\n' % ( + (epoch + 1), + time.time() - begin_time, + train_loss_ema, + test_loss, + 100 - 100. * test_acc, + )) + + print( + 'Epoch {0:3d} | Time {1:5d} | Train Loss {2:.4f} | Test Loss {3:.3f} |' + ' Test Error {4:.2f}' + .format((epoch + 1), int(time.time() - begin_time), train_loss_ema, + test_loss, 100 - 100. * test_acc)) + + test_c_acc = test_c(net, test_data, base_c_path) + print('Mean Corruption Error: {:.3f}'.format(100 - 100. * test_c_acc)) + + with open(log_path, 'a') as f: + f.write('%03d,%05d,%0.6f,%0.5f,%0.2f\n' % + (args.epochs + 1, 0, 0, 0, 100 - 100 * test_c_acc)) + + +if __name__ == '__main__': + main() diff --git a/evaluation/augmix/requirements.txt b/evaluation/augmix/requirements.txt new file mode 100644 index 0000000..f36c578 --- /dev/null +++ b/evaluation/augmix/requirements.txt @@ -0,0 +1,4 @@ +numpy>=1.15.0 +Pillow>=6.1.0 +torch==1.2.0 +torchvision==0.2.2 diff --git a/evaluation/augmix/third_party/ResNeXt_DenseNet/LICENSE b/evaluation/augmix/third_party/ResNeXt_DenseNet/LICENSE new file mode 100644 index 0000000..d05cea3 --- /dev/null +++ b/evaluation/augmix/third_party/ResNeXt_DenseNet/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Xuanyi Dong + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/evaluation/augmix/third_party/ResNeXt_DenseNet/METADATA b/evaluation/augmix/third_party/ResNeXt_DenseNet/METADATA new file mode 100644 index 0000000..43dc477 --- /dev/null +++ b/evaluation/augmix/third_party/ResNeXt_DenseNet/METADATA @@ -0,0 +1,12 @@ +name: "ResNeXt-DenseNet" +description: "PyTorch implementations of ResNeXt and DenseNet." + +third_party { + url { + type: GIT + value: "https://github.com/D-X-Y/ResNeXt-DenseNet" + } + version: "0de9a8c8fd095b37eb60945f8dafefdbfe1cef6b" + last_upgrade_date { year: 2019 month: 12 day: 4 } + license_type: PERMISSIVE +} diff --git a/evaluation/augmix/third_party/ResNeXt_DenseNet/models/densenet.py b/evaluation/augmix/third_party/ResNeXt_DenseNet/models/densenet.py new file mode 100644 index 0000000..2c52a2b --- /dev/null +++ b/evaluation/augmix/third_party/ResNeXt_DenseNet/models/densenet.py @@ -0,0 +1,125 @@ +"""DenseNet implementation (https://arxiv.org/abs/1608.06993).""" +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Bottleneck(nn.Module): + """Bottleneck block for DenseNet.""" + + def __init__(self, n_channels, growth_rate): + super(Bottleneck, self).__init__() + inter_channels = 4 * growth_rate + self.bn1 = nn.BatchNorm2d(n_channels) + self.conv1 = nn.Conv2d( + n_channels, inter_channels, kernel_size=1, bias=False) + self.bn2 = nn.BatchNorm2d(inter_channels) + self.conv2 = nn.Conv2d( + inter_channels, growth_rate, kernel_size=3, padding=1, bias=False) + + def forward(self, x): + out = self.conv1(F.relu(self.bn1(x))) + out = self.conv2(F.relu(self.bn2(out))) + out = torch.cat((x, out), 1) + return out + + +class SingleLayer(nn.Module): + """Layer container for blocks.""" + + def __init__(self, n_channels, growth_rate): + super(SingleLayer, self).__init__() + self.bn1 = nn.BatchNorm2d(n_channels) + self.conv1 = nn.Conv2d( + n_channels, growth_rate, kernel_size=3, padding=1, bias=False) + + def forward(self, x): + out = self.conv1(F.relu(self.bn1(x))) + out = torch.cat((x, out), 1) + return out + + +class Transition(nn.Module): + """Transition block.""" + + def __init__(self, n_channels, n_out_channels): + super(Transition, self).__init__() + self.bn1 = nn.BatchNorm2d(n_channels) + self.conv1 = nn.Conv2d( + n_channels, n_out_channels, kernel_size=1, bias=False) + + def forward(self, x): + out = self.conv1(F.relu(self.bn1(x))) + out = F.avg_pool2d(out, 2) + return out + + +class DenseNet(nn.Module): + """DenseNet main class.""" + + def __init__(self, growth_rate, depth, reduction, n_classes, bottleneck): + super(DenseNet, self).__init__() + + if bottleneck: + n_dense_blocks = int((depth - 4) / 6) + else: + n_dense_blocks = int((depth - 4) / 3) + + n_channels = 2 * growth_rate + self.conv1 = nn.Conv2d(3, n_channels, kernel_size=3, padding=1, bias=False) + + self.dense1 = self._make_dense(n_channels, growth_rate, n_dense_blocks, + bottleneck) + n_channels += n_dense_blocks * growth_rate + n_out_channels = int(math.floor(n_channels * reduction)) + self.trans1 = Transition(n_channels, n_out_channels) + + n_channels = n_out_channels + self.dense2 = self._make_dense(n_channels, growth_rate, n_dense_blocks, + bottleneck) + n_channels += n_dense_blocks * growth_rate + n_out_channels = int(math.floor(n_channels * reduction)) + self.trans2 = Transition(n_channels, n_out_channels) + + n_channels = n_out_channels + self.dense3 = self._make_dense(n_channels, growth_rate, n_dense_blocks, + bottleneck) + n_channels += n_dense_blocks * growth_rate + + self.bn1 = nn.BatchNorm2d(n_channels) + self.fc = nn.Linear(n_channels, n_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.bias.data.zero_() + + def _make_dense(self, n_channels, growth_rate, n_dense_blocks, bottleneck): + layers = [] + for _ in range(int(n_dense_blocks)): + if bottleneck: + layers.append(Bottleneck(n_channels, growth_rate)) + else: + layers.append(SingleLayer(n_channels, growth_rate)) + n_channels += growth_rate + return nn.Sequential(*layers) + + def forward(self, x): + out = self.conv1(x) + out = self.trans1(self.dense1(out)) + out = self.trans2(self.dense2(out)) + out = self.dense3(out) + out = torch.squeeze(F.avg_pool2d(F.relu(self.bn1(out)), 8)) + out = self.fc(out) + return out + + +def densenet(growth_rate=12, depth=40, num_classes=10): + model = DenseNet(growth_rate, depth, 1., num_classes, False) + return model diff --git a/evaluation/augmix/third_party/ResNeXt_DenseNet/models/resnext.py b/evaluation/augmix/third_party/ResNeXt_DenseNet/models/resnext.py new file mode 100644 index 0000000..2863a7a --- /dev/null +++ b/evaluation/augmix/third_party/ResNeXt_DenseNet/models/resnext.py @@ -0,0 +1,144 @@ +"""ResNeXt implementation (https://arxiv.org/abs/1611.05431).""" +import math +import torch.nn as nn +from torch.nn import init +import torch.nn.functional as F + + +class ResNeXtBottleneck(nn.Module): + """ResNeXt Bottleneck Block type C (https://github.com/facebookresearch/ResNeXt/blob/master/models/resnext.lua).""" + expansion = 4 + + def __init__(self, + inplanes, + planes, + cardinality, + base_width, + stride=1, + downsample=None): + super(ResNeXtBottleneck, self).__init__() + + dim = int(math.floor(planes * (base_width / 64.0))) + + self.conv_reduce = nn.Conv2d( + inplanes, + dim * cardinality, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn_reduce = nn.BatchNorm2d(dim * cardinality) + + self.conv_conv = nn.Conv2d( + dim * cardinality, + dim * cardinality, + kernel_size=3, + stride=stride, + padding=1, + groups=cardinality, + bias=False) + self.bn = nn.BatchNorm2d(dim * cardinality) + + self.conv_expand = nn.Conv2d( + dim * cardinality, + planes * 4, + kernel_size=1, + stride=1, + padding=0, + bias=False) + self.bn_expand = nn.BatchNorm2d(planes * 4) + + self.downsample = downsample + + def forward(self, x): + residual = x + + bottleneck = self.conv_reduce(x) + bottleneck = F.relu(self.bn_reduce(bottleneck), inplace=True) + + bottleneck = self.conv_conv(bottleneck) + bottleneck = F.relu(self.bn(bottleneck), inplace=True) + + bottleneck = self.conv_expand(bottleneck) + bottleneck = self.bn_expand(bottleneck) + + if self.downsample is not None: + residual = self.downsample(x) + + return F.relu(residual + bottleneck, inplace=True) + + +class CifarResNeXt(nn.Module): + """ResNext optimized for the Cifar dataset, as specified in https://arxiv.org/pdf/1611.05431.pdf.""" + + def __init__(self, block, depth, cardinality, base_width, num_classes): + super(CifarResNeXt, self).__init__() + + # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + assert (depth - 2) % 9 == 0, 'depth should be one of 29, 38, 47, 56, 101' + layer_blocks = (depth - 2) // 9 + + self.cardinality = cardinality + self.base_width = base_width + self.num_classes = num_classes + + self.conv_1_3x3 = nn.Conv2d(3, 64, 3, 1, 1, bias=False) + self.bn_1 = nn.BatchNorm2d(64) + + self.inplanes = 64 + self.stage_1 = self._make_layer(block, 64, layer_blocks, 1) + self.stage_2 = self._make_layer(block, 128, layer_blocks, 2) + self.stage_3 = self._make_layer(block, 256, layer_blocks, 2) + self.avgpool = nn.AvgPool2d(8) + self.classifier = nn.Linear(256 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + init.kaiming_normal(m.weight) + m.bias.data.zero_() + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, self.cardinality, self.base_width, stride, + downsample)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block(self.inplanes, planes, self.cardinality, self.base_width)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv_1_3x3(x) + x = F.relu(self.bn_1(x), inplace=True) + x = self.stage_1(x) + x = self.stage_2(x) + x = self.stage_3(x) + x = self.avgpool(x) + x = x.view(x.size(0), -1) + return self.classifier(x) + + +def resnext29(num_classes=10, cardinality=4, base_width=32): + model = CifarResNeXt(ResNeXtBottleneck, 29, cardinality, base_width, + num_classes) + return model diff --git a/evaluation/mnist/LICENSE b/evaluation/mnist/LICENSE new file mode 100644 index 0000000..c1b92dd --- /dev/null +++ b/evaluation/mnist/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Aleksander Madry, Aleksandar Makelov, Ludwig Schmidt, Dimitris Tsipras, and Adrian Vladu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/evaluation/mnist/config.json b/evaluation/mnist/config.json new file mode 100644 index 0000000..af5efb1 --- /dev/null +++ b/evaluation/mnist/config.json @@ -0,0 +1,25 @@ +{ + "_comment": "===== MODEL CONFIGURATION =====", + "model_dir": "models/a_very_robust_model", + + "_comment": "===== TRAINING CONFIGURATION =====", + "random_seed": 4557077, + "max_num_training_steps": 100000, + "num_output_steps": 100, + "num_summary_steps": 100, + "num_checkpoint_steps": 300, + "training_batch_size": 50, + + "_comment": "===== EVAL CONFIGURATION =====", + "num_eval_examples": 10000, + "eval_batch_size": 200, + "eval_on_cpu": true, + + "_comment": "=====ADVERSARIAL EXAMPLES CONFIGURATION=====", + "epsilon": 0.3, + "k": 40, + "a": 0.01, + "random_start": true, + "loss_func": "xent", + "store_adv_path": "attack.npy" +} diff --git a/evaluation/mnist/eval.py b/evaluation/mnist/eval.py new file mode 100644 index 0000000..20b8e5e --- /dev/null +++ b/evaluation/mnist/eval.py @@ -0,0 +1,158 @@ +""" +Infinite evaluation loop going through the checkpoints in the model directory +as they appear and evaluating them. Accuracy and average loss are printed and +added as tensorboard summaries. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import json +import math +import os +import sys +import time + +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data + +from model import Model +from pgd_attack import LinfPGDAttack + +# Global constants +with open('config.json') as config_file: + config = json.load(config_file) +num_eval_examples = config['num_eval_examples'] +eval_batch_size = config['eval_batch_size'] +eval_on_cpu = config['eval_on_cpu'] + +model_dir = config['model_dir'] + +# Set upd the data, hyperparameters, and the model +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) + +if eval_on_cpu: + with tf.device("/cpu:0"): + model = Model() + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) +else: + model = Model() + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + +global_step = tf.contrib.framework.get_or_create_global_step() + +# Setting up the Tensorboard and checkpoint outputs +if not os.path.exists(model_dir): + os.makedirs(model_dir) +eval_dir = os.path.join(model_dir, 'eval') +if not os.path.exists(eval_dir): + os.makedirs(eval_dir) + +last_checkpoint_filename = '' +already_seen_state = False + +saver = tf.train.Saver() +summary_writer = tf.summary.FileWriter(eval_dir) + +# A function for evaluating a single checkpoint +def evaluate_checkpoint(filename): + with tf.Session() as sess: + # Restore the checkpoint + saver.restore(sess, filename) + + # Iterate over the samples batch-by-batch + num_batches = int(math.ceil(num_eval_examples / eval_batch_size)) + total_xent_nat = 0. + total_xent_adv = 0. + total_corr_nat = 0 + total_corr_adv = 0 + + for ibatch in range(num_batches): + bstart = ibatch * eval_batch_size + bend = min(bstart + eval_batch_size, num_eval_examples) + + x_batch = mnist.test.images[bstart:bend, :] + y_batch = mnist.test.labels[bstart:bend] + + dict_nat = {model.x_input: x_batch, + model.y_input: y_batch} + + x_batch_adv = attack.perturb(x_batch, y_batch, sess) + + dict_adv = {model.x_input: x_batch_adv, + model.y_input: y_batch} + + cur_corr_nat, cur_xent_nat = sess.run( + [model.num_correct,model.xent], + feed_dict = dict_nat) + cur_corr_adv, cur_xent_adv = sess.run( + [model.num_correct,model.xent], + feed_dict = dict_adv) + + total_xent_nat += cur_xent_nat + total_xent_adv += cur_xent_adv + total_corr_nat += cur_corr_nat + total_corr_adv += cur_corr_adv + + avg_xent_nat = total_xent_nat / num_eval_examples + avg_xent_adv = total_xent_adv / num_eval_examples + acc_nat = total_corr_nat / num_eval_examples + acc_adv = total_corr_adv / num_eval_examples + + summary = tf.Summary(value=[ + tf.Summary.Value(tag='xent adv eval', simple_value= avg_xent_adv), + tf.Summary.Value(tag='xent adv', simple_value= avg_xent_adv), + tf.Summary.Value(tag='xent nat', simple_value= avg_xent_nat), + tf.Summary.Value(tag='accuracy adv eval', simple_value= acc_adv), + tf.Summary.Value(tag='accuracy adv', simple_value= acc_adv), + tf.Summary.Value(tag='accuracy nat', simple_value= acc_nat)]) + summary_writer.add_summary(summary, global_step.eval(sess)) + + print('natural: {:.2f}%'.format(100 * acc_nat)) + print('adversarial: {:.2f}%'.format(100 * acc_adv)) + print('avg nat loss: {:.4f}'.format(avg_xent_nat)) + print('avg adv loss: {:.4f}'.format(avg_xent_adv)) + +# Infinite eval loop +while True: + cur_checkpoint = tf.train.latest_checkpoint(model_dir) + + # Case 1: No checkpoint yet + if cur_checkpoint is None: + if not already_seen_state: + print('No checkpoint yet, waiting ...', end='') + already_seen_state = True + else: + print('.', end='') + sys.stdout.flush() + time.sleep(10) + # Case 2: Previously unseen checkpoint + elif cur_checkpoint != last_checkpoint_filename: + print('\nCheckpoint {}, evaluating ... ({})'.format(cur_checkpoint, + datetime.now())) + sys.stdout.flush() + last_checkpoint_filename = cur_checkpoint + already_seen_state = False + evaluate_checkpoint(cur_checkpoint) + # Case 3: Previously evaluated checkpoint + else: + if not already_seen_state: + print('Waiting for the next checkpoint ... ({}) '.format( + datetime.now()), + end='') + already_seen_state = True + else: + print('.', end='') + sys.stdout.flush() + time.sleep(10) diff --git a/evaluation/mnist/fetch_model.py b/evaluation/mnist/fetch_model.py new file mode 100644 index 0000000..ab08e38 --- /dev/null +++ b/evaluation/mnist/fetch_model.py @@ -0,0 +1,46 @@ +"""Downloads a model, computes its SHA256 hash and unzips it + at the proper location.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import zipfile +import hashlib + +if len(sys.argv) != 2 or sys.argv[1] not in ['natural', + 'adv_trained', + 'secret']: + print('Usage: python fetch_model.py [natural, adv_trained, secret]') + sys.exit(1) + +if sys.argv[1] == 'natural': + url = 'https://github.com/MadryLab/mnist_challenge_models/raw/master/natural.zip' +elif sys.argv[1] == 'secret': + url = 'https://github.com/MadryLab/mnist_challenge_models/raw/master/secret.zip' +else: # fetch adv_trained model + url = 'https://github.com/MadryLab/mnist_challenge_models/raw/master/adv_trained.zip' + +fname = url.split('/')[-1] # get the name of the file + +# model download +print('Downloading models') +if sys.version_info >= (3,): + import urllib.request + urllib.request.urlretrieve(url, fname) +else: + import urllib + urllib.urlretrieve(url, fname) + +# computing model hash +sha256 = hashlib.sha256() +with open(fname, 'rb') as f: + data = f.read() + sha256.update(data) +print('SHA256 hash: {}'.format(sha256.hexdigest())) + +# extracting model +print('Extracting model') +with zipfile.ZipFile(fname, 'r') as model_zip: + model_zip.extractall() + print('Extracted model in {}'.format(model_zip.namelist()[0])) diff --git a/evaluation/mnist/gbt_eval.py b/evaluation/mnist/gbt_eval.py new file mode 100644 index 0000000..fd82e36 --- /dev/null +++ b/evaluation/mnist/gbt_eval.py @@ -0,0 +1,161 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import json +import math +import os +import sys +import time + +import numpy as np +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data +from pgd_attack import LinfPGDAttack + +from tensorflow.python import pywrap_tensorflow + +from model import Model +# from pgd_attack import LinfPGDAttack + +print("acc"+"b") + +with open('config.json') as config_file: + config = json.load(config_file) +num_eval_examples = config['num_eval_examples'] +eval_batch_size = config['eval_batch_size'] +eval_on_cpu = config['eval_on_cpu'] + +model = Model() +eps = config['epsilon'] +model_dir = config['model_dir'] + +# ============= set output file path ============== +path_folder = "./mnist_data_"+str(eps)+"_dl/"; + +saver = tf.train.Saver() + +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) +use_set = mnist.train + +if eval_on_cpu: + with tf.device("/cpu:0"): + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) +else: + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + + +global_step = tf.contrib.framework.get_or_create_global_step() + + + +with tf.Session() as sess: + print(sess.list_devices()) + + # model_dir = 'models/mnist-'+str(eps) + model_dir = config['model_dir'] + + # for eps in np.arange(0.4, 0.6, 0.1): + cur_checkpoint = tf.train.latest_checkpoint(model_dir) + # cur_checkpoint = 'models/mnist-0.3-dl/checkpoint-99900' + print('restoring '+ cur_checkpoint) + saver = tf.train.Saver() + saver.restore(sess, cur_checkpoint) + + + num_batches = int(math.ceil(use_set.labels.shape[0] / eval_batch_size)) + if not os.path.exists(path_folder): + os.mkdir(path_folder) + + batch_index = 0 + op_data = np.zeros((1,10)) + for ibatch in range(num_batches): + bstart = ibatch * eval_batch_size + bend = min(bstart + eval_batch_size, use_set.labels.shape[0]) + + x_batch = use_set.images[bstart:bend, :] + y_batch = use_set.labels[bstart:bend] + + dict_nat = {model.x_input: x_batch, + model.y_input: y_batch} + # x_batch_adv = attack.perturb(x_batch, y_batch, sess) + # dict_adv = {model.x_input: x_batch_adv, + # model.y_input: y_batch} + + # h_conv1, h_conv2, h_fc1, pred_softmax = sess.run([model.h_conv1, model.h_conv2, model.h_fc1, model.pre_softmax], + # feed_dict = dict_nat) + h_fc1, y_xent, xent, pred_softmax = sess.run([model.h_fc1, model.y_xent, model.xent, model.pre_softmax], + feed_dict = dict_nat) + + # shp_conv1 = h_conv1.shape + # shp_conv2 = h_conv2.shape + shp_fc1 = h_fc1.shape + + print(y_xent.shape) + print(xent.shape) + + op_data = np.vstack((op_data, pred_softmax)) + + # pred_softmax = sess.run(model.pre_softmax, + # feed_dict = dict_nat) + + # print('conv1') + # print(np.reshape(h_conv1, (eval_batch_size, shp_conv1[1]*shp_conv1[2]*shp_conv1[3])).shape) + + + # fp_out = open("I:/process/mltest/mnist_data_0.3/gbt_output_conv1.csv","ab"); + # np.savetxt(fp_out, np.reshape(h_conv1, (eval_batch_size, shp_conv1[1]*shp_conv1[2]*shp_conv1[3]))) + # fp_out.close() + + # fp_out = open("I:/process/mltest/mnist_data_0.3/gbt_output_conv2.csv","ab"); + # np.savetxt(fp_out, np.reshape(h_conv2, (eval_batch_size, shp_conv2[1]*shp_conv2[2]*shp_conv2[3]))) + # fp_out.close() + + # fp_out = open(path_folder+"/gbt_output_fc1.csv","ab"); + # np.savetxt(fp_out, h_fc1, fmt="%.3f"); + # fp_out.close(); + + fp_out = open(path_folder+"gbt_output_out.csv","ab"); + np.savetxt(fp_out, pred_softmax, fmt="%.4f") + fp_out.close() + + fp_in = open(path_folder+"gbt_input.csv", "ab") + np.savetxt(fp_in, x_batch, fmt="%.4f") + fp_in.close() + + fp_label = open(path_folder+"gbt_label.csv", "ab") + np.savetxt(fp_label, y_batch, fmt="%d") + fp_label.close() + + # break; + # batch_index=batch_index+1 + # print(batch_index) + # if batch_index>=2: + # break + print(ibatch) + # op_data = op_data[1:] + # print(np.sum(np.argmax(op_data, axis=1) == mnist.test.labels)/mnist.test.labels.shape[0]) + # print(mnist.test.images[0]) + + + # =========for debug=========== + # ckpt = tf.train.get_checkpoint_state('models/mnist-0') + # print(ckpt.model_checkpoint_path) + # 'models/mnist-0\checkpint-19800' + # reader = pywrap_tensorflow.NewCheckpointReader(cur_checkpoint) + # var_to_shape_map = reader.get_variable_to_shape_map() + # for key in var_to_shape_map: + # print(key) + + diff --git a/evaluation/mnist/model.py b/evaluation/mnist/model.py new file mode 100644 index 0000000..65ed350 --- /dev/null +++ b/evaluation/mnist/model.py @@ -0,0 +1,76 @@ +""" +The model is adapted from the tensorflow tutorial: +https://www.tensorflow.org/get_started/mnist/pros +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +class Model(object): + def __init__(self): + self.x_input = tf.placeholder(tf.float32, shape = [None, 784]) + self.y_input = tf.placeholder(tf.int64, shape = [None]) + + self.x_image = tf.reshape(self.x_input, [-1, 28, 28, 1]) + + # first convolutional layer + W_conv1 = self._weight_variable([5,5,1,32]) + b_conv1 = self._bias_variable([32]) + + h_conv1 = tf.nn.relu(self._conv2d(self.x_image, W_conv1) + b_conv1) + h_pool1 = self._max_pool_2x2(h_conv1) + + # second convolutional layer + W_conv2 = self._weight_variable([5,5,32,64]) + b_conv2 = self._bias_variable([64]) + + h_conv2 = tf.nn.relu(self._conv2d(h_pool1, W_conv2) + b_conv2) + h_pool2 = self._max_pool_2x2(h_conv2) + + # first fully connected layer + W_fc1 = self._weight_variable([7 * 7 * 64, 1024]) + b_fc1 = self._bias_variable([1024]) + + h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) + h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) + + # output layer + W_fc2 = self._weight_variable([1024,10]) + b_fc2 = self._bias_variable([10]) + + self.pre_softmax = tf.matmul(h_fc1, W_fc2) + b_fc2 + + y_xent = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=self.y_input, logits=self.pre_softmax) + + self.xent = tf.reduce_sum(y_xent) + + self.y_pred = tf.argmax(self.pre_softmax, 1) + + correct_prediction = tf.equal(self.y_pred, self.y_input) + + self.num_correct = tf.reduce_sum(tf.cast(correct_prediction, tf.int64)) + self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + + @staticmethod + def _weight_variable(shape): + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + + @staticmethod + def _bias_variable(shape): + initial = tf.constant(0.1, shape = shape) + return tf.Variable(initial) + + @staticmethod + def _conv2d(x, W): + return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME') + + @staticmethod + def _max_pool_2x2( x): + return tf.nn.max_pool(x, + ksize = [1,2,2,1], + strides=[1,2,2,1], + padding='SAME') diff --git a/evaluation/mnist/model_converter.py b/evaluation/mnist/model_converter.py new file mode 100644 index 0000000..8bb3603 --- /dev/null +++ b/evaluation/mnist/model_converter.py @@ -0,0 +1,99 @@ +import tensorflow as tf +from tensorflow import keras +import numpy as np +import os +import sys +from model import Model + +from tensorflow.contrib.keras.api.keras.models import Sequential +from tensorflow.contrib.keras.api.keras.layers import Dense, Dropout, Activation, Flatten, Lambda +from tensorflow.contrib.keras.api.keras.layers import Conv2D, MaxPooling2D +from tensorflow.contrib.keras.api.keras.models import load_model +from tensorflow.contrib.keras.api.keras.optimizers import SGD, Adam + +from tensorflow.python import pywrap_tensorflow + +# read checkpoint +with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + +# set path of model in and out + model_name = 'mnist-0.3-selftrained' + saver = tf.train.import_meta_graph('models/mnist-0.3/checkpoint-19800.meta') + saver.restore(sess, 'models/mnist-0.3/checkpoint-19800') + + vars_global = tf.global_variables() + model_vars = {} + var_shapes = [] + var_names = [] + + for var in vars_global: + # try: + model_vars[var.name] = var.eval() + var_shapes.append((model_vars[var.name].shape)) + var_names.append(var.name) + # except: + # print("For var={}, an exception occurred".format(var.name)) + +labels = tf.placeholder(tf.int64, shape=[None], name='labels') +logits = tf.placeholder(tf.float32, shape=[None, 10], name='logits') + +# create keras model according to the model structure +model = Sequential() +conv1 = Conv2D(32, (5,5), input_shape=(28,28,1), activation='relu', padding='SAME', +use_bias=True +) +model.add(conv1) +model.add(MaxPooling2D(pool_size=(2,2), + padding='SAME')) +conv2 = Conv2D(64, (5,5), activation='relu', padding='SAME', +use_bias=True +) +model.add(conv2) +model.add(MaxPooling2D(pool_size=(2,2), + padding='SAME')) +model.add(Flatten()) + +fc1 = Dense(1024, activation='relu', use_bias=True) +model.add(fc1) +fc2 = Dense(10, activation='softmax', use_bias=True) +model.add(fc2) + +def fn(correct, predicted): + return tf.nn.softmax_cross_entropy_with_logits(labels=correct, + logits=predicted) +sgd = keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) +model.compile(optimizer=sgd, loss=fn, metrics=['accuracy']) + +# match layers of two models +model.layers[0].set_weights([model_vars['Variable:0'], model_vars['Variable_1:0']]) +model.layers[2].set_weights([model_vars['Variable_2:0'], model_vars['Variable_3:0']]) +model.layers[5].set_weights([model_vars['Variable_4:0'], model_vars['Variable_5:0']]) +model.layers[6].set_weights([model_vars['Variable_6:0'], model_vars['Variable_7:0']]) + + +model.save('models/'+model_name+'.h5', overwrite=True) + +# validate the accuracy of models + +dataset = keras.datasets.mnist.load_data() + +# mnist +test_data = dataset[1][0] +test_labels = dataset[1][1] + +extra_bias = 0 + +batch_size = 100 +op_data = np.zeros((1,10)) +for i in range(int(test_data.shape[0]/batch_size)): + # x = test_data[i*batch_size:(i+1)*batch_size] + x = test_data[i*batch_size:(i+1)*batch_size]/255 - extra_bias + x = np.reshape(x, (batch_size, test_data.shape[1], test_data.shape[2], 1)) + # op = sess.run(model.output, feed_dict = {model.input: x}) + op = model.predict(x, batch_size=batch_size) + op_data = np.vstack((op_data, op)) +op_data = op_data[1:] +print('accuracy is:') +print(np.sum(np.argmax(op_data, axis=1) == test_labels)/op_data.shape[0]) + diff --git a/evaluation/mnist/model_robustml.py b/evaluation/mnist/model_robustml.py new file mode 100644 index 0000000..3b3792c --- /dev/null +++ b/evaluation/mnist/model_robustml.py @@ -0,0 +1,45 @@ +import robustml +import tensorflow as tf + +import model + +class Model(robustml.model.Model): + def __init__(self, sess): + self._model = model.Model() + + saver = tf.train.Saver() + checkpoint = tf.train.latest_checkpoint('models/secret') + saver.restore(sess, checkpoint) + + self._sess = sess + self._input = self._model.x_input + self._logits = self._model.pre_softmax + self._predictions = self._model.y_pred + self._dataset = robustml.dataset.MNIST() + self._threat_model = robustml.threat_model.Linf(epsilon=0.3) + + @property + def dataset(self): + return self._dataset + + @property + def threat_model(self): + return self._threat_model + + def classify(self, x): + return self._sess.run(self._predictions, + {self._input: x})[0] + + # expose attack interface + + @property + def input(self): + return self._input + + @property + def logits(self): + return self._logits + + @property + def predictions(self): + return self._predictions diff --git a/evaluation/mnist/pgd_attack.py b/evaluation/mnist/pgd_attack.py new file mode 100644 index 0000000..88186db --- /dev/null +++ b/evaluation/mnist/pgd_attack.py @@ -0,0 +1,121 @@ +""" +Implementation of attack methods. Running this file as a program will +apply the attack to the model specified by the config file and store +the examples in an .npy file. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +import numpy as np + + +class LinfPGDAttack: + def __init__(self, model, epsilon, k, a, random_start, loss_func): + """Attack parameter initialization. The attack performs k steps of + size a, while always staying within epsilon from the initial + point.""" + self.model = model + self.epsilon = epsilon + self.k = k + self.a = a + self.rand = random_start + + if loss_func == 'xent': + loss = model.xent + elif loss_func == 'cw': + label_mask = tf.one_hot(model.y_input, + 10, + on_value=1.0, + off_value=0.0, + dtype=tf.float32) + correct_logit = tf.reduce_sum(label_mask * model.pre_softmax, axis=1) + wrong_logit = tf.reduce_max((1-label_mask) * model.pre_softmax + - 1e4*label_mask, axis=1) + loss = -tf.nn.relu(correct_logit - wrong_logit + 50) + else: + print('Unknown loss function. Defaulting to cross-entropy') + loss = model.xent + + self.grad = tf.gradients(loss, model.x_input)[0] + + def perturb(self, x_nat, y, sess): + """Given a set of examples (x_nat, y), returns a set of adversarial + examples within epsilon of x_nat in l_infinity norm.""" + if self.rand: + x = x_nat + np.random.uniform(-self.epsilon, self.epsilon, x_nat.shape) + x = np.clip(x, 0, 1) # ensure valid pixel range + else: + x = np.copy(x_nat) + + for i in range(self.k): + grad = sess.run(self.grad, feed_dict={self.model.x_input: x, + self.model.y_input: y}) + + x += self.a * np.sign(grad) + + x = np.clip(x, x_nat - self.epsilon, x_nat + self.epsilon) + x = np.clip(x, 0, 1) # ensure valid pixel range + + return x + + +if __name__ == '__main__': + import json + import sys + import math + + from tensorflow.examples.tutorials.mnist import input_data + + from model import Model + + with open('config.json') as config_file: + config = json.load(config_file) + + model_file = tf.train.latest_checkpoint(config['model_dir']) + if model_file is None: + print('No model found') + sys.exit() + + model = Model() + attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + saver = tf.train.Saver() + + mnist = input_data.read_data_sets('MNIST_data', one_hot=False) + + with tf.Session() as sess: + # Restore the checkpoint + saver.restore(sess, model_file) + + # Iterate over the samples batch-by-batch + num_eval_examples = config['num_eval_examples'] + eval_batch_size = config['eval_batch_size'] + num_batches = int(math.ceil(num_eval_examples / eval_batch_size)) + + x_adv = [] # adv accumulator + + print('Iterating over {} batches'.format(num_batches)) + + for ibatch in range(num_batches): + bstart = ibatch * eval_batch_size + bend = min(bstart + eval_batch_size, num_eval_examples) + print('batch size: {}'.format(bend - bstart)) + + x_batch = mnist.test.images[bstart:bend, :] + y_batch = mnist.test.labels[bstart:bend] + + x_batch_adv = attack.perturb(x_batch, y_batch, sess) + + x_adv.append(x_batch_adv) + + print('Storing examples') + path = config['store_adv_path'] + x_adv = np.concatenate(x_adv, axis=0) + np.save(path, x_adv) + print('Examples stored in {}'.format(path)) diff --git a/evaluation/mnist/run_attack.py b/evaluation/mnist/run_attack.py new file mode 100644 index 0000000..2a0f680 --- /dev/null +++ b/evaluation/mnist/run_attack.py @@ -0,0 +1,93 @@ +"""Evaluates a model against examples from a .npy file as specified + in config.json""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import json +import math +import os +import sys +import time + +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data + +import numpy as np + +from model import Model + +def run_attack(checkpoint, x_adv, epsilon): + mnist = input_data.read_data_sets('MNIST_data', one_hot=False) + + model = Model() + + saver = tf.train.Saver() + + num_eval_examples = 10000 + eval_batch_size = 64 + + num_batches = int(math.ceil(num_eval_examples / eval_batch_size)) + total_corr = 0 + + x_nat = mnist.test.images + l_inf = np.amax(np.abs(x_nat - x_adv)) + + if l_inf > epsilon + 0.0001: + print('maximum perturbation found: {}'.format(l_inf)) + print('maximum perturbation allowed: {}'.format(epsilon)) + return + + y_pred = [] # label accumulator + + with tf.Session() as sess: + # Restore the checkpoint + saver.restore(sess, checkpoint) + + # Iterate over the samples batch-by-batch + for ibatch in range(num_batches): + bstart = ibatch * eval_batch_size + bend = min(bstart + eval_batch_size, num_eval_examples) + + x_batch = x_adv[bstart:bend, :] + y_batch = mnist.test.labels[bstart:bend] + + dict_adv = {model.x_input: x_batch, + model.y_input: y_batch} + cur_corr, y_pred_batch = sess.run([model.num_correct, model.y_pred], + feed_dict=dict_adv) + + total_corr += cur_corr + y_pred.append(y_pred_batch) + + accuracy = total_corr / num_eval_examples + + print('Accuracy: {:.2f}%'.format(100.0 * accuracy)) + y_pred = np.concatenate(y_pred, axis=0) + np.save('pred.npy', y_pred) + print('Output saved at pred.npy') + +if __name__ == '__main__': + import json + + with open('config.json') as config_file: + config = json.load(config_file) + + model_dir = config['model_dir'] + + checkpoint = tf.train.latest_checkpoint(model_dir) + x_adv = np.load(config['store_adv_path']) + + if checkpoint is None: + print('No checkpoint found') + elif x_adv.shape != (10000, 784): + print('Invalid shape: expected (10000,784), found {}'.format(x_adv.shape)) + elif np.amax(x_adv) > 1.0001 or \ + np.amin(x_adv) < -0.0001 or \ + np.isnan(np.amax(x_adv)): + print('Invalid pixel range. Expected [0, 1], found [{}, {}]'.format( + np.amin(x_adv), + np.amax(x_adv))) + else: + run_attack(checkpoint, x_adv, config['epsilon']) diff --git a/evaluation/mnist/train.py b/evaluation/mnist/train.py new file mode 100644 index 0000000..3844284 --- /dev/null +++ b/evaluation/mnist/train.py @@ -0,0 +1,122 @@ +""" +The code is adapted from the MadryLab's repo: +https://github.com/MadryLab/mnist_challenge/train.py +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from datetime import datetime +import json +import os +import shutil +from timeit import default_timer as timer + +import tensorflow as tf +import numpy as np +from tensorflow.examples.tutorials.mnist import input_data + +from model import Model +from pgd_attack import LinfPGDAttack + +with open('config.json') as config_file: + config = json.load(config_file) + +# Setting up training parameters +tf.set_random_seed(config['random_seed']) + +max_num_training_steps = config['max_num_training_steps'] +num_output_steps = config['num_output_steps'] +num_summary_steps = config['num_summary_steps'] +num_checkpoint_steps = config['num_checkpoint_steps'] + +batch_size = config['training_batch_size'] + +# Setting up the data and the model +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) +global_step = tf.contrib.framework.get_or_create_global_step() +model = Model() + +# Setting up the optimizer +train_step = tf.train.AdamOptimizer(1e-4).minimize(model.xent, + global_step=global_step) + +# Set up adversary +attack = LinfPGDAttack(model, + config['epsilon'], + config['k'], + config['a'], + config['random_start'], + config['loss_func']) + +# Setting up the Tensorboard and checkpoint outputs +model_dir = config['model_dir'] +if not os.path.exists(model_dir): + os.makedirs(model_dir) + +# We add accuracy and xent twice so we can easily make three types of +# comparisons in Tensorboard: +# - train vs eval (for a single run) +# - train of different runs +# - eval of different runs + +saver = tf.train.Saver(max_to_keep=3) +tf.summary.scalar('accuracy adv train', model.accuracy) +tf.summary.scalar('accuracy adv', model.accuracy) +tf.summary.scalar('xent adv train', model.xent / batch_size) +tf.summary.scalar('xent adv', model.xent / batch_size) +tf.summary.image('images adv train', model.x_image) +merged_summaries = tf.summary.merge_all() + +shutil.copy('config.json', model_dir) + +with tf.Session() as sess: + # Initialize the summary writer, global variables, and our time counter. + summary_writer = tf.summary.FileWriter(model_dir, sess.graph) + sess.run(tf.global_variables_initializer()) + training_time = 0.0 + + # Main training loop + for ii in range(max_num_training_steps): + x_batch, y_batch = mnist.train.next_batch(batch_size) + + # Compute Adversarial Perturbations + start = timer() + x_batch_adv = attack.perturb(x_batch, y_batch, sess) + end = timer() + training_time += end - start + + nat_dict = {model.x_input: x_batch, + model.y_input: y_batch} + + adv_dict = {model.x_input: x_batch_adv, + model.y_input: y_batch} + + # Output to stdout + if ii % num_output_steps == 0: + nat_acc = sess.run(model.accuracy, feed_dict=nat_dict) + adv_acc = sess.run(model.accuracy, feed_dict=adv_dict) + print('Step {}: ({})'.format(ii, datetime.now())) + print(' training nat accuracy {:.4}%'.format(nat_acc * 100)) + print(' training adv accuracy {:.4}%'.format(adv_acc * 100)) + if ii != 0: + print(' {} examples per second'.format( + num_output_steps * batch_size / training_time)) + training_time = 0.0 + # Tensorboard summaries + if ii % num_summary_steps == 0: + summary = sess.run(merged_summaries, feed_dict=adv_dict) + summary_writer.add_summary(summary, global_step.eval(sess)) + + # Write a checkpoint + if ii % num_checkpoint_steps == 0: + saver.save(sess, + os.path.join(model_dir, 'checkpoint'), + global_step=global_step) + + # Actual training step + start = timer() + sess.run(train_step, feed_dict=adv_dict) + end = timer() + training_time += end - start diff --git a/evaluation/readme.md b/evaluation/readme.md new file mode 100644 index 0000000..c8a3a9b --- /dev/null +++ b/evaluation/readme.md @@ -0,0 +1,48 @@ +SPADE_evaluation +=============================== + +SPADE: A Spectral Method for Black-Box Adversarial Robustness Evaluation + +Usage +----- + +**SPADE-Guided Robustness Evaluation** +1. To reproduce our experiment, extract data from following Models: + +`mnist(modified from https://github.com/MadryLab/mnist_challenge)` + + 1. config parameters and paths in config.json, mainly model_path and eps + 2. run train.py to train tensorflow model for mnist dataset(with adversial examples), the models are saved in checkpoints in ./models folder + 2b. model with eps=0 and 0.3 can also be downloaded using `fetch_model.py`, in this way: `python fetch_model.py natural`(or `adv_trained`) + 3a. when analyzing characteristic of model, run `gbt_eval.py` to evaluate with trained model through the train/test set(output paths can be set in the py file, while input and other config in config.json) + 3b. when runing for clever score, run model_converter.py to get the .h5 file containing structure and weights of the model. Paths are set at the head of the py file. + by default .5h model files are also stored in ./models folder + +`robustness(modified from https://github.com/MadryLab/robustness)` + + 1. install robustness package using pip or other similar tools + 2. download model from github page https://github.com/MadryLab/robustness, + or train model with robustness, e.g. + `python -m robustness.main --dataset restrict_imagenet --adv-train 0 --arch resnet50 --out-dir logs/checkpoints/dir/` + for a naturally trained resnet-50 model for cifar dataset, + more parameters see + https://robustness.readthedocs.io/en/latest/example_usage/cli_usage.html#training-a-standard-nonrobust-model + 3. evaluate through the model trained using `run.py`. + by default .pt model files are put together with `run.py`, and output files are stored in `./train_eval_results` foler + + +`CLEVER(modified from https://github.com/huanzhang12/CLEVER)` + + 1. run `python3 collect_gradients.py --data mnist --model_name 2-layer --target_type 16 --numimg 10 -s ./your data storage location` to get gredients. Data set options: `mnist` and `cifar`. Model_name to "2-layer" (MLP), "normal" (7-layer CNN), "distilled" (7-layer CNN with defensive distillation). For `mnist`, two more Model_name are `mnist_0` and `mnist_03` + 1b(SPADE-Guided CLEVER training). to calculate SPADE-Guided gredients, an example is adding `--ids ./normal_spade_nodelist/vanilla/nodes_cifar_cnn.csv` to 1. you can switch all csv file to ids to calculate different network gredients. nodes rankings are calculated with 10NN, `vanilla` stands for top ranked nodes, `vanilla_reverse` stands for reversed top ranked nodes. + 2. run `python3 clever.py --untargeted ./your_data_storage_location/` to get clever score. + + `AugMix(modified from https://github.com/google-research/augmix)` + + 1. run `python3 cifar.py -s ./your_data_storage_location/ -m networks_name --no-jsd ` for AugMix trained data + 2. run `python3 cifar.py -s ./your_data_storage_location/ -m networks_name --mix_off --no-jsd ` for standard trained data + networks options have `allconv`, `resnext`, `densenet`, and `WRN` + +`GAIRAT(modified from https://github.com/zjfheart/Geometry-aware-Instance-reweighted-Adversarial-Training.git)` + + 1. run `python3 GAIRAT.py --epsilon 0.25 --net 'resnet18' --out-dir './your_data_save_location' ` diff --git a/evaluation/robustness/run.py b/evaluation/robustness/run.py new file mode 100644 index 0000000..bcf663e --- /dev/null +++ b/evaluation/robustness/run.py @@ -0,0 +1,133 @@ +import torch +import torchvision.models as models +from robustness import model_utils, datasets, train, defaults +from robustness.datasets import CIFAR +from robustness.datasets import CINIC +from robustness.model_utils import make_and_restore_model +import numpy as np + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + +# read dataset from file +dataset = CIFAR() + +# read trained model +model_rn_0, _ = model_utils.make_and_restore_model(arch='resnet50', dataset=dataset, resume_path='cifar_nat_0.pt') +model_rn_025, _ = model_utils.make_and_restore_model(arch='resnet50', dataset=dataset, resume_path='cifar_l2_0_25.pt') +model_rn_05, _ = model_utils.make_and_restore_model(arch='resnet50', dataset=dataset, resume_path='cifar_l2_0_5.pt') +model_rn_1, _ = model_utils.make_and_restore_model(arch='resnet50', dataset=dataset, resume_path='cifar_l2_1_0.pt') + + +# img = img.to(device) +batch_size = 10 +element_size = 3*32*32 + +inp_data = np.zeros((1,3*32*32)) +op_data = np.zeros((1,10)) +label_data = np.zeros((1,1)) +acc_data = np.array([]) + +count_beat = 0 + +# create dataset loader +train_loader, _ = dataset.make_loaders(workers=0, batch_size=10) + + +for img, label in train_loader: + img = img.to(device) + op = model_rn_0(img) + op_data = np.vstack((op_data, op[0].detach().cpu().numpy())) + inp_data = np.vstack((inp_data, img.cpu().numpy().reshape(batch_size, element_size))) + label_data = np.append(label_data, label.cpu().numpy()) + count_beat = count_beat + 1 + print(count_beat) + + # if count_beat == 1: break; + +inp_data = inp_data[1:,:] +op_data = op_data[1:,:] +label_data = label_data[1:] +acc_data = np.append(acc_data, np.sum(np.argmax(op_data,1) == label_data)/op_data.shape[0]) + +np.savetxt('train_eval_results/testset_input_0.csv', inp_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_output_0.csv', op_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_label_0.csv', label_data, fmt="%d") + + +# 0.25 ========================= +inp_data = np.zeros((1,3*32*32)) +op_data = np.zeros((1,10)) +label_data = np.zeros((1,1)) +train_loader, _ = dataset.make_loaders(workers=0, batch_size=10) + +count_beat = 0 + +for img, label in train_loader: + img = img.to(device) + # target_label = (label + torch.randint_like(label, high=3))%10 + op = model_rn_025(img) + # adv_op = model_rn_025(img, target_label, **attack_kwargs) + inp_data = np.vstack((inp_data, img.cpu().numpy().reshape(batch_size, element_size))) + op_data = np.vstack((op_data, op[0].detach().cpu().numpy())) + # op_data = np.vstack((op_data, adv_op[0].detach().cpu().numpy())) + label_data = np.append(label_data, label.cpu().numpy()) + count_beat = count_beat + 1 + print(count_beat) + +inp_data = inp_data[1:,:] +op_data = op_data[1:,:] +label_data = label_data[1:] +acc_data = np.append(acc_data, np.sum(np.argmax(op_data,1) == label_data)/op_data.shape[0]) +np.savetxt('train_eval_results/testset_input_025.csv', inp_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_output_025.csv', op_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_label_025.csv', label_data, fmt="%d") + +# # 0.5 ================================= +inp_data = np.zeros((1,3*32*32)) +op_data = np.zeros((1,10)) +label_data = np.zeros((1,1)) +train_loader, _ = dataset.make_loaders(workers=0, batch_size=10) + +count_beat = 0 + +for img, label in train_loader: + img = img.to(device) + op = model_rn_05(img) + op_data = np.vstack((op_data, op[0].detach().cpu().numpy())) + inp_data = np.vstack((inp_data, img.cpu().numpy().reshape(batch_size, element_size))) + label_data = np.append(label_data, label.cpu().numpy()) + count_beat = count_beat + 1 + print(count_beat) + +inp_data = inp_data[1:,:] +op_data = op_data[1:,:] +label_data = label_data[1:] +acc_data = np.append(acc_data, np.sum(np.argmax(op_data,1) == label_data)/op_data.shape[0]) +np.savetxt('train_eval_results/testset_input_05.csv', inp_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_output_05.csv', op_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_label_05.csv', label_data, fmt="%d") + +# # # 1.0 ================================= +inp_data = np.zeros((1,3*32*32)) +op_data = np.zeros((1,10)) +label_data = np.zeros((1,1)) +train_loader, _ = dataset.make_loaders(workers=0, batch_size=10) + +count_beat = 0 + +for img, label in train_loader: + img = img.to(device) + op = model_rn_1(img) + op_data = np.vstack((op_data, op[0].detach().cpu().numpy())) + inp_data = np.vstack((inp_data, img.cpu().numpy().reshape(batch_size, element_size))) + label_data = np.append(label_data, label.cpu().numpy()) + count_beat = count_beat + 1 + print(count_beat) + +inp_data = inp_data[1:,:] +op_data = op_data[1:,:] +label_data = label_data[1:] +acc_data = np.append(acc_data,np.sum(np.argmax(op_data,1) == label_data)/op_data.shape[0]) +np.savetxt('train_eval_results/testset_input_1.csv', inp_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_output_1.csv', op_data, fmt="%.4f") +np.savetxt('train_eval_results/testset_label_1.csv', label_data, fmt="%d")