-
Notifications
You must be signed in to change notification settings - Fork 32
/
Copy pathplot_crossval.m
91 lines (77 loc) · 3.2 KB
/
plot_crossval.m
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
%
% Bayesian Optimization of Combinatorial Structures
%
% Copyright (C) 2018 R. Baptista & M. Poloczek
%
% BOCS is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% BOCS is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with BOCS. If not, see <http://www.gnu.org/licenses/>.
%
% Copyright (C) 2018 MIT & University of Arizona
% Authors: Ricardo Baptista & Matthias Poloczek
% E-mails: rsb@mit.edu & poloczek@email.arizona.edu
%
% Script generates validation plots that compare the effect of
% increasing set size and order of the regression model for
% the specified data sets in test_case. The data is loaded from
% the files generated by the cross_validate script.
clear; close all; clc
% Tests and order to plot
test_case = {'ising','quad','contamination'};
order = 1:3;
% Declare forest green RGB
fg_rgb = [0.130000 0.550000 0.130000];
for k=1:length(test_case)
for ord=1:length(order)
% Set case parameters
test = test_case{k};
ordstr = num2str(order(ord));
% load results from file
res_file = ['../validation/results/' test '_order' ordstr '.mat'];
load(res_file)
bayes_data = reshape(bayes_err, length(n_train), n_func*n_test);
mle_data = reshape(mle_err, length(n_train), n_func*n_test);
hs_data = reshape(hs_err, length(n_train), n_func*n_test);
bayes_mean = mean(bayes_data,2);
mle_mean = mean(mle_data,2);
hs_mean = mean(hs_data,2);
bayes_std = 1.96*std(bayes_data,[],2)/sqrt(n_func*n_test);
mle_std = 1.96*std(mle_data,[],2)/sqrt(n_func*n_test);
hs_std = 1.96*std(hs_data,[],2)/sqrt(n_func*n_test);
% plot average results
figure
hold on
grid on
errorbar(n_train, bayes_mean, bayes_std, '-o','color',fg_rgb,'MarkerSize',10,'LineWidth',2)
errorbar(n_train, mle_mean, mle_std, '-or','MarkerSize',10,'LineWidth',2)
errorbar(n_train, hs_mean, hs_std, '-ob','MarkerSize',10,'LineWidth',2)
xlim([min(n_train)-5,max(n_train)+5])
xlabel('$N$','interpreter','latex')
ylabel('$\frac{1}{T} \sum_{i=1}^{T} |y^{(i)} - f(x^{(i)})|$','interpreter','latex')
legend({'Bayes Posterior','MLE Estimate','Horseshoe Estimator'})
file_name = ['../validation/figures/testerr_' test '_order' ordstr];
print('-depsc', file_name)
end
% plot errors versus order
figure
hold on
grid on
errorbar(n_train, bayes_mean, bayes_std, '-o','color',fg_rgb,'MarkerSize',10,'LineWidth',2)
errorbar(n_train, mle_mean, mle_std, '-or','MarkerSize',10,'LineWidth',2)
errorbar(n_train, hs_mean, hs_std, '-ob','MarkerSize',10,'LineWidth',2)
xlim([min(n_train)-5,max(n_train)+5])
xlabel('$N$','interpreter','latex')
ylabel('$\frac{1}{T} \sum_{i=1}^{T} |y^{(i)} - f(x^{(i)})|$','interpreter','latex')
legend({'Bayes Posterior','MLE Estimate','Horseshoe Estimator'})
file_name = ['../validation/figures/testerr_' test '_order' ordstr];
print('-depsc', file_name)
end