figureType = 'png';

% try to figure out if we're running octave of matlab
versionIds = version=='7';
if versionIds(1),
  isMatlab = 1;
  fprintf('it appears that we are running matlab\n'); myflush;
else
  isMatlab = 0;
  fprintf('it appears that we are running octave\n'); myflush;
end;

% first, load the results if they exist
h = fopen('results.mat','rb');  % open to check if the file exists
if h >= 0,  % open successful -- read data
  fclose(h);
  fprintf('loading data from results.mat\n'); myflush;
  load results;
else  % otherwise, start from beginning
  fprintf('initializing empty results\n'); myflush;
  results = {};
end;

% begin by loading data
fname = '5ng';
if ~isMatlab, fname = '5ng-octave'; end;
fprintf('loading data from %s.mat\n', fname); myflush;
load(fname);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
time_start = cputime;

% find the right classes
id = find(trY==1 | trY==2);
X0 = trX(id,:);
Y0 = trY(id) - 1;   % make zero versus one

% separate into train/development
id = zeros(size(Y0));
id(1:3:end) = 1;
id = logical(id);
X1 = X0( id,:); Y1 = Y0( id);
X0 = X0(~id,:); Y0 = Y0(~id);
% permute the training data
id = randperm(size(X0,1));
X0 = X0(id,:); Y0 = Y0(id,:);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

fprintf(['\n\nIn the first experiment we start looking at boosting this\n' ...
         'binary problem with decision stumps.  What we are particularly\n' ...
         'interested in is how accuracy changes as a function of the\n' ...
         'number of boosted iterations.\n\n']); myflush;

numBoost = [1 3 5 7 9 10];
if ~isfield(results,'binboosttrain'),
  tic
  fprintf('Training...'); myflush;
  boost = Boost('train', @DT, X0, Y0, max(numBoost), 1);

  tracc = zeros(1,length(numBoost));
  teacc = zeros(1,length(numBoost));
  fprintf('\nPredicting...');
  for ii=1:length(numBoost),
    fprintf('%d...', ii); myflush;
    tracc(ii) = mean( Y0 == Boost('predict', @DT, boost, X0, numBoost(ii)) );
    teacc(ii) = mean( Y1 == Boost('predict', @DT, boost, X1, numBoost(ii)) );
  end;
  
  results.binboosttrain = tracc;
  results.binboosttest  = teacc;
  mysave(isMatlab,results);
  fprintf('\n'); myflush;
  toc
end;
tracc = results.binboosttrain;
teacc = results.binboosttest;

figure(1);
if isMatlab, clf; else, clg; end;
hold on;
% plot the accuracies
if isMatlab,
  plot(numBoost, tracc, 'bo-', ...
       numBoost, teacc, 'rx-');
  legend('training data', 'test data');
else
  plot(numBoost, tracc, 'bo-;training data;', ...
       numBoost, teacc, 'rx-;test data;');
end;

xlabel('number of boosting iterations');
ylabel('accuracy');
title('figure 1: boosted decision stumps on binary classification problem');
hold off;
mysaveas(figure(1), sprintf('fig1.%s', figureType));

mypause;


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

fprintf('\n\nWe continue our boosting experiments by looking at the\ndifference between learning stumps and learning "real" decision\ntrees.  We will train a baseline model and boosted models at\na range of different "maximum depths".\n\n');

dtDepths = [1 3 5 10];
if ~isfield(results,'bindttest'),
  tic
  fprintf('Training and testing...\n'); myflush;
  
  dtacc = zeros(1, length(dtDepths));
  btacc = zeros(1, length(dtDepths));
  for ii=1:length(dtDepths),
    dd = dtDepths(ii);
    fprintf('  maxDepth=%d...', dd); myflush;
    
    boost = Boost('train', @DT, X0, Y0, 5, dd);
    dtacc(ii) = mean( Y1 == Boost('predict', @DT, boost, X1, 1) );
    btacc(ii) = mean( Y1 == Boost('predict', @DT, boost, X1) );
    
    fprintf('\n'); myflush;
  end;
  
  
  results.bindttest = dtacc;
  results.binbttest = btacc;
  mysave(isMatlab,results);
  toc
end;


figure(2);
if isMatlab, clf; else, clg; end;
hold on;
% plot the accuracies
if isMatlab,
  plot(dtDepths, results.bindttest, 'bo-', ...
       dtDepths, results.binbttest, 'rx-');
  legend('without boosting', 'with boosting');
else
  plot(dtDepths, results.bindttest, 'bo-;without boosting;', ...
       dtDepths, results.binbttest, 'rx-;with boosting;');
end;

xlabel('maximum depth of decision trees');
ylabel('test data accuracy');
title('figure 2: boosted decision trees on binary classification problem');
hold off;
mysaveas(figure(2), sprintf('fig2.%s', figureType));

mypause;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

fprintf('\n\nNow, we test our multiclass-to-binary reduction.  For this,\nwe will only test DTs with a depth of 5 because it is a good deal\nfaster.  We will vary the amount\nof training data.  However, now we will use all 5 classes.\n\n');

% separate into train/development
X0 = trX;
Y0 = trY;
id = zeros(size(Y0));
id(1:3:end) = 1;
id = logical(id);
X1 = X0( id,:); Y1 = Y0( id);
X0 = X0(~id,:); Y0 = Y0(~id);
% permute the training data
id = randperm(size(X0,1));
X0 = X0(id,:); Y0 = Y0(id,:);


dataSize = [100 200 800 1600];
if ~isfield(results, 'ovatrain'),
  tic
  ovaTrain = zeros(4,1);
  ovaTest  = zeros(4,1);
  avaTrain = zeros(4,1);
  avaTest  = zeros(4,1);

  for ii=1:length(dataSize),
    nn = dataSize(ii);
    fprintf('Training on %d examples...\n', nn); myflush;
  
      fprintf('  ova: '); myflush;
      ova = OVA('train', @DT, X0(1:nn,:), Y0(1:nn,:), 5);

      fprintf('  ava: '); myflush;
      ava = AVA('train', @DT, X0(1:nn,:), Y0(1:nn,:), 5);

      ovaTrain(ii) = mean( Y0(1:nn) == OVA('predict', @DT, ova, X0(1:nn,:)) );
      avaTrain(ii) = mean( Y0(1:nn) == AVA('predict', @DT, ava, X0(1:nn,:)) );
      ovaTest( ii) = mean( Y1 == OVA('predict', @DT, ova, X1) );
      avaTest( ii) = mean( Y1 == AVA('predict', @DT, ava, X1) );

      fprintf('\n'); myflush;
  end;
  results.ovatrain = ovaTrain;
  results.avatrain = avaTrain;
  results.ovatest  = ovaTest;
  results.avatest  = avaTest;

  % save results
  mysave(isMatlab,results);
  toc
end;



figure(3);
if isMatlab, clf; else, clg; end;
hold on;
% plot the accuracies
if isMatlab,
  plot(dataSize, mean(results.avatrain,2), 'bo-', ...
       dataSize, mean(results.avatest ,2), 'rx-', ...
       dataSize, mean(results.ovatrain,2), 'go-', ...
       dataSize, mean(results.ovatest ,2), 'mx-');

  legend('ava training data', 'ava test data', 'ova training data', 'ova test data');
else
  plot(dataSize, mean(results.avatrain,2), 'bo-;ava training data;', ...
       dataSize, mean(results.avatest ,2), 'rx-;ava test data;', ...
       dataSize, mean(results.ovatrain,2), 'go-;ova training data;', ...
       dataSize, mean(results.ovatest ,2), 'mx-;ova test data;');
end;

xlabel('number of training points');
ylabel('accuracy');
title('figure 3: DT by multiclass reductions');
hold off;
mysaveas(figure(3), sprintf('fig3.%s', figureType));

mypause;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

fprintf('\n\nIn the last experiment, we test multiclass-to-binary\nreduction (with DTs), but with a variable number of classes.\n\n'); myflush;

if ~isfield(results, 'ovaclasses'),
  rand('seed', 1);
  tic
  ovaScore = zeros(1,4);
  avaScore = zeros(1,4);
  for numC=[2:5],
    fprintf('Training on %d-tuples of classes...\n', numC); myflush;
    classPairs = nchoosem(5, numC);

    ovas = 0; avas = 0;
    
    for nn=1:size(classPairs,1),
      allY = classPairs(nn,:);
      % figure out which examples match any of these Ys
      id = sum(repmat(Y0,1,numC) == repmat(allY,length(Y0),1),2) > 0;
      X0a = X0(id,:); Y0a = Y0(id);
      
      id = sum(repmat(Y1,1,numC) == repmat(allY,length(Y1),1),2) > 0;
      X1a = X1(id,:); Y1a = Y1(id);
      
      % cut out some training data
      X0a = X0a(1:500,:);
      Y0a = Y0a(1:500);
      
      % display what we're doing
      fprintf('  current Y =');
      fprintf(' %d', allY);
      fprintf(':\n'); myflush;
      
      % train and test
      fprintf('    OVA: ');
      ova = OVA('train', @DT, X0a, Y0a, 5);
      ovas = ovas + mean( Y1a == OVA('predict', @DT, ova, X1a) );
      
      fprintf('    AVA: ');
      ava = AVA('train', @DT, X0a, Y0a, 5);
      avas = avas + mean( Y1a == AVA('predict', @DT, ava, X1a) );
      
      fprintf('\n');
    end;
    
    % take average
    ovaScore(numC-1) = ovas / size(classPairs,1);
    avaScore(numC-1) = avas / size(classPairs,1);
  end;

  results.ovaclasses = ovaScore;
  results.avaclasses = avaScore;
  mysave(isMatlab,results);
  toc
end;


figure(4);
if isMatlab, clf; else, clg; end;
hold on;
% plot the accuracies
if isMatlab,
  plot(2:5, results.ovaclasses, 'bo-', ...
       2:5, results.avaclasses, 'rx-');
  legend('one-versus-all', 'all-versus-all');
else
  plot(2:5, results.ovaclasses, 'bo-;without boosting;', ...
       2:5, results.avaclasses, 'rx-;with boosting;');
end;

xlabel('number of classes');
ylabel('test data accuracy');
title('figure 4: DT on multiclass with varying number of classes');
hold off;
mysaveas(figure(4), sprintf('fig4.%s', figureType));

mypause;

time_end = cputime;

fprintf('total elapsed CPU time: %d minutes\n', ceil((time_end-time_start)/60));
