clear;clc;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Set Flags
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
search_flag = 0;
fs_flag = 0;
svm_flag = 0;
nn_flag = 1;
pca_flag = 0;
pcaclass_flag = 0;

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Set Sample Size
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
nrsamples = 1000;      %randomly selected data nrsamples per class
nssamples = 1000;      %stratified sample size

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Search Description Terms
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

terms = cellstr(strvcat('debt','loans','payoff','consolidate','divorce','new business',...
    'child support','good credit'));
% terms = cellstr(strvcat('debt','loans','payoff','credit','help','debts',...
%     'consolidate','landlord','credit card','divorce','new business',...
%     'fixup'));

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% IgnoreFeatures for Feature Search Algorithm
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
keepoutfeatures = [2];
mcreps = 1;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Initialize SVM Variables
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
rfoundfeatures = [];
sfoundfeatures = [];

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% PCA\LDA Variables
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
rankfeaturesnumber = 11; %number of features to extract from data sample
princompclassify = 4; %number of principle components used to classify data

if search_flag == 1
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('% Begin searching for terms')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    load clistings.mat;
    load flistings.mat;

    c1searchresults = zeros(length(clistings),length(terms));
    c0searchresults = zeros(length(flistings),length(terms));

    for i=1:length(terms);
        c1searchresults(:,i) = stringfreq(clistings, terms{i});
    end
    for i=1:length(terms);
        c0searchresults(:,i) = stringfreq(flistings, terms{i});
    end
    clearvars clistings flistings
end
    


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Load Data
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
load '.\datasets\default\defaultpaidcurrentlate.mat';
%load '.\datasets\paid_default\paid_default.mat'; %good one of charlies
%load '.\datasets\paid_default\paid_default16-19-45.mat';
%load 96featureset.mat
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Reduce Data (If needed for Charlies Features)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
cfeaturecolumns = 1:size(paid,2);
%[]
%[20]
%[17, 20]
%[17, 20, 46]
%cfeaturecolumns = [8, 17, 20, 46];
%[8, 17, 20, 46, 90]
%[8, 17, 20, 26, 46, 90]
%[8, 17, 19, 20, 46, 73, 90]
%[8, 17, 19, 20, 46, 50, 73, 90]
%[8, 17, 19, 20, 46, 50, 73, 78, 90]
%cfeaturecolumns = [8, 17, 19, 20, 26, 46, 50, 73, 78, 90];
%[2, 8, 17, 19, 20, 26, 46, 50, 73, 78, 90]
%[2, 8, 16, 17, 19, 20, 26, 46, 50, 73, 78, 90]
%[2, 8, 17, 19, 20, 26, 35, 37, 46, 50, 73, 78, 90]
%[2, 8, 17, 19, 20, 26, 35, 37, 46, 50, 73, 78, 85, 90]
%[2, 8, 17, 19, 20, 26, 35, 36, 37, 46, 50, 73, 78, 85, 90]
%[2, 8, 17, 19, 20, 26, 35, 36, 37, 41, 46, 50, 73, 78, 85, 90]
%[2, 8, 17, 19, 20, 26, 35, 36, 37, 41, 46, 50, 73, 78, 85, 89, 90]
%cfeaturecolumns = [2, 8, 16, 17, 19, 20, 26, 35, 36, 37, 41, 46, 50, 73, 78, 85, 89, 90];
%[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 46, 50, 73, 78, 85, 89, 90]
%[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 46, 50, 73, 78, 85, 89, 90]
%cfeaturecolumns = [2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 73, 78, 85, 89, 90];
%[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 73, 78, 85, 89, 90]
%[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 78, 85, 89, 90]
%[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 74, 78, 85, 89, 90]
%[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91]
%[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91, 92]
%[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91, 92, 96]
%[2, 8, 16, 17, 19, 20, 26, 30, 34, 35, 36, 37, 41, 44, 45, 46, 47, 50, 54, 55, 73, 78, 85, 89, 90, 91, 92, 96]
%[2, 8, 16, 17, 19, 20, 26, 30, 34, 35, 36, 37, 41, 44, 45, 46, 47, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91, 92, 96]
%[2, 8, 16, 17, 19, 20, 26, 30, 32, 34, 35, 36, 37, 41, 44, 45, 46, 47, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91, 92, 96]

paid = paid(:,cfeaturecolumns);
default = default(:,cfeaturecolumns);

disp(' ')
display(sprintf('loaded dataset objects'))
disp(sprintf('population of class 1 (paid) %d', length(paid)))
disp(sprintf('population of class 0 (default) %d', length(default)))
disp(sprintf('class 1 accounts for %f of total', length(paid)/(length(default)+length(paid))))
display(' ')

if search_flag == 1
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('% Adding Search Results to Feature Vector')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    paidloans = horzcat(paid,c1searchresults);
    defaultedloans = horzcat(default,c0searchresults);
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Remove NaNs
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
display(sprintf('removed rows of data with NaN'))
paid(any(isnan(paid)'),:) = [];
default(any(isnan(default)'),:) = [];
disp(sprintf('new population of class 1 (paid) %d', length(paid)))
disp(sprintf('new population of class 0 (default) %d', length(default)))
disp(sprintf('class 1 accounts for %f of total', length(paid)/(length(default)+length(paid))))
display(' ')


display(sprintf('recalculated class observation vector for NaN'))
display(' ')
paidloanobs = ones(length(paid),1);
defaultedloansobs = zeros(length(default),1);

%COLUMN VECTORS
% 1   AmountRequested
% 2   BidCount
% 3   BorrowerRate
% 4   CreditGrade
% 5   DebtToIncomeRatio
% 6   GroupKey
% 7   Images
% 8   CurrentDelinquencies
% 9   DelinquenciesLast7Years
% 10  OpenCreditLines
% 11  Income

disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
disp(sprintf('Total Number of Features %d',size(cfeaturecolumns,2)))
disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
disp(sprintf('Used Feature in Column: %d\n',cfeaturecolumns))

if search_flag == 1
    disp(sprintf('Added %d text features',length(terms)))
    disp(terms)
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Handle Randomly Sampled Data (Creates Equal Priors)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%randomly sample completed listings and create observations
%returns nrsamples of each population
c1rsamples = rsample(paid,nrsamples);
c1robs = ones(length(c1rsamples),1);

%randomly sample failed listings and create observations
%returns nrsamples of each population
c0rsamples = rsample(default,nrsamples);
c0robs = zeros(length(c0rsamples),1);

%create randomly sampled combined class feature vector and observation
%vector for matlab processing

rsamples = vertcat(c1rsamples,c0rsamples);
robs = vertcat(c1robs,c0robs);


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Handle Stratified Sampled Data (Preserves Prior Probabilities)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%return stratified samples and stratified observation vector
%returns nssamples of entire population
[ssamples sobs] = stratifiedsample(vertcat(paid,default),...
    vertcat(paidloanobs,defaultedloansobs),nssamples);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Free MATLAB Memory
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

clearvars -except rsamples robs ssamples sobs fs_flag svm_flag nn_flag pca_flag keepoutfeatures...
    mcreps rfoundfeatures sfoundfeatures pcaclass_flag princompclassify rankfeaturesnumber

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Show Sample Information
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
disp('random sample info:')
disp(sprintf('total random samples from c1, c0: %d', length(robs)))
disp(sprintf('random samples from class 1 (loan): %d',length(robs)/2))
disp(sprintf('random samples from class 0 (no loan): %d',length(robs)/2))
disp(' ')
disp('stratified sample info:')
disp(sprintf('total stratified samples from c1, c0: %d', length(sobs)))
disp(sprintf('stratified samples from class 1 (loan) (approx): %d',(sum(sobs)/length(sobs))*length(sobs)))
disp(sprintf('stratified samples from class 0 (no loan) (approx): %d',length(sobs)-((sum(sobs)/length(sobs))*length(sobs))))
disp(' ')

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Sequential Feature Search Analysis & Classification
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if fs_flag == 1
    
    load 96loannoloan_10000.mat
    %cfeaturecolumns = [20,18,8,95,6,14,11,16,3,5,13,15,1,10,53,4,29,43,38,37;];
    
    %default
    %cfeaturecolumns = [1,2,3,4,6,5,8,11,10,13,12,9,7,18,19,14,20,15,16,17;];
    cfeaturecolumns = 1:size(rsamples,2);
    
    
    %a11 best features, from listings
    cfeaturecolumns=[18, 3, 8, 10, 77, 17, 50, 20, 21, 24]
    
    
    %[]
    %[20]
    %[17, 20]
    %[17, 20, 46]
    %cfeaturecolumns = [8, 17, 20, 46];
    %[8, 17, 20, 46, 90]
    %[8, 17, 20, 26, 46, 90]
    %[8, 17, 19, 20, 46, 73, 90]
    %[8, 17, 19, 20, 46, 50, 73, 90]
    %[8, 17, 19, 20, 46, 50, 73, 78, 90]
    %cfeaturecolumns = [8, 17, 19, 20, 26, 46, 50, 73, 78, 90];
    %[2, 8, 17, 19, 20, 26, 46, 50, 73, 78, 90]
    %[2, 8, 16, 17, 19, 20, 26, 46, 50, 73, 78, 90]
    %[2, 8, 17, 19, 20, 26, 35, 37, 46, 50, 73, 78, 90]
    %[2, 8, 17, 19, 20, 26, 35, 37, 46, 50, 73, 78, 85, 90]
    %cfeaturecolumns= [2, 8, 17, 19, 20, 26, 35, 36, 37, 46, 50, 73, 78, 85, 90]
    %[2, 8, 17, 19, 20, 26, 35, 36, 37, 41, 46, 50, 73, 78, 85, 90]
    %[2, 8, 17, 19, 20, 26, 35, 36, 37, 41, 46, 50, 73, 78, 85, 89, 90]
    %cfeaturecolumns = [2, 8, 16, 17, 19, 20, 26, 35, 36, 37, 41, 46, 50, 73, 78, 85, 89, 90];
    %[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 46, 50, 73, 78, 85, 89, 90]
    %[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 46, 50, 73, 78, 85, 89, 90]
    %cfeaturecolumns = [2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 73, 78, 85, 89, 90];
    %[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 73, 78, 85, 89, 90]
    %[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 78, 85, 89, 90]
    %cfeaturecolumns = [2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 74, 78, 85, 89, 90]
    %[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91]
    %[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91, 92]
    %[2, 8, 16, 17, 19, 20, 26, 34, 35, 36, 37, 41, 44, 45, 46, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91, 92, 96]
    %[2, 8, 16, 17, 19, 20, 26, 30, 34, 35, 36, 37, 41, 44, 45, 46, 47, 50, 54, 55, 73, 78, 85, 89, 90, 91, 92, 96]
    %[2, 8, 16, 17, 19, 20, 26, 30, 34, 35, 36, 37, 41, 44, 45, 46, 47, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91, 92, 96]
    %[2, 8, 16, 17, 19, 20, 26, 30, 32, 34, 35, 36, 37, 41, 44, 45, 46, 47, 50, 54, 55, 73, 74, 78, 85, 89, 90, 91, 92, 96]
    
    rsamples = rsamples(:,cfeaturecolumns);
    %default = rsamples(:,cfeaturecolumns);
    ssamples = ssamples(:,cfeaturecolumns);
    
    
    
    %initialize foundfeatures to store the feature results, make sure
    %rsamples are same size of ssamples
    if size(rsamples,2) == size(ssamples,2)
        %foundfeatures = zeros(1,(size(rsamples,2)));
        rfoundfeatures = [];
        sfoundfeatures = [];
    end
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    % Partition Random & Statified Data
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    cv1 = cvpartition(robs,'kfold', 10);     %10 Fold CV for Random Data
    cv2 = cvpartition(sobs,'kfold', 10);     %10 Fold CV for Stratified Data
    split = round(.30*length(sobs));
    cv3 = cvpartition(sobs,'holdout',split); %70/30 Holdout for Statified Data
    
%     %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%     % Sequential Feature Search 10 Fold Cross Validation (Random Sample)
%     %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Classification ALL FEATURES 10 Fold Cross Validation (Random Sample): Linear')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    opts = statset('display','iter');
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'linear')));
    [fs, history] = sequentialfs(fun,rsamples,robs,...
        'cv',cv1,'options',opts,'mcreps',mcreps,'direction','forward','KeepIn',...
        1:size(rsamples,2))
%     plot(1:11,[history.Crit history.Crit history.Crit history.Crit history.Crit history.Crit history.Crit history.Crit history.Crit history.Crit history.Crit])   
%     hold
    rfoundfeatures = (vertcat(rfoundfeatures,fs)); 
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 10 Fold Cross Validation (Random Sample): Linear')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    opts = statset('display','iter');
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'linear')));
    [fs, history] = sequentialfs(fun,rsamples,robs,...
        'cv',cv1,'options',opts,'mcreps',mcreps,'direction','forward','KeepIn',...
       1:size(ssamples,2))
    plot(1:length(history.Crit),history.Crit)
    hold
    rfoundfeatures = (vertcat(rfoundfeatures,fs)); 
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 10 Fold Cross Validation (Random Sample): Mahalanobis')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'mahalanobis')));
    [fs, history] = sequentialfs(fun,rsamples,robs,...
        'cv',cv1,'options',opts,'mcreps',mcreps,'direction','forward','KeepOut',...
        keepoutfeatures)
    rfoundfeatures = (vertcat(rfoundfeatures,fs)); 
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 10 Fold Cross Validation (Random Sample): Quadratic')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'quadratic','empirical')));
    [fs, history] = sequentialfs(fun,rsamples,robs,...
        'cv',cv1,'options',opts,'mcreps',mcreps,'direction','forward','KeepOut',...
        keepoutfeatures)
    rfoundfeatures = (vertcat(rfoundfeatures,fs)); 
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 10 Fold Cross Validation (Random Sample): Naive Bayes')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'diagLinear','empirical')));
    [fs, history] = sequentialfs(fun,rsamples,robs,...
        'cv',cv1,'options',opts,'mcreps',mcreps,'direction','forward','KeepOut',...
        keepoutfeatures)
     rfoundfeatures = (vertcat(rfoundfeatures,fs)); 
    
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    % Sequential Feature Search 10 Fold Cross Validation (Stratified Sample)
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Classification ALL FEATURES 10 Fold Cross Validation (Stratified Sample): Linear')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    opts = statset('display','iter');
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'linear')));
    [fs, history] = sequentialfs(fun,ssamples,sobs,...
        'cv',cv2,'options',opts,'mcreps',mcreps,'direction','forward','KeepIn',...
        1:size(ssamples,2))   
    plot(1:length(history.Crit),history.Crit)
    hold
    sfoundfeatures = (vertcat(sfoundfeatures,fs));     
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 10 Fold Cross Validation (Stratified Sample): Linear')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    opts = statset('display','iter');
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'diagLinear','empirical')));
    [fs, history] = sequentialfs(fun,ssamples,sobs,...
        'cv',cv2,'options',opts,'mcreps',mcreps,'direction','forward','KeepOut',...
        keepoutfeatures)
    sfoundfeatures = (vertcat(sfoundfeatures,fs));     
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 10 Fold Cross Validation (Stratified Sample): Mahalanobis')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    opts = statset('display','iter');
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'mahalanobis','empirical')));
    [fs, history] = sequentialfs(fun,ssamples,sobs,...
        'cv',cv2,'options',opts,'mcreps',mcreps,'direction','forward','KeepOut',...
        keepoutfeatures)
    sfoundfeatures = (vertcat(sfoundfeatures,fs)); 
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 10 Fold Cross Validation (Stratified Sample): Quadratic')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    opts = statset('display','iter');
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'quadratic','empirical')));
    [fs, history] = sequentialfs(fun,ssamples,sobs,...
        'cv',cv2,'options',opts,'mcreps',mcreps,'direction','forward','KeepOut',...
        keepoutfeatures)    
    sfoundfeatures = (vertcat(sfoundfeatures,fs)); 
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 10 Fold Cross Validation (Stratified Sample): Naive Bayes')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'diagLinear')));
    [fs, history] = sequentialfs(fun,ssamples,sobs,...
        'cv',cv3,'options',opts,'mcreps',mcreps,'direction','forward','KeepOut',...
        keepoutfeatures)
    sfoundfeatures = (vertcat(sfoundfeatures,fs)); 
    
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    % Sequential Feature Search 70/30 Split (Stratified Sample)
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 70/30 Split (Stratified Sample): Linear')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    opts = statset('display','iter');
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'linear','empirical')));
    [fs, history] = sequentialfs(fun,ssamples,sobs,...
        'cv',cv3,'options',opts,'mcreps',mcreps,'direction','forward','KeepOut',...
        keepoutfeatures)
    sfoundfeatures = (vertcat(sfoundfeatures,fs)); 
    
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('Sequential Feature Search 70/30 Split (Stratified Sample): Naive Bayes')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'diagQuadratic')));
    [fs, history] = sequentialfs(fun,ssamples,sobs,...
        'cv',cv3,'options',opts,'mcreps',mcreps,'direction','forward','KeepOut',...
        keepoutfeatures)
    sfoundfeatures = (vertcat(sfoundfeatures,fs)); 
end

clearvars -except rsamples robs ssamples sobs fs_flag svm_flag nn_flag pca_flag keepoutfeatures...
    mcreps rfoundfeatures sfoundfeatures pcaclass_flag princompclassify rankfeaturesnumber

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% SVM - Inherits Features from SFS
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if svm_flag == 1
    %load 96featureset.mat
    %load 11featureset.mat
    clearvars -except rfoundfeatures sfoundfeatures
    load 96loannoloan_2000.mat
    
    rsamples =double(rsamples);
    ssamples =double(ssamples);
    robs = double(robs);
    sobs = double(sobs);
    
    %cfeaturecolumns=[18, 3, 8, 10, 77, 17, 50, 20, 21, 24]
    
    
    rsamples = rsamples(:,[18, 3, 8, 10, 77, 17, 50, 20, 21, 24]);
    ssamples = ssamples(:,[18, 3, 8, 10, 77, 17, 50, 20, 21, 24]);
    
      %a11 best features, from listings
    %[2, 3, 8, 10, 17, 18, 20, 21, 22, 50, 77]
    
    
    split = round(.30*length(sobs));
    cv1 = cvpartition(robs,'holdout',split); %70/30 Holdout for Random Data
    cv2 = cvpartition(sobs,'holdout',split); %70/30 Holdout for Statified Data
    
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    % SVM - Reduce Features Random Sample
    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('% BEGIN Random Sample Data SVM')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    rfeatcolumns = 1:size(rsamples,2);
    sfeatcolumns = 1:size(ssamples,2);
    
    if isempty(rfoundfeatures) == 1 || isempty(sfoundfeatures) == 1
        rfoundfeatures = ones(1,size(rsamples,2));
        sfoundfeatures = ones(1,size(ssamples,2));
    end
    
    for i=1:size(rfoundfeatures,1)
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        % SVM - Random Sample Data
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%        

        svmfeatures = unique(rfeatcolumns.*(rfoundfeatures(i,:)));
        svmfeatures(:,~any(svmfeatures,1)) = [];
        svmrsamples = rsamples(:,svmfeatures);

        cp1 = classperf(robs);
        %cv1 = cvpartition(robs,'holdout',split);
        trIdx = cv1.training; teIdx = cv1.test;
        QuadProgOpts = svmsmoset('Display','final','MaxIter',30000);
        svmStruct = svmtrain(svmrsamples(trIdx,:),...
            robs(trIdx),'showplot',true,'QuadProg_Opts',QuadProgOpts,'Kernel_Function','linear');
        classes = svmclassify(svmStruct,svmrsamples(teIdx,:),'showplot',true);
        classperf(cp1,classes,teIdx);
        
        disp(' ')
        disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
        disp('% SVM - Random Sample Data Performance')
        disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
        disp(sprintf('Used Feature in Column: %d\n',svmfeatures))
        cp1
        figure;
    end
    
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('% BEGIN Stratified Sample Data SVM')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    
    for i=1:size(sfoundfeatures,1)
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        % SVM - Stratified Sample Data
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%        
        svmfeatures = unique(sfeatcolumns.*(sfoundfeatures(i,:)));
        svmfeatures(:,~any(svmfeatures,1)) = [];
        svmssamples = ssamples(:,svmfeatures);
        
        cp2 = classperf(sobs);
        %cv2 = cvpartition(sobs,'holdout',split);
        trIdx = cv2.training; teIdx = cv2.test;
        QuadProgOpts = svmsmoset('Display','final','MaxIter',30000);
        svmStruct = svmtrain(svmssamples(trIdx,:),...
            sobs(trIdx),'showplot',true,'QuadProg_Opts',QuadProgOpts,'Kernel_Function','rbf');
        classes = svmclassify(svmStruct,svmssamples(teIdx,:),'showplot',true);
        classperf(cp2,classes,teIdx);
        
        disp(' ')
        disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
        disp('% SVM - Stratified Sample Data Performance')
        disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
        disp(sprintf('Used Feature in Column: %d\n',svmfeatures))
        cp2
        figure;
    end
 end


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% NN Classification
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if nn_flag == 1
    clearvars -except rsamples robs ssamples sobs fs_flag svm_flag nn_flag keepoutfeatures
    disp(' ')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    disp('% NN Classification')
    disp('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    load 96loannoloan_2000.mat
    %nrobs = zeros(length(rsamples),length(unique(robs)));
%     nrobs = horzcat(robs,~robs);
%     rfeatcolumns = 1:size(rsamples,2);
%     %ssamples = ssamples(:,[1 3 4 5 6 7 8 9 10 11]);
%     outputs = create_pr_net(rsamples',nrobs')
    
    
    nsobs = horzcat(sobs,~sobs);
    rfeatcolumns = 1:size(ssamples,2);
    ssamples = ssamples(:,[1 3 4 5 6 7 8 9 10 11]);
    outputs = create_pr_net(ssamples',nsobs')
end

if pca_flag == 1
    [pc, zscores, pcvars] = princomp(ssamples);
    percent_explained = 100*pcvars/sum(pcvars);
    figure
    scatter(zscores(:,1),zscores(:,2));
    xlabel('First Principal Component');
    ylabel('Second Principal Component');
    title('Principal Component Scatter Plot');
end
    
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% PCA\LDA 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% if pcaclass_flag == 1
%     load 96loannoloan_5000.mat
%     %rsamples = rsamples(:,[1 3 4 5 6 7 8 9 10 11]);
%     split = round(.30*length(robs));
%     cv = cvpartition(robs,'holdout',split);
%     cp_pcalda0 = classperf(robs); % initializes the CP object
%     cp_pcalda1 = classperf(robs); % initializes the CP object
%     cp_pcalda2 = classperf(robs); % initializes the CP object
%     cp_pcalda3 = classperf(robs); % initializes the CP object
%     cp_pcalda4 = classperf(robs); % initializes the CP object
%     cp_pcalda5 = classperf(robs); % initializes the CP object
%     cp_pcalda6 = classperf(robs); % initializes the CP object
%     cp_pcalda7 = classperf(robs); % initializes the CP object
%     cp_pcalda8 = classperf(robs); % initializes the CP object
%     cp_pcalda9 = classperf(robs); % initializes the CP object
%     cp_pcalda10 = classperf(robs); % initializes the CP object
%     opts = statset('display','iter');
%     for k=1:10 % run cross-validation 10 times
%         cv = repartition(cv);
%         % select the 50 most significant features.
%         feat = rankfeatures(rsamples(training(cv),:)',robs(training(cv)),'NUMBER',96,'CRITERION','wilcoxon');
%         % PCA to reduce dimensionality
%         rsamples = rsamples(:,feat');
%         [pc, zscores, pcvars] = princomp(rsamples(training(cv),:),'econ');
%         
%         % Project into PCA space
%         x0 = rsamples * pc(:,1:1);
%         % Use LDA
%         [c0, err0] = classify(x0(test(cv),:),x0(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda0,c0,test(cv));
%         
%         
%         % Project into PCA space
%         x1 = rsamples * pc(:,1:2);
%         % Use LDA
%         [c1, err1] = classify(x1(test(cv),:),x1(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda1,c1,test(cv));
%         
%         % Project into PCA space
%         x2 = rsamples * pc(:,1:3);
%         % Use LDA
%         [c2, err2] = classify(x2(test(cv),:),x2(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda2,c2,test(cv));
%         
%         % Project into PCA space
%         x3 = rsamples * pc(:,1:4);
%         % Use LDA
%         [c3, err3] = classify(x3(test(cv),:),x3(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda3,c3,test(cv));
%         
%         % Project into PCA space
%         x4 = rsamples * pc(:,1:5);
%         % Use LDA
%         [c4, err4] = classify(x4(test(cv),:),x4(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda4,c4,test(cv));
%         
%         % Project into PCA space
%         x5 = rsamples * pc(:,1:6);
%         % Use LDA
%         [c5, err5] = classify(x5(test(cv),:),x5(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda5,c5,test(cv));
%         
%         
%         % Project into PCA space
%         x6 = rsamples * pc(:,1:7);
%         % Use LDA
%         [c6, err6] = classify(x6(test(cv),:),x6(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda6,c6,test(cv));
%         
%         
%         % Project into PCA space
%         x7 = rsamples * pc(:,1:8);
%         % Use LDA
%         [c7, err7] = classify(x7(test(cv),:),x7(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda7,c7,test(cv));
%         
%         % Project into PCA space
%         x8 = rsamples * pc(:,1:9);
%         % Use LDA
%         [c8, err8] = classify(x8(test(cv),:),x8(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda8,c8,test(cv));
%         
%         % Project into PCA space
%         x9 = rsamples * pc(:,1:10);
%         % Use LDA
%         [c9, err9] = classify(x9(test(cv),:),x9(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda9,c9,test(cv));
%         
%         % Project into PCA space
%         x10 = rsamples * pc(:,1:11);
%         % Use LDA
%         %         fun = @(xT,yT,xt,yt)(sum(yt ~= classify(xt,xT,yT,'Linear')));
%         %         [fs, history] = sequentialfs(fun,x10,robs,...
%         %         'cv',cv,'options',opts,'mcreps',mcreps,'direction','forward')
%         
%         [c10, err10] = classify(x10(test(cv),:),x10(training(cv),:),robs(training(cv)));
%         classperf(cp_pcalda10,c10,test(cv));
%     end
% end
%      gscatter(zscores(:,1),zscores(:,2),robs(training(cv)),'rb','v^');
%     errorratetest = horzcat(cp_pcalda0.ErrorRate,cp_pcalda1.ErrorRate,cp_pcalda2.ErrorRate,cp_pcalda3.ErrorRate,...
%         cp_pcalda4.ErrorRate,cp_pcalda5.ErrorRate,cp_pcalda6.ErrorRate,...
%         cp_pcalda7.ErrorRate,cp_pcalda8.ErrorRate,cp_pcalda9.ErrorRate,...
%         cp_pcalda10.ErrorRate);
%     errorratetraining = horzcat(err0,err1,err2,err3,err4,err5,err6,err7,err8,err9,err10);
%     clear err0 err1 err2 err3 err4 err5 err6 err7 err8 err9 err10
%     percent_explained = 100*pcvars/sum(pcvars);
 
%     
%     plot([1 2 3 4 5 6 7 8 9 10 11],errorratetest)
%     hold
%     plot([1 2 3 4 5 6 7 8 9 10 11],errorratetraining,'r')
%     vertcat(errorratetest,errorratetraining);
    
%plotpca([1 2 3 4 5 6 7 8 9 10 11],vertcat(errorratetest,errorratetraining))
    
    
   
    % gscatter(zscores(:,1),zscores(:,2),robs(training(cv)),'rb','v^');
%     xlabel('First Principal Component');
%     ylabel('Second Principal Component');
%     title('Principal Component Scatter Plot (11 Features)');
%     figure
%     gscatter(zscores(:,1),zscores(:,4),robs(training(cv)),'rb','v^');
%     xlabel('First Principal Component');
%     ylabel('Fourth Principal Component');
%     title('Principal Component Scatter Plot (11 Features)');





    
%end

if pcaclass_flag == 1
    load 96loannoloan_2000.mat
    %ssamples = ssamples(:,[2, 3, 8, 10, 17, 18, 20, 21, 22, 50, 77]);
    split = round(.30*length(sobs));
    cv = cvpartition(sobs,'holdout',split);
    cp_pcalda0 = classperf(sobs); % initializes the CP object
    cp_pcalda1 = classperf(sobs); % initializes the CP object
    cp_pcalda2 = classperf(sobs); % initializes the CP object
    cp_pcalda3 = classperf(sobs); % initializes the CP object
    cp_pcalda4 = classperf(sobs); % initializes the CP object
    cp_pcalda5 = classperf(sobs); % initializes the CP object
    cp_pcalda6 = classperf(sobs); % initializes the CP object
    cp_pcalda7 = classperf(sobs); % initializes the CP object
    cp_pcalda8 = classperf(sobs); % initializes the CP object
    cp_pcalda9 = classperf(sobs); % initializes the CP object
    cp_pcalda10 = classperf(sobs); % initializes the CP object
    opts = statset('display','iter');
    for k=1:10 % run cross-validation 10 times
        cv = repartition(cv);
        % select the 50 most significant features.
        feat = rankfeatures(ssamples(training(cv),:)',sobs(training(cv)),'NUMBER',11,'CRITERION','wilcoxon');
        % PCA to reduce dimensionality
        ssamples = ssamples(:,feat');
        [pc, zscores, pcvars] = princomp(ssamples(training(cv),:),'econ');
        
        % Project into PCA space
        x0 = ssamples * pc(:,1:1);
        % Use LDA
        [c0, err0] = classify(x0(test(cv),:),x0(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda0,c0,test(cv));
        
        
        % Project into PCA space
        x1 = ssamples * pc(:,1:2);
        % Use LDA
        [c1, err1] = classify(x1(test(cv),:),x1(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda1,c1,test(cv));
        
        % Project into PCA space
        x2 = ssamples * pc(:,1:3);
        % Use LDA
        [c2, err2] = classify(x2(test(cv),:),x2(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda2,c2,test(cv));
        
        % Project into PCA space
        x3 = ssamples * pc(:,1:4);
        % Use LDA
        [c3, err3] = classify(x3(test(cv),:),x3(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda3,c3,test(cv));
        
        % Project into PCA space
        x4 = ssamples * pc(:,1:5);
        % Use LDA
        [c4, err4] = classify(x4(test(cv),:),x4(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda4,c4,test(cv));
        
        % Project into PCA space
        x5 = ssamples * pc(:,1:6);
        % Use LDA
        [c5, err5] = classify(x5(test(cv),:),x5(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda5,c5,test(cv));
        
        
        % Project into PCA space
        x6 = ssamples * pc(:,1:7);
        % Use LDA
        [c6, err6] = classify(x6(test(cv),:),x6(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda6,c6,test(cv));        
        
        
        % Project into PCA space
        x7 = ssamples * pc(:,1:8);
        % Use LDA
        [c7, err7] = classify(x7(test(cv),:),x7(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda7,c7,test(cv));
        
        % Project into PCA space
        x8 = ssamples * pc(:,1:9);
        % Use LDA
        [c8, err8] = classify(x8(test(cv),:),x8(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda8,c8,test(cv));
        
        % Project into PCA space
        x9 = ssamples * pc(:,1:10);
        % Use LDA
        [c9, err9] = classify(x9(test(cv),:),x9(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda9,c9,test(cv));
        
        %Project into PCA space
        x10 = ssamples * pc(:,1:11);
        %Use LDA
        [c10, err10] = classify(x10(test(cv),:),x10(training(cv),:),sobs(training(cv)));
        classperf(cp_pcalda10,c10,test(cv));
        
    end
%     gscatter(zscores(:,5),zscores(:,10),sobs(training(cv)),'rb','v^');
    errorratetest = horzcat(cp_pcalda0.ErrorRate,cp_pcalda1.ErrorRate,cp_pcalda2.ErrorRate,cp_pcalda3.ErrorRate,...
        cp_pcalda4.ErrorRate,cp_pcalda5.ErrorRate,cp_pcalda6.ErrorRate,...
        cp_pcalda7.ErrorRate,cp_pcalda8.ErrorRate,cp_pcalda9.ErrorRate,...
        cp_pcalda10.ErrorRate)
    errorratetraining = horzcat(err0,err1,err2,err3,err4,err5,err6,err7,err8,err9,err10);
%     clear err0 err1 err2 err3 err4 err5 err6 err7 err8 err9 err10
%     percent_explained = 100*pcvars/sum(pcvars);
 
    
%     plot([1 2 3 4 5 6 7 8 9 10 11],errorratetest)
%     hold
%     plot([1 2 3 4 5 6 7 8 9 10 11],errorratetraining,'r')
%     vertcat(errorratetest,errorratetraining);
%     
plotpca([1 2 3 4 5 6 7 8 9 10 11],vertcat(errorratetest,errorratetraining))
    
    
   
     %gscatter(zscores(:,1),zscores(:,2),sobs(training(cv)),'rb','v^');
%     xlabel('First Principal Component');
%     ylabel('Second Principal Component');
%     title('Principal Component Scatter Plot (11 Features)');
%     figure
%     gscatter(zscores(:,1),zscores(:,4),sobs(training(cv)),'rb','v^');
%     xlabel('First Principal Component');
%     ylabel('Fourth Principal Component');
%     title('Principal Component Scatter Plot (11 Features)');





    
end

