// Copyright (C) 2006 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#include <dlib/matrix.h>
#include <sstream>
#include <string>
#include <cstdlib>
#include <ctime>
#include <vector>
#include "../stl_checked.h"
#include "../array.h"
#include "../rand.h"
#include "checkerboard.h"
#include <dlib/statistics.h>
#include "tester.h"
#include <dlib/svm_threaded.h>
namespace
{
using namespace test;
using namespace dlib;
using namespace std;
logger dlog("test.svm");
// ----------------------------------------------------------------------------------------
void test_clutering (
)
{
dlog << LINFO << " being test_clutering()";
// Here we declare that our samples will be 2 dimensional column vectors.
typedef matrix<double,2,1> sample_type;
// Now we are making a typedef for the kind of kernel we want to use. I picked the
// radial basis kernel because it only has one parameter and generally gives good
// results without much fiddling.
typedef radial_basis_kernel<sample_type> kernel_type;
// Here we declare an instance of the kcentroid object. The first argument to the constructor
// is the kernel we wish to use. The second is a parameter that determines the numerical
// accuracy with which the object will perform part of the learning algorithm. Generally
// smaller values give better results but cause the algorithm to run slower. You just have
// to play with it to decide what balance of speed and accuracy is right for your problem.
// Here we have set it to 0.01.
kcentroid<kernel_type> kc(kernel_type(0.1),0.01);
// Now we make an instance of the kkmeans object and tell it to use kcentroid objects
// that are configured with the parameters from the kc object we defined above.
kkmeans<kernel_type> test(kc);
std::vector<sample_type> samples;
std::vector<sample_type> initial_centers;
sample_type m;
dlib::rand rnd;
print_spinner();
// we will make 50 points from each class
const long num = 50;
// make some samples near the origin
double radius = 0.5;
for (long i = 0; i < num; ++i)
{
double sign = 1;
if (rnd.get_random_double() < 0.5)
sign = -1;
m(0) = 2*radius*rnd.get_random_double()-radius;
m(1) = sign*sqrt(radius*radius - m(0)*m(0));
// add this sample to our set of samples we will run k-means
samples.push_back(m);
}
// make some samples in a circle around the origin but far away
radius = 10.0;
for (long i = 0; i < num; ++i)
{
double sign = 1;
if (rnd.get_random_double() < 0.5)
sign = -1;
m(0) = 2*radius*rnd.get_random_double()-radius;
m(1) = sign*sqrt(radius*radius - m(0)*m(0));
// add this sample to our set of samples we will run k-means
samples.push_back(m);
}
// make some samples in a circle around the point (25,25)
radius = 4.0;
for (long i = 0; i < num; ++i)
{
double sign = 1;
if (rnd.get_random_double() < 0.5)
sign = -1;
m(0) = 2*radius*rnd.get_random_double()-radius;
m(1) = sign*sqrt(radius*radius - m(0)*m(0));
// translate this point away from the origin
m(0) += 25;
m(1) += 25;
// add this sample to our set of samples we will run k-means
samples.push_back(m);
}
print_spinner();
// tell the kkmeans object we made that we want to run k-means with k set to 3.
// (i.e. we want 3 clusters)
test.set_number_of_centers(3);
// You need to pick some initial centers for the k-means algorithm. So here
// we will use the dlib::pick_initial_centers() function which tries to find
// n points that are far apart (basically).
pick_initial_centers(3, initial_centers, samples, test.get_kernel());
print_spinner();
// now run the k-means algorithm on our set of samples.
test.train(samples,initial_centers);
print_spinner();
const unsigned long class1 = test(samples[0]);
const unsigned long class2 = test(samples[num]);
const unsigned long class3 = test(samples[2*num]);
// now loop over all our samples and print out their predicted class. In this example
// all points are correctly identified.
for (unsigned long i = 0; i < samples.size()/3; ++i)
{
DLIB_TEST(test(samples[i]) == class1);
DLIB_TEST(test(samples[i+num]) == class2);
DLIB_TEST(test(samples[i+2*num]) == class3);
}
dlog << LINFO << " end test_clutering()";
}
// ----------------------------------------------------------------------------------------
// Here is the sinc function we will be trying to learn with the krls
// object.
double sinc(double x)
{
if (x == 0)
return 1;
return sin(x)/x;
}
void test_regression (
)
{
dlog << LINFO << " being test_regression()";
// Here we declare that our samples will be 1 dimensional column vectors. The reason for
// using a matrix here is that in general you can use N dimensional vectors as inputs to the
// krls object. But here we only have 1 dimension to make the example simple.
typedef matrix<double,1,1> sample_type;
// Now we are making a typedef for the kind of kernel we want to use. I picked the
// radial basis kernel because it only has one parameter and generally gives good
// results without much fiddling.
typedef radial_basis_kernel<sample_type> kernel_type;
// Here we declare an instance of the krls object. The first argument to the constructor
// is the kernel we wish to use. The second is a parameter that determines the numerical
// accuracy with which the object will perform part of the regression algorithm. Generally
// smaller values give better results but cause the algorithm to run slower. You just have
// to play with it to decide what balance of speed and accuracy is right for your problem.
// Here we have set it to 0.001.
krls<kernel_type> test(kernel_type(0.1),0.001);
rvm_regression_trainer<kernel_type> rvm_test;
rvm_test.set_kernel(test.get_kernel());
krr_trainer<kernel_type> krr_test;
krr_test.set_kernel(test.get_kernel());
svr_trainer<kernel_type> svr_test;
svr_test.set_kernel(test.get_kernel());
svr_test.set_epsilon_insensitivity(0.0001);
svr_test.set_c(10);
rbf_network_trainer<kernel_type> rbf_test;
rbf_test.set_kernel(test.get_kernel());
rbf_test.set_num_centers(13);
print_spinner();
std::vector<sample_type> samples;
std::vector<sample_type> samples2;
std::vector<double> labels;
std::vector<double> labels2;
// now we train our object on a few samples of the sinc function.
sample_type m;
for (double x = -10; x <= 5; x += 0.6)
{
m(0) = x;
test.train(m, sinc(x));
samples.push_back(m);
samples2.push_back(m);
labels.push_back(sinc(x));
labels2.push_back(2);
}
print_spinner();
decision_function<kernel_type> test2 = rvm_test.train(samples, labels);
print_spinner();
decision_function<kernel_type> test3 = rbf_test.train(samples, labels);
print_spinner();
decision_function<kernel_type> test4 = krr_test.train(samples, labels);
print_spinner();
decision_function<kernel_type> test5 = svr_test.train(samples, labels);
print_spinner();
// now we output the value of the sinc function for a few test points as well as the
// value predicted by krls object.
m(0) = 2.5; dlog << LDEBUG << "krls: " << sinc(m(0)) << " " << test(m); DLIB_TEST(abs(sinc(m(0)) - test(m)) < 0.01);
m(0) = 0.1; dlog << LDEBUG << "krls: " << sinc(m(0)) << " " << test(m); DLIB_TEST(abs(sinc(m(0)) - test(m)) < 0.01);
m(0) = -4; dlog << LDEBUG << "krls: " << sinc(m(0)) << " " << test(m); DLIB_TEST(abs(sinc(m(0)) - test(m)) < 0.01);
m(0) = 5.0; dlog << LDEBUG << "krls: " << sinc(m(0)) << " " << test(m); DLIB_TEST(abs(sinc(m(0)) - test(m)) < 0.01);
m(0) = 2.5; dlog << LDEBUG << "rvm: " << sinc(m(0)) << " " << test2(m); DLIB_TEST(abs(sinc(m(0)) - test2(m)) < 0.01);
m(0) = 0.1; dlog << LDEBUG << "rvm: " << sinc(m(0)) << " " << test2(m); DLIB_TEST(abs(sinc(m(0)) - test2(m)) < 0.01);
m(0) = -4; dlog << LDEBUG << "rvm: " << sinc(m(0)) << " " << test2(m); DLIB_TEST(abs(sinc(m(0)) - test2(m)) < 0.01);
m(0) = 5.0; dlog << LDEBUG << "rvm: " << sinc(m(0)) << " " << test2(m); DLIB_TEST(abs(sinc(m(0)) - test2(m)) < 0.01);
m(0) = 2.5; dlog << LDEBUG << "rbf: " << sinc(m(0)) << " " << test3(m); DLIB_TEST(abs(sinc(m(0)) - test3(m)) < 0.01);
m(0) = 0.1; dlog << LDEBUG << "rbf: " << sinc(m(0)) << " " << test3(m); DLIB_TEST(abs(sinc(m(0)) - test3(m)) < 0.01);
m(0) = -4; dlog << LDEBUG << "rbf: " << sinc(m(0)) << " " << test3(m); DLIB_TEST(abs(sinc(m(0)) - test3(m)) < 0.01);
m(0) = 5.0; dlog << LDEBUG << "rbf: " << sinc(m(0)) << " " << test3(m); DLIB_TEST(abs(sinc(m(0)) - test3(m)) < 0.01);
m(0) = 2.5; dlog << LDEBUG << "krr: " << sinc(m(0)) << " " << test4(m); DLIB_TEST(abs(sinc(m(0)) - test4(m)) < 0.01);
m(0) = 0.1; dlog << LDEBUG << "krr: " << sinc(m(0)) << " " << test4(m); DLIB_TEST(abs(sinc(m(0)) - test4(m)) < 0.01);
m(0) = -4; dlog << LDEBUG << "krr: " << sinc(m(0)) << " " << test4(m); DLIB_TEST(abs(sinc(m(0)) - test4(m)) < 0.01);
m(0) = 5.0; dlog << LDEBUG << "krr: " << sinc(m(0)) << " " << test4(m); DLIB_TEST(abs(sinc(m(0)) - test4(m)) < 0.01);
m(0) = 2.5; dlog << LDEBUG << "svr: " << sinc(m(0)) << " " << test5(m); DLIB_TEST(abs(sinc(m(0)) - test5(m)) < 0.01);
m(0) = 0.1; dlog << LDEBUG << "svr: " << sinc(m(0)) << " " << test5(m); DLIB_TEST(abs(sinc(m(0)) - test5(m)) < 0.01);
m(0) = -4; dlog << LDEBUG << "svr: " << sinc(m(0)) << " " << test5(m); DLIB_TEST(abs(sinc(m(0)) - test5(m)) < 0.01);
m(0) = 5.0; dlog << LDEBUG << "svr: " << sinc(m(0)) << " " << test5(m); DLIB_TEST(abs(sinc(m(0)) - test5(m)) < 0.01);
randomize_samples(samples, labels);
dlog << LINFO << "KRR MSE and R-squared: "<< cross_validate_regression_trainer(krr_test, samples, labels, 6);
dlog << LINFO << "SVR MSE and R-squared: "<< cross_validate_regression_trainer(svr_test, samples, labels, 6);
matrix<double,1,2> cv = cross_validate_regression_trainer(krr_test, samples, labels, 6);
DLIB_TEST(cv(0) < 1e-4);
DLIB_TEST(cv(1) > 0.99);
cv = cross_validate_regression_trainer(svr_test, samples, labels, 6);
DLIB_TEST(cv(0) < 1e-4);
DLIB_TEST(cv(1) > 0.99);
randomize_samples(samples2, labels2);
dlog << LINFO << "KRR MSE and R-squared: "<< cross_validate_regression_trainer(krr_test, samples2, labels2, 6);
dlog << LINFO << "SVR MSE and R-squared: "<< cross_validate_regression_trainer(svr_test, samples2, labels2, 6);
cv = cross_validate_regression_trainer(krr_test, samples2, labels2, 6);
DLIB_TEST(cv(0) < 1e-4);
cv = cross_validate_regression_trainer(svr_test, samples2, labels2, 6);
DLIB_TEST(cv(0) < 1e-4);
dlog << LINFO << " end test_regression()";
}
// ----------------------------------------------------------------------------------------
void test_anomaly_detection (
)
{
dlog << LINFO << " begin test_anomaly_detection()";
// Here we declare that our samples will be 2 dimensional column vectors.
typedef matrix<double,2,1> sample_type;
// Now we are making a typedef for the kind of kernel we want to use. I picked the
// radial basis kernel because it only has one parameter and generally gives good
// results without much fiddling.
typedef radial_basis_kernel<sample_type> kernel_type;
// Here we declare an instance of the kcentroid object. The first argument to the constructor
// is the kernel we wish to use. The second is a parameter that determines the numerical
// accuracy with which the object will perform part of the learning algorithm. Generally
// smaller values give better results but cause the algorithm to run slower. You just have
// to play with it to decide what balance of speed and accuracy is right for your problem.
// Here we have set it to 0.01.
kcentroid<kernel_type> test(kernel_type(0.1),0.01);
svm_one_class_trainer<kernel_type> one_class_trainer;
one_class_trainer.set_nu(0.4);
one_class_trainer.set_kernel(kernel_type(0.2));
std::vector<sample_type> samples;
// now we train our object on a few samples of the sinc function.
sample_type m;
for (double x = -15; x <= 8; x += 1)
{
m(0) = x;
m(1) = sinc(x);
test.train(m);
samples.push_back(m);
}
decision_function<kernel_type> df = one_class_trainer.train(samples);
running_stats<double> rs;
// Now lets output the distance from the centroid to some points that are from the sinc function.
// These numbers should all be similar. We will also calculate the statistics of these numbers
// by accumulating them into the running_stats object called rs. This will let us easily
// find the mean and standard deviation of the distances for use below.
dlog << LDEBUG << "Points that are on the sinc function:\n";
m(0) = -1.5; m(1) = sinc(m(0)); dlog << LDEBUG << " " << test(m); rs.add(test(m));
m(0) = -1.5; m(1) = sinc(m(0)); dlog << LDEBUG << " " << test(m); rs.add(test(m));
m(0) = -0; m(1) = sinc(m(0)); dlog << LDEBUG << " " << test(m); rs.add(test(m));
m(0) = -0.5; m(1) = sinc(m(0)); dlog << LDEBUG << " " << test(m); rs.add(test(m));
m(0) = -4.1; m(1) = sinc(m(0)); dlog << LDEBUG << " " << test(m); rs.add(test(m));
m(0) = -1.5; m(1) = sinc(m(0)); dlog << LDEBUG << " " << test(m); rs.add(test(m));
m(0) = -0.5; m(1) = sinc(m(0)); dlog << LDEBUG << " " << test(m); rs.add(test(m));
m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
m(0) = -0; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
m(0) = -0.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
m(0) = -4.1; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
m(0) = -0.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
const double thresh = 0.01;
m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
m(0) = -0; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
m(0) = -0.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
m(0) = -4.1; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
m(0) = -0.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
dlog << LDEBUG;
// Lets output the distance from the centroid to some points that are NOT from the sinc function.
// These numbers should all be significantly bigger than previous set of numbers. We will also
// use the rs.scale() function to find out how many standard deviations they are away from the
// mean of the test points from the sinc function. So in this case our criterion for "significantly bigger"
// is > 3 or 4 standard deviations away from the above points that actually are on the sinc function.
dlog << LDEBUG << "Points that are NOT on the sinc function:\n";
m(0) = -1.5; m(1) = sinc(m(0))+4;
dlog << LDEBUG << " " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
DLIB_TEST_MSG(df(m) + thresh < 0, df(m));
m(0) = -1.5; m(1) = sinc(m(0))+3;
dlog << LDEBUG << " " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
DLIB_TEST_MSG(df(m) + thresh < 0, df(m));
m(0) = -0; m(1) = -sinc(m(0));
dlog << LDEBUG << " " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
DLIB_TEST_MSG(df(m) + thresh < 0, df(m));
m(0) = -0.5; m(1) = -sinc(m(0));
dlog << LDEBUG << " " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
DLIB_TEST_MSG(df(m) + thresh < 0, df(m));
m(0) = -4.1; m(1) = sinc(m(0))+2;
dlog << LDEBUG << " " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
DLIB_TEST_MSG(df(m) + thresh < 0, df(m));
m(0) = -1.5; m(1) = sinc(m(0))+0.9;
dlog << LDEBUG << " " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
DLIB_TEST_MSG(df(m) + thresh < 0, df(m));
m(0) = -0.5; m(1) = sinc(m(0))+1;
dlog << LDEBUG << " " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
DLIB_TEST_MSG(df(m) + thresh < 0, df(m));
dlog << LINFO << " end test_anomaly_detection()";
}
// ----------------------------------------------------------------------------------------
void unittest_binary_classification (
)
/*!
ensures
- runs tests on the svm stuff compliance with the specs
!*/
{
dlog << LINFO << " begin unittest_binary_classification()";
print_spinner();
typedef double scalar_type;
typedef matrix<scalar_type,2,1> sample_type;
std::vector<sample_type> x;
std::vector<matrix<double,0,1> > x_linearized;
std::vector<scalar_type> y;
get_checkerboard_problem(x,y, 300, 2);
const scalar_type gamma = 1;
typedef radial_basis_kernel<sample_type> kernel_type;
rbf_network_trainer<kernel_type> rbf_trainer;
rbf_trainer.set_kernel(kernel_type(gamma));
rbf_trainer.set_num_centers(100);
rvm_trainer<kernel_type> rvm_trainer;
rvm_trainer.set_kernel(kernel_type(gamma));
krr_trainer<kernel_type> krr_trainer;
krr_trainer.use_classification_loss_for_loo_cv();
krr_trainer.set_kernel(kernel_type(gamma));
svm_pegasos<kernel_type> pegasos_trainer;
pegasos_trainer.set_kernel(kernel_type(gamma));
pegasos_trainer.set_lambda(0.00001);
svm_c_ekm_trainer<kernel_type> ocas_ekm_trainer;
ocas_ekm_trainer.set_kernel(kernel_type(gamma));
ocas_ekm_trainer.set_c(100000);
svm_nu_trainer<kernel_type> trainer;
trainer.set_kernel(kernel_type(gamma));
trainer.set_nu(0.05);
svm_c_trainer<kernel_type> c_trainer;
c_trainer.set_kernel(kernel_type(gamma));
c_trainer.set_c(100);
svm_c_linear_trainer<linear_kernel<matrix<double,0,1> > > lin_trainer;
lin_trainer.set_c(100000);
// use an ekm to linearize this dataset so we can use it with the lin_trainer
empirical_kernel_map<kernel_type> ekm;
ekm.load(kernel_type(gamma), x);
for (unsigned long i = 0; i < x.size(); ++i)
x_linearized.push_back(ekm.project(x[i]));
print_spinner();
matrix<scalar_type> rvm_cv = cross_validate_trainer_threaded(rvm_trainer, x,y, 4, 2);
print_spinner();
matrix<scalar_type> krr_cv = cross_validate_trainer_threaded(krr_trainer, x,y, 4, 2);
print_spinner();
matrix<scalar_type> svm_cv = cross_validate_trainer(trainer, x,y, 4);
print_spinner();
matrix<scalar_type> svm_c_cv = cross_validate_trainer(c_trainer, x,y, 4);
print_spinner();
matrix<scalar_type> rbf_cv = cross_validate_trainer_threaded(rbf_trainer, x,y, 10, 2);
print_spinner();
matrix<scalar_type> lin_cv = cross_validate_trainer_threaded(lin_trainer, x_linearized, y, 4, 2);
print_spinner();
matrix<scalar_type> ocas_ekm_cv = cross_validate_trainer_threaded(ocas_ekm_trainer, x, y, 4, 2);
print_spinner();
ocas_ekm_trainer.set_basis(randomly_subsample(x, 300));
matrix<scalar_type> ocas_ekm_cv2 = cross_validate_trainer_threaded(ocas_ekm_trainer, x, y, 4, 2);
print_spinner();
matrix<scalar_type> peg_cv = cross_validate_trainer_threaded(batch(pegasos_trainer,1.0), x,y, 4, 2);
print_spinner();
matrix<scalar_type> peg_c_cv = cross_validate_trainer_threaded(batch_cached(pegasos_trainer,1.0), x,y, 4, 2);
print_spinner();
dlog << LDEBUG << "rvm cv: " << rvm_cv;
dlog << LDEBUG << "krr cv: " << krr_cv;
dlog << LDEBUG << "nu-svm cv: " << svm_cv;
dlog << LDEBUG << "C-svm cv: " << svm_c_cv;
dlog << LDEBUG << "rbf cv: " << rbf_cv;
dlog << LDEBUG << "lin cv: " << lin_cv;
dlog << LDEBUG << "ocas_ekm cv: " << ocas_ekm_cv;
dlog << LDEBUG << "ocas_ekm cv2: " << ocas_ekm_cv2;
dlog << LDEBUG << "peg cv: " << peg_cv;
dlog << LDEBUG << "peg cached cv: " << peg_c_cv;
// make sure the cached version of pegasos computes the same result
DLIB_TEST_MSG(sum(abs(peg_cv - peg_c_cv)) < std::sqrt(std::numeric_limits<double>::epsilon()),
sum(abs(peg_cv - peg_c_cv)) << " \n" << peg_cv << peg_c_cv );
DLIB_TEST_MSG(mean(rvm_cv) > 0.9, rvm_cv);
DLIB_TEST_MSG(mean(krr_cv) > 0.9, krr_cv);
DLIB_TEST_MSG(mean(svm_cv) > 0.9, svm_cv);
DLIB_TEST_MSG(mean(svm_c_cv) > 0.9, svm_c_cv);
DLIB_TEST_MSG(mean(rbf_cv) > 0.9, rbf_cv);
DLIB_TEST_MSG(mean(lin_cv) > 0.9, lin_cv);
DLIB_TEST_MSG(mean(peg_cv) > 0.9, peg_cv);
DLIB_TEST_MSG(mean(peg_c_cv) > 0.9, peg_c_cv);
DLIB_TEST_MSG(mean(ocas_ekm_cv) > 0.9, ocas_ekm_cv);
DLIB_TEST_MSG(mean(ocas_ekm_cv2) > 0.9, ocas_ekm_cv2);
const long num_sv = trainer.train(x,y).basis_vectors.size();
print_spinner();
const long num_rv = rvm_trainer.train(x,y).basis_vectors.size();
print_spinner();
dlog << LDEBUG << "num sv: " << num_sv;
dlog << LDEBUG << "num rv: " << num_rv;
print_spinner();
ocas_ekm_trainer.clear_basis();
const long num_bv = ocas_ekm_trainer.train(x,y).basis_vectors.size();
dlog << LDEBUG << "num ekm bv: " << num_bv;
DLIB_TEST(num_rv <= 17);
DLIB_TEST_MSG(num_sv <= 45, num_sv);
DLIB_TEST_MSG(num_bv <= 45, num_bv);
decision_function<kernel_type> df = reduced2(trainer, 19).train(x,y);
print_spinner();
matrix<scalar_type> svm_reduced_error = test_binary_decision_function(df, x, y);
print_spinner();
dlog << LDEBUG << "svm reduced test error: " << svm_reduced_error;
dlog << LDEBUG << "svm reduced num sv: " << df.basis_vectors.size();
DLIB_TEST(mean(svm_reduced_error) > 0.9);
svm_cv = cross_validate_trainer(reduced(trainer,30), x,y, 4);
dlog << LDEBUG << "svm reduced cv: " << svm_cv;
DLIB_TEST_MSG(mean(svm_cv) > 0.9, svm_cv);
DLIB_TEST(df.basis_vectors.size() <= 19);
dlog << LINFO << " end unittest_binary_classification()";
}
// ----------------------------------------------------------------------------------------
template <typename kernel_type>
struct kernel_der_obj
{
typename kernel_type::sample_type x;
kernel_type k;
double operator()(const typename kernel_type::sample_type& y) const { return k(x,y); }
};
template <typename kernel_type>
void test_kernel_derivative (
const kernel_type& k,
const typename kernel_type::sample_type& x,
const typename kernel_type::sample_type& y
)
{
kernel_der_obj<kernel_type> obj;
obj.x = x;
obj.k = k;
kernel_derivative<kernel_type> der(obj.k);
DLIB_TEST(dlib::equal(derivative(obj)(y) , der(obj.x,y), 1e-5));
}
void test_kernel_derivative (
)
{
typedef matrix<double, 2, 1> sample_type;
sigmoid_kernel<sample_type> k1;
radial_basis_kernel<sample_type> k2;
linear_kernel<sample_type> k3;
polynomial_kernel<sample_type> k4(2,3,4);
offset_kernel<sigmoid_kernel<sample_type> > k5;
offset_kernel<radial_basis_kernel<sample_type> > k6;
dlib::rand rnd;
sample_type x, y;
for (int i = 0; i < 10; ++i)
{
x = randm(2,1,rnd);
y = randm(2,1,rnd);
test_kernel_derivative(k1, x, y);
test_kernel_derivative(k2, x, y);
test_kernel_derivative(k3, x, y);
test_kernel_derivative(k4, x, y);
test_kernel_derivative(k5, x, y);
test_kernel_derivative(k6, x, y);
}
}
// ----------------------------------------------------------------------------------------
void test_svm_trainer2()
{
typedef matrix<double, 2, 1> sample_type;
typedef linear_kernel<sample_type> kernel_type;
std::vector<sample_type> samples;
std::vector<double> labels;
sample_type samp;
samp(0) = 1;
samp(1) = 1;
samples.push_back(samp);
labels.push_back(+1);
samp(0) = 1;
samp(1) = 2;
samples.push_back(samp);
labels.push_back(-1);
svm_c_trainer<kernel_type> trainer;
decision_function<kernel_type> df = trainer.train(samples, labels);
samp(0) = 1;
samp(1) = 1;
dlog << LINFO << "test +1 : "<< df(samp);
DLIB_TEST(df(samp) > 0);
samp(0) = 1;
samp(1) = 2;
dlog << LINFO << "test -1 : "<< df(samp);
DLIB_TEST(df(samp) < 0);
svm_nu_trainer<kernel_type> trainer2;
df = trainer2.train(samples, labels);
samp(0) = 1;
samp(1) = 1;
dlog << LINFO << "test +1 : "<< df(samp);
DLIB_TEST(df(samp) > 0);
samp(0) = 1;
samp(1) = 2;
dlog << LINFO << "test -1 : "<< df(samp);
DLIB_TEST(df(samp) < 0);
}
// ----------------------------------------------------------------------------------------
class svm_tester : public tester
{
public:
svm_tester (
) :
tester ("test_svm",
"Runs tests on the svm/kernel algorithm components.")
{}
void perform_test (
)
{
test_kernel_derivative();
unittest_binary_classification();
test_clutering();
test_regression();
test_anomaly_detection();
test_svm_trainer2();
}
} a;
}