// Copyright (C) 2013 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#undef DLIB_SCAN_IMAGE_CuSTOM_ABSTRACT_Hh_
#ifdef DLIB_SCAN_IMAGE_CuSTOM_ABSTRACT_Hh_
#include <vector>
#include "../matrix.h"
#include "../geometry.h"
#include "../image_processing/full_object_detection_abstract.h"
namespace dlib
{
// ----------------------------------------------------------------------------------------
class example_feature_extractor
{
/*!
WHAT THIS OBJECT REPRESENTS
This object defines the interface a feature extractor must implement if it
is to be used with the scan_image_custom object defined at the bottom of
this file.
In this case, the purpose of a feature extractor is to associated a
complete feature vector with each rectangle in an image. In particular,
each rectangle is scored by taking the dot product between this feature
vector and a weight vector. If this score is greater than a threshold then
the rectangle is output as a detection.
!*/
public:
template <
typename image_type
>
void load (
const image_type& image,
std::vector<rectangle>& candidate_objects
);
/*!
ensures
- Loads the given image into this feature extractor. This means that
subsequent calls to get_feature_vector() will return the feature vector
corresponding to locations in the image given to load().
- #candidate_objects == a set of bounding boxes in the given image that
might contain objects of interest. These are the locations that will be
checked for the presents of objects when this feature extractor is used
with the scan_image_custom object.
!*/
void copy_configuration (
const feature_extractor& item
);
/*!
ensures
- Copies all the state information of item into *this, except for state
information populated by load(). More precisely, given two
feature extractor objects S1 and S2, the following sequence of
instructions should always result in both of them having the exact same
state:
S2.copy_configuration(S1);
S1.load(img, temp);
S2.load(img, temp);
!*/
unsigned long get_num_dimensions (
) const;
/*!
ensures
- returns the dimensionality of the feature vectors output by this object.
!*/
void get_feature_vector (
const rectangle& obj,
matrix<double,0,1>& psi
) const;
/*!
requires
- psi.size() >= get_num_dimensions()
(i.e. psi must have preallocated its memory before this function is called)
ensures
- This function computes the feature vector associated with the given rectangle
in obj. This rectangle is interpreted as a bounding box within the last image
given to this->load() and a feature vector describing that bounding box is
output into psi.
- The feature vector is added into psi. That is, it does not overwrite the
previous contents of psi, but instead, it adds the vector to psi.
- The dimensionality of the vector added to psi is get_num_dimensions(). This
means that elements of psi after psi(get_num_dimensions()-1) are not modified.
- #psi.size() == psi.size()
(i.e. this function does not change the size of the psi vector)
!*/
double compute_object_score (
const matrix<double,0,1>& w,
const rectangle& obj
) const;
/*!
requires
- w.size() >= get_num_dimensions()
ensures
- This function returns the dot product between the feature vector for
object box obj and the given w vector. That is, this function computes
the same number as the following code snippet:
matrix<double,0,1> psi(w.size());
psi = 0;
get_feature_vector(obj, psi);
return dot(psi, w);
The point of the compute_object_score() routine is to compute this dot
product in a much more efficient way than directly calling
get_feature_vector() and dot(). Therefore, compute_object_score() is an
optional function. If you can't think of a faster way to compute these
scores then do not implement compute_object_score() and the
scan_image_custom object will simply compute these scores for you.
However, it is often the case that there is something clever you can do
to make this computation faster. If that is the case, then you can
provide an implementation of this function with your feature extractor
and then scan_image_custom will use it instead of using the default
calculation method shown in the above code snippet.
!*/
};
// ----------------------------------------------------------------------------------------
void serialize(
const feature_extractor& item,
std::ostream& out
);
/*!
provides serialization support
!*/
void deserialize(
feature_extractor& item,
std::istream& in
);
/*!
provides deserialization support
!*/
// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------
template <
typename Feature_extractor_type
>
class scan_image_custom : noncopyable
{
/*!
REQUIREMENTS ON Feature_extractor_type
- must be an object with an interface compatible with the
example_feature_extractor defined at the top of this file.
INITIAL VALUE
- is_loaded_with_image() == false
WHAT THIS OBJECT REPRESENTS
This object is a tool for running a classifier over an image with the goal
of localizing each object present. The localization is in the form of the
bounding box around each object of interest.
Unlike the scan_image_pyramid and scan_image_boxes objects, this image
scanner delegates all the work of constructing the object feature vector to
its Feature_extractor_type template argument. That is, scan_image_custom
simply asks the supplied feature extractor what boxes in the image we
should investigate and then asks the feature extractor for the complete
feature vector for each box. That is, scan_image_custom does not apply any
kind of pyramiding or other higher level processing to the features coming
out of the feature extractor. That means that when you use
scan_image_custom it is completely up to you to define the feature vector
used with each image box.
THREAD SAFETY
Concurrent access to an instance of this object is not safe and should be
protected by a mutex lock except for the case where you are copying the
configuration (via copy_configuration()) of a scan_image_custom object to
many other threads. In this case, it is safe to copy the configuration of
a shared object so long as no other operations are performed on it.
!*/
public:
typedef matrix<double,0,1> feature_vector_type;
typedef Feature_extractor_type feature_extractor_type;
scan_image_custom (
);
/*!
ensures
- this object is properly initialized
!*/
template <
typename image_type
>
void load (
const image_type& img
);
/*!
requires
- image_type must be a type with the following properties:
- image_type objects can be loaded into Feature_extractor_type
objects via Feature_extractor_type::load().
ensures
- #is_loaded_with_image() == true
- Calls get_feature_extractor().load() on the given image. That is, we
will have loaded the image into the feature extractor in this
scan_image_custom object. We will also have stored the candidate
object locations generated by the feature extractor and will scan
over them when this->detect() is called.
- This object is ready to run a classifier over img to detect object
locations. Call detect() to do this.
!*/
bool is_loaded_with_image (
) const;
/*!
ensures
- returns true if this object has been loaded with an image to process and
false otherwise.
!*/
const feature_extractor_type& get_feature_extractor (
) const;
/*!
ensures
- returns a const reference to the feature_extractor_type object used
internally for local feature extraction.
!*/
void copy_configuration(
const feature_extractor_type& fe
);
/*!
ensures
- This function performs the equivalent of
get_feature_extractor().copy_configuration(fe) (i.e. this function allows
you to configure the parameters of the underlying feature extractor used
by a scan_image_custom object)
!*/
void copy_configuration (
const scan_image_custom& item
);
/*!
ensures
- Copies all the state information of item into *this, except for state
information populated by load(). More precisely, given two
scan_image_custom objects S1 and S2, the following sequence of
instructions should always result in both of them having the exact same
state:
S2.copy_configuration(S1);
S1.load(img);
S2.load(img);
!*/
long get_num_dimensions (
) const;
/*!
ensures
- returns the number of dimensions in the feature vector for a candidate
object location. That is, this function returns get_feature_extractor().get_num_dimensions().
!*/
void detect (
const feature_vector_type& w,
std::vector<std::pair<double, rectangle> >& dets,
const double thresh
) const;
/*!
requires
- w.size() >= get_num_dimensions()
- is_loaded_with_image() == true
ensures
- Scans over all the candidate object locations produced by the feature
extractor during image loading and stores all detections into #dets.
- for all valid i:
- #dets[i].second == The candidate object location which produced this
detection. This rectangle gives the location of the detection.
- #dets[i].first == The score for this detection. This value is equal
to dot(w, feature vector for this candidate object location).
- #dets[i].first >= thresh
- #dets will be sorted in descending order.
(i.e. #dets[i].first >= #dets[j].first for all i, and j>i)
- Elements of w beyond index get_num_dimensions()-1 are ignored. I.e. only
the first get_num_dimensions() are used.
- Note that no form of non-max suppression is performed. If a locations
has a score >= thresh then it is reported in #dets.
!*/
void get_feature_vector (
const full_object_detection& obj,
feature_vector_type& psi
) const;
/*!
requires
- obj.num_parts() == 0
- is_loaded_with_image() == true
- psi.size() >= get_num_dimensions()
(i.e. psi must have preallocated its memory before this function is called)
ensures
- This function allows you to determine the feature vector used for a
candidate object location output from detect(). Note that this vector is
added to psi. Note also that you must use get_full_object_detection() to
convert a rectangle from detect() into the needed full_object_detection.
- The dimensionality of the vector added to psi is get_num_dimensions(). This
means that elements of psi after psi(get_num_dimensions()-1) are not modified.
- Since scan_image_custom only searches a limited set of object locations,
not all possible rectangles can be output by detect(). So in the case
where obj.get_rect() could not arise from a call to detect(), this
function will map obj.get_rect() to the nearest possible rectangle and
then add the feature vector for the mapped rectangle into #psi.
- get_best_matching_rect(obj.get_rect()) == the rectangle obj.get_rect()
gets mapped to for feature extraction.
!*/
full_object_detection get_full_object_detection (
const rectangle& rect,
const feature_vector_type& w
) const;
/*!
ensures
- returns full_object_detection(rect)
(This function is here only for compatibility with the scan_image_pyramid
object)
!*/
const rectangle get_best_matching_rect (
const rectangle& rect
) const;
/*!
requires
- is_loaded_with_image() == true
ensures
- Since scan_image_custom only searches a limited set of object locations,
not all possible rectangles can be represented. Therefore, this function
allows you to supply a rectangle and obtain the nearest possible
candidate object location rectangle.
!*/
unsigned long get_num_detection_templates (
) const { return 1; }
/*!
ensures
- returns 1. Note that this function is here only for compatibility with
the scan_image_pyramid object. Notionally, its return value indicates
that a scan_image_custom object is always ready to detect objects once an
image has been loaded.
!*/
unsigned long get_num_movable_components_per_detection_template (
) const { return 0; }
/*!
ensures
- returns 0. Note that this function is here only for compatibility with
the scan_image_pyramid object. Its return value means that this object
does not support using movable part models.
!*/
};
// ----------------------------------------------------------------------------------------
template <typename T>
void serialize (
const scan_image_custom<T>& item,
std::ostream& out
);
/*!
provides serialization support
!*/
template <typename T>
void deserialize (
scan_image_custom<T>& item,
std::istream& in
);
/*!
provides deserialization support
!*/
// ----------------------------------------------------------------------------------------
}
#endif // DLIB_SCAN_IMAGE_CuSTOM_ABSTRACT_Hh_