From 209cfd9fe0ea398b794d1d1995629a826125f035 Mon Sep 17 00:00:00 2001 From: Peng Li Date: Fri, 20 Jul 2018 22:57:45 +0800 Subject: [PATCH] upgrade opencv to 3.3.1, complete patch similarity --- README.md | 10 +++-- SConstruct | 3 +- include/MultiTracker.h | 10 +++-- include/PredictorWrapper.h | 2 +- python/predictor.py | 2 - script/install_opencv.sh | 19 ++++++++++ src/Engine.cpp | 1 - src/MultiTracker.cpp | 91 +++++++++++++++++++++++++++++++++++----------- src/PredictorWrapper.cpp | 15 +++++--- test/TestHungarian.cpp | 38 +++++++++++++++++++ 10 files changed, 153 insertions(+), 38 deletions(-) create mode 100755 script/install_opencv.sh diff --git a/README.md b/README.md index 202ad06..b40c9ee 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,15 @@ Tracker++ cpp version on Linux (arm) ## install dependencies +- opencv - 3.4.1 + + Run the script `script/install_opencv.sh`, the prefix is `/usr/local` + - log4cpp : logger utils -- opencv - eigen : matrix library of C++ +- scons + `apt-get install liblog4cpp5-dev libeigen3-dev scons` + - boost-python - `apt-get install libpython-dev python-dev` - build boost with python @@ -12,10 +18,8 @@ Tracker++ cpp version on Linux (arm) As boost has already - `sudo ./b2 install link=static cxxflags=-fPIC --with-filesystem --with-test --with-log --with-program_options --with-python` -`apt-get install liblog4cpp5-dev libopencv-dev libeigen3-dev` ## Build diff --git a/SConstruct b/SConstruct index b2e7560..75897cf 100644 --- a/SConstruct +++ b/SConstruct @@ -8,7 +8,7 @@ AddOption('--all', dest='all', action='store_true', help='Build all include test env = Environment(CXX="g++", CPPPATH=['#include'], - CCFLAGS=['-Wall', '-std=c++11', '-O3']) + CCFLAGS=['-Wall', '-std=c++11', '-O2']) env['ENV']['TERM'] = os.environ['TERM'] @@ -17,6 +17,7 @@ env.ParseConfig("pkg-config --libs opencv log4cpp") env.ParseConfig("python-config --cflags --libs") env.Append(LIBS = ['pthread', 'boost_python']) env['CCFLAGS'].remove('-Wstrict-prototypes') # invalid in C++ +env['CCFLAGS'].remove('-g') # invalid in C++ env.Append(LIBPATH=['#.']) diff --git a/include/MultiTracker.h b/include/MultiTracker.h index 190225d..58ef922 100644 --- a/include/MultiTracker.h +++ b/include/MultiTracker.h @@ -6,6 +6,7 @@ #include "SharedPtr.h" #include "PredictorWrapper.h" #include +#include namespace suanzi { @@ -23,7 +24,7 @@ namespace suanzi { private: std::vector trackers; int max_id = 0; - PatchPtr createPatch(const cv::Mat& image); + PatchPtr createPatch(const cv::Mat& image, const Detection& d); double distance(TrackerPtr t, const cv::Mat& image, const Detection& d); PredictorWrapperPtr predictor; cv::HOGDescriptor descriptor; @@ -33,9 +34,12 @@ namespace suanzi { { public: ~Patch(){}; - // bb_ltrb cv::Mat image_crop; - std::vector features; + //std::vector features; + // hog is a hog descriptor of the image (calculated by hog.compute. size is 3780 ) + // hue is a histogram of the image, (calcHist(), is a Mat with (width x height) 64 x 45) + std::pair, cv::Mat> features; + Detection detection; protected: friend class MultiTracker; Patch(){}; diff --git a/include/PredictorWrapper.h b/include/PredictorWrapper.h index 9ceb21a..41f818f 100644 --- a/include/PredictorWrapper.h +++ b/include/PredictorWrapper.h @@ -17,7 +17,7 @@ namespace suanzi { static PredictorWrapperPtr create(const std::string& fname); ~PredictorWrapper(){} void dump(); - double predict(int index, std::vector f); + double predict(int index, const std::vector& f); private: PredictorWrapper(const std::string& fname); diff --git a/python/predictor.py b/python/predictor.py index 5dc12af..2b77aac 100644 --- a/python/predictor.py +++ b/python/predictor.py @@ -19,8 +19,6 @@ def dump(): for i in predictors: ss += str(i.__dict__) ss += '\n' -# ss += str(i.coef_) -# ss += '\n' return ss diff --git a/script/install_opencv.sh b/script/install_opencv.sh new file mode 100755 index 0000000..757b33c --- /dev/null +++ b/script/install_opencv.sh @@ -0,0 +1,19 @@ +#!/bin/bash -xe + +echo "Download and build opencv from source" +cd $(mktemp -d) +wget https://github.com/opencv/opencv/archive/3.4.1.tar.gz || exit 1 + +echo "Build" +tar zxvf 3.4.1.tar.gz || exit 1 + +cd opencv-3.4.1 && mkdir build && cd build || exit 1 +cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/usr/local .. || exit 1 + +make -j4 || exit 1 + +sudo make install || exit 1 + +echo "Install opencv to /usr/local/ Done" + +sudo ldconfig diff --git a/src/Engine.cpp b/src/Engine.cpp index a1defdb..74659a0 100644 --- a/src/Engine.cpp +++ b/src/Engine.cpp @@ -58,7 +58,6 @@ void Engine::run() Detection detections[128]; while (reader->read(frame)){ LOG_DEBUG(TAG, "Size: " << frame.cols << "x" << frame.rows); - // TODO int total = detector->detect(frame, detections); multiTracker->update(total, detections, frame); } diff --git a/src/MultiTracker.cpp b/src/MultiTracker.cpp index 267ba0f..b1f380e 100644 --- a/src/MultiTracker.cpp +++ b/src/MultiTracker.cpp @@ -6,7 +6,7 @@ using namespace suanzi; using namespace cv; -using namespace Eigen; +using namespace std; static const std::string TAG = "MultiTracker"; static const cv::Size PREFERRED_SIZE = Size(64, 128); @@ -15,16 +15,18 @@ static const cv::Size PREFERRED_SIZE = Size(64, 128); MultiTracker::MultiTracker() { - LOG_DEBUG(TAG, "init - load model.pkl"); + LOG_DEBUG(TAG, "init - loading model.pkl"); predictor = PredictorWrapper::create("./python/model.pkl"); predictor->dump(); - this->descriptor = {cv::Size(64, 128), cv::Size(16, 16), cv::Size(8, 8), cv::Size(8, 8), 9}; + this->descriptor = {Size(64, 128), Size(16, 16), Size(8, 8), Size(8, 8), 9}; + std::vector ff (40, 1); - predictor->predict(4, ff); + double prob = predictor->predict(4, ff); } MultiTracker::~MultiTracker() { + predictor.reset(); trackers.clear(); } @@ -42,28 +44,49 @@ static std::vector similarity(const PatchPtr p1, const PatchPtr p2) cv::matchTemplate(im1, im2, result, CV_TM_CCORR_NORMED); feature.push_back(result.at(0, 0)); + + vector& f1_hog = p1->features.first; Mat f1_hue = p1->features.second; + vector& f2_hog = p1->features.first; Mat f2_hue = p1->features.second; + feature.push_back(distance_cosine(Eigen::Map(f1_hog.data(), f1_hog.size()), + Eigen::Map(f2_hog.data(), f2_hog.size()))); + feature.push_back(distance_euclidean(Eigen::Map(f1_hog.data(), f1_hog.size()), + Eigen::Map(f2_hog.data(), f2_hog.size()))); + feature.push_back(compareHist(f1_hue, f2_hue, HISTCMP_CORREL)); + feature.push_back(compareHist(f1_hue, f2_hue, HISTCMP_HELLINGER)); + + Detection& d1 = p1->detection; + Detection& d2 = p2->detection; + + double center_distance = sqrt(pow((d1.center_x - d2.center_x), 2) + pow((d1.center_y - d2.center_y), 2)); + feature.push_back(center_distance / (d1.width + d1.height + d2.width + d2.height) * 4); + + //TODO + double iou_ratio = 0.03; + feature.push_back(iou_ratio); + return feature; } -double MultiTracker::distance(TrackerPtr t, const cv::Mat& image, const Detection& d) +double MultiTracker::distance(TrackerPtr tracker, const cv::Mat& image, const Detection& d) { - PatchPtr patch = createPatch(image); + PatchPtr patch = createPatch(image, d); std::vector features; + std::vector ss; - for (const auto i : t->patches){ + for (auto i : tracker->patches){ ss = similarity(i, patch); features.insert(features.end(), ss.begin(), ss.end()); } - //predictor->predict(); - return 0.1; + double prob = predictor->predict(4, features); + return prob; } void MultiTracker::update(unsigned int total, const Detection* detections, const Mat& image) { int row = trackers.size(); int col = total; - MatrixXi cost_matrix = MatrixXi::Zero(row, col); + Eigen::MatrixXi cost_matrix = Eigen::MatrixXi::Zero(row, col); for (int i = 0; i < row; i++){ for (int j = 0; j < col; j++){ // TODO @@ -72,7 +95,7 @@ void MultiTracker::update(unsigned int total, const Detection* detections, const } // assignment - VectorXi tracker_inds, bb_inds; + Eigen::VectorXi tracker_inds, bb_inds; linear_sum_assignment(cost_matrix, tracker_inds, bb_inds); // handle unmatched trackers @@ -86,7 +109,6 @@ void MultiTracker::update(unsigned int total, const Detection* detections, const t->updateState(image); } - // handle unmatched detections vector unmatched_detection; for(int j = 0; j < col; j++){ @@ -99,18 +121,45 @@ void MultiTracker::update(unsigned int total, const Detection* detections, const TrackerPtr t (new Tracker(image)); this->trackers.push_back(t); } + + Detection dd; + + PatchPtr pp = createPatch(image, dd); } -PatchPtr MultiTracker::createPatch(const cv::Mat& image) +// Get image crop from input image within given bounding box - Detecinon +static cv::Mat image_crop(const cv::Mat& image, const Detection& bb) +{ + // RECT + // TODO; + return image.clone(); +} + +PatchPtr MultiTracker::createPatch(const Mat& image, const Detection& detect) { PatchPtr patch(new Patch()); - std::vector feature_hog; - cv::Mat img; - cv::resize(image, img, PREFERRED_SIZE); - this->descriptor.compute(img, feature_hog); - - cv::Mat hist; - //cv::calcHist() - patch->image_crop = image; + + // calculate hog descriptors, size is 3780 + Mat im, im2; + im = image_crop(image, detect); + resize(im, im2, PREFERRED_SIZE); + vector feature_hog; + this->descriptor.compute(im2, feature_hog); + + // calculate histogram, size is (64 x 45) + Mat hsv, hist; + cvtColor(im, hsv, COLOR_BGR2HSV); + int channels[] = {0, 1}; + int histSize[] = {45, 64}; + float hranges[] = {0, 180}; + float sranges[] = {0, 256}; + const float* ranges[] = {hranges, sranges}; + calcHist(&hsv, 1, channels, Mat(), hist, 2, histSize, ranges, true, false); + Size sm = hist.size(); + + patch->image_crop = im.clone(); + patch->detection = detect; + std::vector feature_hog_double (feature_hog.begin(), feature_hog.end()); // convert to double + patch->features = std::make_pair(feature_hog_double, hist); return patch; } diff --git a/src/PredictorWrapper.cpp b/src/PredictorWrapper.cpp index 20fefc3..2dc7eb4 100644 --- a/src/PredictorWrapper.cpp +++ b/src/PredictorWrapper.cpp @@ -18,11 +18,11 @@ const static std::string TAG = "PredictorWrapper"; static std::string parse_python_exception(); template -boost::python::list toPythonList(std::vector vector) { +boost::python::list toPythonList(const std::vector& v) { typename std::vector::iterator iter; boost::python::list list; - for (iter = vector.begin(); iter != vector.end(); ++iter) { - list.append(*iter); + for(const auto& vv : v){ + list.append(vv); } return list; } @@ -51,16 +51,19 @@ void PredictorWrapper::dump() LOG_DEBUG(TAG, ss); } -double PredictorWrapper::predict(int index, std::vector ff) +double PredictorWrapper::predict(int index, const std::vector& ff) { LOG_DEBUG(TAG, "predict"); + py::object ret; try{ - this->predict_func(index, toPythonList(ff)); + ret = this->predict_func(index, toPythonList(ff)); } catch (boost::python::error_already_set const &){ std::string perror_str = parse_python_exception(); LOG_ERROR(TAG, "Error in Python: " + perror_str) } - return 0.1; + double rr = py::extract(ret); + LOG_DEBUG(TAG, "return: " + std::to_string(rr)); + return rr; } PredictorWrapper::PredictorWrapper(const std::string& fname) diff --git a/test/TestHungarian.cpp b/test/TestHungarian.cpp index 3d6173f..55620a9 100644 --- a/test/TestHungarian.cpp +++ b/test/TestHungarian.cpp @@ -1,6 +1,7 @@ #include "hungarian.h" #include "gtest/gtest.h" #include +#include using namespace std; using namespace Eigen; @@ -46,6 +47,15 @@ TEST(Hungarian, 4x3) EXPECT_TRUE(expect_col_ind == col_ind); } +TEST(Hungarian, 0x0) +{ + MatrixXi C = MatrixXi::Zero(0, 0); + VectorXi row_ind, col_ind; + int ret = linear_sum_assignment(C, row_ind, col_ind); + EXPECT_EQ(ret, 0); +} + + TEST(Distance, consine) { Vector3d u, v; @@ -78,3 +88,31 @@ TEST(Distance, euclidean) d = distance_euclidean(u, v); EXPECT_DOUBLE_EQ(d, 1.0); } + +TEST(Distance, vector) +{ + std::vector sv = {1, 2, 3, 4, 5, 6}; + VectorXi v1; + VectorXi b = Eigen::Map(sv.data(), sv.size()); + std::cout << b << std::endl; + std::vector f1_hog = { 0.1, 0.2, 0,3}; +// Eigen::Map(f2_hog.data(), f2_hog.size()) + + //VectorXd mf = Map >(sv.data(), sv.size()); + std::vector sd = {1, 2, 3, 4, 5, 6}; + VectorXd mm = Map(sd.data(), sd.size()); + VectorXd xd = Map >(sd.data(), sd.size()); + cout << Map >(sd.data(), sd.size()) << endl; + + int array[12]; + for(int i = 0; i < 12; ++i) array[i] = i; + cout << Map >(sv.data(), sv.size()) // the inner stride has already been passed as template parameter + << endl; + + + //Vector3d v = Vector3d::Random(); + //std::cout << v << std::endl; + +} + + -- 2.11.0