21 #include "objdetectdata.pb.h"
31 ObjectDetection::ObjectDetection(std::string clipObDetectDataPath)
34 init_effect_details();
37 LoadObjDetectdData(clipObDetectDataPath);
40 selectedObjectIndex = trackedObjects.begin()->first;
44 ObjectDetection::ObjectDetection()
47 init_effect_details();
50 selectedObjectIndex = trackedObjects.begin()->first;
54 void ObjectDetection::init_effect_details()
60 info.class_name =
"ObjectDetection";
61 info.name =
"Object Detector";
62 info.description =
"Detect objects through the video.";
63 info.has_audio =
false;
64 info.has_video =
true;
65 info.has_tracked_object =
true;
70 std::shared_ptr<Frame> ObjectDetection::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
73 cv::Mat cv_image = frame->GetImageCV();
81 std::vector<QRectF> boxRects;
83 std::vector<std::shared_ptr<QImage>> childClipImages;
86 if (detectionsData.find(frame_number) != detectionsData.end()) {
87 float fw = cv_image.size().width;
88 float fh = cv_image.size().height;
91 for(
int i = 0; i<detections.
boxes.size(); i++){
94 if(detections.
confidences.at(i) < confidence_threshold){
98 if( display_classes.size() > 0 &&
99 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
104 int objectId = detections.
objectIds.at(i);
107 auto trackedObject_it = trackedObjects.find(objectId);
110 std::shared_ptr<TrackedObjectBBox> trackedObject = std::static_pointer_cast<TrackedObjectBBox>(trackedObject_it->second);
113 if (trackedObject->Contains(frame_number) &&
114 trackedObject->visible.GetValue(frame_number) == 1)
117 BBox trackedBox = trackedObject->GetBox(frame_number);
118 bool draw_text = !display_box_text.GetValue(frame_number);
119 std::vector<int> stroke_rgba = trackedObject->stroke.GetColorRGBA(frame_number);
120 int stroke_width = trackedObject->stroke_width.GetValue(frame_number);
121 float stroke_alpha = trackedObject->stroke_alpha.GetValue(frame_number);
122 std::vector<int> bg_rgba = trackedObject->background.GetColorRGBA(frame_number);
123 float bg_alpha = trackedObject->background_alpha.GetValue(frame_number);
126 (
int)( (trackedBox.
cx-trackedBox.
width/2)*fw),
127 (
int)( (trackedBox.
cy-trackedBox.
height/2)*fh),
128 (
int)( trackedBox.
width*fw),
129 (
int)( trackedBox.
height*fh)
133 if (trackedObject->draw_box.GetValue(frame_number) == 0)
140 box, cv_image, detections.
objectIds.at(i), bg_rgba, bg_alpha, 1,
true, draw_text);
142 box, cv_image, detections.
objectIds.at(i), stroke_rgba, stroke_alpha, stroke_width,
false, draw_text);
146 if (trackedObject->ChildClipId() !=
""){
151 Clip* childClip = parentTimeline->
GetClip(trackedObject->ChildClipId());
155 std::shared_ptr<Frame> childClipFrame = childClip->
GetFrame(frame_number);
156 childClipImages.push_back(childClipFrame->GetImage());
160 boxRect.setRect((
int)((trackedBox.
cx-trackedBox.
width/2)*fw),
161 (int)((trackedBox.
cy - trackedBox.
height/2)*fh),
162 (
int)(trackedBox.
width*fw),
163 (
int)(trackedBox.
height*fh));
164 boxRects.push_back(boxRect);
173 frame->SetImageCV(cv_image);
176 if(boxRects.size() > 0){
178 QImage frameImage = *(frame->GetImage());
179 for(
int i; i < boxRects.size();i++){
181 QPainter painter(&frameImage);
183 painter.drawImage(boxRects[i], *childClipImages[i]);
186 frame->AddImage(std::make_shared<QImage>(frameImage));
192 void ObjectDetection::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color,
float alpha,
193 int thickness,
bool is_background){
195 cv::Point2f vertices2f[4];
196 box.points(vertices2f);
204 cv::Mat overlayFrame;
205 frame_image.copyTo(overlayFrame);
208 cv::Point vertices[4];
209 for(
int i = 0; i < 4; ++i){
210 vertices[i] = vertices2f[i];}
212 cv::Rect rect = box.boundingRect();
213 cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
215 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
218 cv::Mat overlayFrame;
219 frame_image.copyTo(overlayFrame);
222 for (
int i = 0; i < 4; i++)
224 cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
225 thickness, cv::LINE_AA);
229 cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
233 void ObjectDetection::drawPred(
int classId,
float conf, cv::Rect2d box, cv::Mat& frame,
int objectNumber, std::vector<int> color,
234 float alpha,
int thickness,
bool is_background,
bool display_text)
238 cv::Mat overlayFrame;
239 frame.copyTo(overlayFrame);
242 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
245 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
248 cv::Mat overlayFrame;
249 frame.copyTo(overlayFrame);
252 cv::rectangle(overlayFrame, box, cv::Scalar(color[2],color[1],color[0]), thickness);
256 std::string label = cv::format(
"%.2f", conf);
257 if (!classNames.empty())
259 CV_Assert(classId < (
int)classNames.size());
260 label = classNames[classId] +
":" + label;
265 cv::Size labelSize = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
268 double top = std::max((
int)box.y, labelSize.height);
270 cv::rectangle(overlayFrame, cv::Point(left, top - round(1.025*labelSize.height)), cv::Point(left + round(1.025*labelSize.width), top + baseLine),
271 cv::Scalar(color[2],color[1],color[0]), cv::FILLED);
272 putText(overlayFrame, label, cv::Point(left+1, top), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0,0,0),1);
275 cv::addWeighted(overlayFrame, 1-alpha, frame, alpha, 0, frame);
280 bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){
282 pb_objdetect::ObjDetect objMessage;
285 std::fstream input(inputFilePath, std::ios::in | std::ios::binary);
286 if (!objMessage.ParseFromIstream(&input)) {
287 std::cerr <<
"Failed to parse protobuf message." << std::endl;
293 detectionsData.clear();
294 trackedObjects.clear();
299 for(
int i = 0; i < objMessage.classnames_size(); i++)
301 classNames.push_back(objMessage.classnames(i));
302 classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50));
306 for (
size_t i = 0; i < objMessage.frame_size(); i++)
309 const pb_objdetect::Frame& pbFrameData = objMessage.frame(i);
312 size_t id = pbFrameData.id();
315 const google::protobuf::RepeatedPtrField<pb_objdetect::Frame_Box > &pBox = pbFrameData.bounding_box();
318 std::vector<int> classIds;
319 std::vector<float> confidences;
320 std::vector<cv::Rect_<float>> boxes;
321 std::vector<int> objectIds;
324 for(
int i = 0; i < pbFrameData.bounding_box_size(); i++)
327 float x = pBox.Get(i).x();
328 float y = pBox.Get(i).y();
329 float w = pBox.Get(i).w();
330 float h = pBox.Get(i).h();
332 int classId = pBox.Get(i).classid();
334 float confidence = pBox.Get(i).confidence();
337 int objectId = pBox.Get(i).objectid();
340 auto trackedObject = trackedObjects.find(objectId);
342 if (trackedObject != trackedObjects.end())
345 trackedObject->second->AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
350 TrackedObjectBBox trackedObj((
int)classesColor[classId](0), (
int)classesColor[classId](1), (
int)classesColor[classId](2), (
int)0);
351 trackedObj.
AddBox(
id, x+(w/2), y+(h/2), w, h, 0.0);
353 std::shared_ptr<TrackedObjectBBox> trackedObjPtr = std::make_shared<TrackedObjectBBox>(trackedObj);
354 ClipBase* parentClip = this->ParentClip();
355 trackedObjPtr->ParentClip(parentClip);
359 trackedObjPtr->Id(std::to_string(objectId));
360 trackedObjects.insert({objectId, trackedObjPtr});
364 cv::Rect_<float> box(x, y, w, h);
367 boxes.push_back(box);
368 classIds.push_back(classId);
369 confidences.push_back(confidence);
370 objectIds.push_back(objectId);
374 detectionsData[id] =
DetectionData(classIds, confidences, boxes,
id, objectIds);
378 google::protobuf::ShutdownProtobufLibrary();
384 std::string ObjectDetection::GetVisibleObjects(int64_t frame_number)
const{
388 root[
"visible_objects_index"] = Json::Value(Json::arrayValue);
389 root[
"visible_objects_id"] = Json::Value(Json::arrayValue);
392 if (detectionsData.find(frame_number) == detectionsData.end()){
393 return root.toStyledString();
398 for(
int i = 0; i<detections.
boxes.size(); i++){
400 if(detections.
confidences.at(i) < confidence_threshold){
405 if( display_classes.size() > 0 &&
406 std::find(display_classes.begin(), display_classes.end(), classNames[detections.
classIds.at(i)]) == display_classes.end()){
410 int objectId = detections.
objectIds.at(i);
412 auto trackedObject = trackedObjects.find(objectId);
415 Json::Value trackedObjectJSON = trackedObject->second->PropertiesJSON(frame_number);
417 if (trackedObjectJSON[
"visible"][
"value"].asBool() &&
418 trackedObject->second->ExactlyContains(frame_number)){
420 root[
"visible_objects_index"].append(trackedObject->first);
421 root[
"visible_objects_id"].append(trackedObject->second->Id());
425 return root.toStyledString();
429 std::string ObjectDetection::Json()
const {
432 return JsonValue().toStyledString();
436 Json::Value ObjectDetection::JsonValue()
const {
439 Json::Value root = EffectBase::JsonValue();
440 root[
"type"] = info.class_name;
441 root[
"protobuf_data_path"] = protobuf_data_path;
442 root[
"selected_object_index"] = selectedObjectIndex;
443 root[
"confidence_threshold"] = confidence_threshold;
444 root[
"display_box_text"] = display_box_text.JsonValue();
448 for (
auto const& trackedObject : trackedObjects){
449 Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
451 objects[trackedObject.second->Id()] = trackedObjectJSON;
453 root[
"objects"] = objects;
460 void ObjectDetection::SetJson(
const std::string value) {
469 catch (
const std::exception& e)
472 throw InvalidJSON(
"JSON is invalid (missing keys or invalid data types)");
477 void ObjectDetection::SetJsonValue(
const Json::Value root) {
479 EffectBase::SetJsonValue(root);
482 if (!root[
"protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1){
483 protobuf_data_path = root[
"protobuf_data_path"].asString();
485 if(!LoadObjDetectdData(protobuf_data_path)){
486 throw InvalidFile(
"Invalid protobuf data path",
"");
487 protobuf_data_path =
"";
492 if (!root[
"selected_object_index"].isNull())
493 selectedObjectIndex = root[
"selected_object_index"].asInt();
495 if (!root[
"confidence_threshold"].isNull())
496 confidence_threshold = root[
"confidence_threshold"].asFloat();
498 if (!root[
"display_box_text"].isNull())
499 display_box_text.SetJsonValue(root[
"display_box_text"]);
501 if (!root[
"class_filter"].isNull()){
502 class_filter = root[
"class_filter"].asString();
503 std::stringstream ss(class_filter);
504 display_classes.clear();
509 std::getline( ss, substr,
',' );
510 display_classes.push_back( substr );
514 if (!root[
"objects"].isNull()){
515 for (
auto const& trackedObject : trackedObjects){
516 std::string obj_id = std::to_string(trackedObject.first);
517 if(!root[
"objects"][obj_id].isNull()){
518 trackedObject.second->SetJsonValue(root[
"objects"][obj_id]);
524 if (!root[
"objects_id"].isNull()){
525 for (
auto const& trackedObject : trackedObjects){
526 Json::Value trackedObjectJSON;
527 trackedObjectJSON[
"box_id"] = root[
"objects_id"][trackedObject.first].asString();
528 trackedObject.second->SetJsonValue(trackedObjectJSON);
534 std::string ObjectDetection::PropertiesJSON(int64_t requested_frame)
const {
540 if(trackedObjects.count(selectedObjectIndex) != 0){
541 auto selectedObject = trackedObjects.at(selectedObjectIndex);
543 Json::Value trackedObjectJSON = selectedObject->PropertiesJSON(requested_frame);
545 objects[selectedObject->Id()] = trackedObjectJSON;
548 root[
"objects"] = objects;
550 root[
"selected_object_index"] = add_property_json(
"Selected Object", selectedObjectIndex,
"int",
"", NULL, 0, 200,
false, requested_frame);
551 root[
"id"] = add_property_json(
"ID", 0.0,
"string", Id(), NULL, -1, -1,
true, requested_frame);
552 root[
"position"] = add_property_json(
"Position", Position(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
553 root[
"layer"] = add_property_json(
"Track", Layer(),
"int",
"", NULL, 0, 20,
false, requested_frame);
554 root[
"start"] = add_property_json(
"Start", Start(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
555 root[
"end"] = add_property_json(
"End", End(),
"float",
"", NULL, 0, 1000 * 60 * 30,
false, requested_frame);
556 root[
"duration"] = add_property_json(
"Duration", Duration(),
"float",
"", NULL, 0, 1000 * 60 * 30,
true, requested_frame);
557 root[
"confidence_threshold"] = add_property_json(
"Confidence Theshold", confidence_threshold,
"float",
"", NULL, 0, 1,
false, requested_frame);
558 root[
"class_filter"] = add_property_json(
"Class Filter", 0.0,
"string", class_filter, NULL, -1, -1,
false, requested_frame);
560 root[
"display_box_text"] = add_property_json(
"Draw Box Text", display_box_text.GetValue(requested_frame),
"int",
"", &display_box_text, 0, 1.0,
false, requested_frame);
561 root[
"display_box_text"][
"choices"].append(add_property_choice_json(
"Off", 1, display_box_text.GetValue(requested_frame)));
562 root[
"display_box_text"][
"choices"].append(add_property_choice_json(
"On", 0, display_box_text.GetValue(requested_frame)));
565 return root.toStyledString();
Header file for all Exception classes.
Header file for Object Detection effect class.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
This class represents a clip (used to arrange readers on the timeline)
std::shared_ptr< openshot::Frame > GetFrame(int64_t clip_frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Exception for files that can not be found or opened.
Exception for invalid JSON.
This class represents a timeline.
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
This class contains the properties of a tracked object and functions to manipulate it.
void AddBox(int64_t _frame_num, float _cx, float _cy, float _width, float _height, float _angle) override
Add a BBox to the BoxVec map.
This namespace is the default namespace for all code in the openshot library.
const Json::Value stringToJson(const std::string value)
std::vector< cv::Rect_< float > > boxes
std::vector< float > confidences
std::vector< int > classIds
std::vector< int > objectIds
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width