Branch3D

.gitignore

@@ -1 +1,8 @@
 *.pyc
+*.json
+*.json~
+.gitignore~
+.idea/
+3d/Hypos_A.txt~
+trackingResultsAPIDIS.xml
+

3d/Hypos_A.txt

@@ -13,7 +13,7 @@
 1113554107.026000 2 1685.494734 3541.761464 0.0
 1113554108.026000 2 1729.33699 2962.741482 0.0
 1113554109.026000 2 1970.671398 2849.704975 0.0
-1113554110.026000 2 2445.542949 2838.848469 0.0 3 573.746117 5186.383372 0.0
+1113554110.026000 2 2445.542949 2838.848469 0.0 3 573.746117 5186.383372 0.0 5 573.746117 5186.383372 0.0 7 573.746117 5186.383372 0.0
 1113554111.026000 2 2834.758231 2653.298761 0.0 3 918.079426 5491.682199 0.0
 1113554112.025000 2 2923.611116 2682.412139 0.0 3 1025.560041 5053.345676 0.0
 1113554113.025000 2 3255.307079 3037.795496 0.0 3 1294.765459 4238.494895 0.0

formatchecker.py

@@ -2,11 +2,12 @@
 
 class FormatChecker:
 
-    def __init__(self, groundtruth, hypotheses):
+    def __init__(self, groundtruth, hypotheses, check3D = True):
         """Constructor """
 
         self.groundtruth_ = groundtruth
         self.hypotheses_ = hypotheses
+        self.is3D = check3D
 
 
     def checkForAmbiguousIDs(self):
@@ -78,7 +79,14 @@ def checkForCompleteness(self):
         """Check ground truth and hypotheses for containing width, height, x and y"""
 
         result = True
-        expectedKeys = ("x", "y", "width", "height")
+
+
+        if self.is3D :
+            expectedKeys = ("x", "y", "z")
+        else :
+            expectedKeys = ("x", "y", "width", "height")
+
+
 
         for f in self.groundtruth_["frames"]:
             for g in f["annotations"]:
@@ -96,3 +104,4 @@ def checkForCompleteness(self):
                         result &= False
 
         return result # true: OK, false: missing id found
+

groundtruth.json

@@ -12,7 +12,8 @@
                         "width": 31.0, 
                         "id": "sheldon", 
                         "y": 105.0, 
-                        "x": 608.0 
+                        "x": 608.0, 
+			"z": 3.23
                     }
                 ]
             }, 
@@ -27,15 +28,18 @@
                         "width": 31.0, 
                         "id": "sheldon", 
                         "y": 105.0, 
-                        "x": 608.0 
+                        "x": 608.0,
+			"z": 3.23
                     }, 
                     {
                         "dco": true, 
                         "height": 38.0, 
                         "width": 29.0, 
                         "id": "leonard", 
                         "y": 145.0, 
-                        "x": 622.0
+                        "x": 622.0,
+			"z": 3.23
+
                     }
                 ]
             }

hypotheses.json

@@ -11,7 +11,8 @@
                         "width": 31.0, 
                         "id": "sheldon", 
                         "y": 105.0, 
-                        "x": 608.0 
+                        "x": 608.0,
+			"z": 3.23 
                     }
                 ]
             }, 
@@ -25,14 +26,16 @@
                         "width": 31.0, 
                         "id": "sheldon", 
                         "y": 105.0, 
-                        "x": 608.0
+                        "x": 608.0,
+			"z": 3.23
                     }, 
                     {
                         "height": 38.0, 
                         "width": 29.0, 
                         "id": "leonard", 
                         "y": 145.0, 
-                        "x": 622.0
+                        "x": 622.0,
+			"z": 3.23
                     }
                 ]
             }

pymot.py

@@ -3,6 +3,8 @@
 import sys
 import json
 import argparse
+import math
+
 from munkres import Munkres
 from rect import Rect
 from importers import MOT_hypo_import
@@ -15,16 +17,22 @@
 
 class MOTEvaluation:
 
-    def __init__(self, groundtruth, hypotheses):
+    def __init__(self, groundtruth, hypotheses, use3Dinput=True):
         """Constructor """
-
+
+        # is the input we use is 3D
+        self.use3D = use3Dinput
+
+        self.distance_threshold = 50 # when using 3D input, the minimum distance between players should be 50 cm
+
         self.overlap_threshold_ = 0.2
         """Bounding box overlap threshold"""
 
         self.munkres_inf_ = sys.maxsize
         """Not quite infinite number for Munkres algorithm"""
-
-        self.sync_delta_ = 0.001
+
+        # time is measure in days in the APIDIS dataset, not milliseconds
+        self.sync_delta_ = 1.0/24/60.0/60.0/100.0  # 2/100th of a second time difference from groundtruth   #0.001 #measured in days,
         """Maximum offset considered for a match of hypothesis and ground truth"""
 
         self.groundtruth_ = groundtruth
@@ -85,13 +93,15 @@ def evaluate(self):
 
         frames = self.groundtruth_["frames"]
         for frame in frames:
+            print "frame: ", frame["num"]
             self.evaluateFrame(frame)
 
 
     def evaluateFrame(self, frame):
         """Update statistics by evaluating a new frame."""
 
         timestamp = frame["timestamp"]
+        LOG.info('timestamp %f', timestamp)
         groundtruths = frame["annotations"]
         hypotheses = self.get_hypotheses_frame(timestamp)["hypotheses"]
 
@@ -133,6 +143,7 @@ def evaluateFrame(self, frame):
         # PAPER STEP 1
         # Valid mappings skip Munkres algorithm, if both ground truth and hypo are found in this frame
         # We call these pairs correspondences and fill the list each frame.
+        # (python dictionary object)
         correspondences = {} # truth id -> hypothesis id
 
         listofprints = []
@@ -141,7 +152,7 @@ def evaluateFrame(self, frame):
 #            print "DIFF Keep correspondence"
 
         for gt_id in self.mappings_.keys():
-            groundtruth = filter(lambda g: g["id"] == gt_id, groundtruths) # Get ground truths with given ground truth id in current frame
+            groundtruth = filter(lambda g: g["id"] == gt_id, groundtruths) # Get ground truths with given ground truth id in current frame (is this necessary, if the formatcheck is done in advance?)
             if len(groundtruth) > 1:
                 LOG.warning("found %d > 1 ground truth tracks for id %s", len(groundtruth), gt_id)
             elif len(groundtruth) < 1:
@@ -153,14 +164,33 @@ def evaluateFrame(self, frame):
                 continue
 
             # Hypothesis found for known mapping
-            # Check hypothesis for overlap
-            overlap = Rect(groundtruth[0]).overlap(Rect(hypothesis[0]))
-            if overlap >= self.overlap_threshold_:
-                LOG.info("Keeping correspondence between %s and %s" % (groundtruth[0]["id"], hypothesis[0]["id"]))
-#                    print "DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0])))
-                listofprints.append("DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0]))))
-                correspondences[gt_id] = hypothesis[0]["id"]
-                self.total_overlap_ += overlap
+
+            if self.use3D :
+                # Check hypothesis for overlap, for 3D data
+                x_gt = groundtruth[0]["x"]
+                y_gt = groundtruth[0]["y"]
+                x_hyp = hypothesis[0]["x"]
+                y_hyp = hypothesis[0]["y"]
+
+                distance = math.sqrt((x_gt-x_hyp)**2 + (y_gt-y_hyp)**2)
+
+                if distance <= self.distance_threshold:
+                    LOG.info("Keeping correspondence between %s and %s" % (groundtruth[0]["id"], hypothesis[0]["id"]))
+                    #                    print "DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0])))
+                    listofprints.append("DIFF Keep corr %s %s %.2f" % (
+                    groundtruth[0]["id"], hypothesis[0]["id"], distance))
+                    correspondences[gt_id] = hypothesis[0]["id"]
+                    self.total_euclidian_distance_error += distance
+
+            else:
+                # Check hypothesis for overlap, for 2D data
+                overlap = Rect(groundtruth[0]).overlap(Rect(hypothesis[0]))
+                if overlap >= self.overlap_threshold_:
+                    LOG.info("Keeping correspondence between %s and %s" % (groundtruth[0]["id"], hypothesis[0]["id"]))
+    #                    print "DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0])))
+                    listofprints.append("DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0]))))
+                    correspondences[gt_id] = hypothesis[0]["id"]
+                    self.total_overlap_ += overlap
 
 
         for p in sorted(listofprints):
@@ -190,16 +220,29 @@ def evaluateFrame(self, frame):
                 if hypothesis["id"] in correspondences.values():
                     LOG.info("Hypothesis %s already in correspondence" % hypothesis["id"])
                     continue
-
-                rect_groundtruth = Rect(groundtruth)
-                rect_hypothesis = Rect(hypothesis)
-                overlap = rect_groundtruth.overlap(rect_hypothesis)
-
-                if overlap >= self.overlap_threshold_:
-#                        print "Fill Hungarian", rect_groundtruth, rect_hypothesis, overlap
-                    munkres_matrix[i][j] = 1 / overlap
-                    LOG.info("DIFF candidate %s %s %.2f" % (groundtruth["id"], hypothesis["id"], overlap))
-
+
+                if self.use3D:
+                    x_gt = groundtruth["x"]
+                    y_gt = groundtruth["y"]
+                    x_hyp = hypothesis["x"]
+                    y_hyp = hypothesis["y"]
+
+                    distance = math.sqrt((x_gt - x_hyp) ** 2 + (y_gt - y_hyp) ** 2)
+
+                    if distance <= self.distance_threshold:
+                        #                        print "Fill Hungarian", rect_groundtruth, rect_hypothesis, overlap
+                        munkres_matrix[i][j] = distance
+                        LOG.info("DIFF candidate %s %s %.2f" % (groundtruth["id"], hypothesis["id"], distance))
+                else:
+                    rect_groundtruth = Rect(groundtruth)
+                    rect_hypothesis = Rect(hypothesis)
+                    overlap = rect_groundtruth.overlap(rect_hypothesis)
+
+                    if overlap >= self.overlap_threshold_:
+    #                        print "Fill Hungarian", rect_groundtruth, rect_hypothesis, overlap
+                        munkres_matrix[i][j] = 1 / overlap
+                        LOG.info("DIFF candidate %s %s %.2f" % (groundtruth["id"], hypothesis["id"], overlap))
+
         # Do the Munkres
         LOG.debug(munkres_matrix)
 
@@ -227,20 +270,30 @@ def evaluateFrame(self, frame):
 
             # Assert no known mappings have been added to hungarian, since keep correspondence should have considered this case.
             if gt_id in self.mappings_:
-                assert self.mappings_[gt_id] != hypo_id 
-            
-            
+                assert self.mappings_[gt_id] != hypo_id
+
+
             # Add to correspondences
-            LOG.info("Correspondence found: %s and %s (overlap: %f)" % (gt_id, hypo_id, 1.0 / munkres_matrix[gt_index][hypo_index]))
+            if self.use3D:
+                LOG.info("Correspondence found: %s and %s (distance: %f)" % (gt_id, hypo_id, munkres_matrix[gt_index][hypo_index]))
+            else:
+                LOG.info("Correspondence found: %s and %s (overlap: %f)" % (gt_id, hypo_id, 1.0 / munkres_matrix[gt_index][hypo_index]))
 #                correspondencelist.append("DIFF correspondence %s %s %.2f" % (gt_id, hypo_id, 1.0 / munkres_matrix[gt_index][hypo_index]))
             correspondencelist.append("DIFF correspondence %s %s" % (gt_id, hypo_id))
             correspondences[gt_id] = hypo_id
-            self.total_overlap_ += overlap
-
+
+            if self.use3D:
+                self.total_euclidian_distance_error += distance
+            else:
+                self.total_overlap_ += overlap
+
+            # The dco flag stands for do not care and can be used to mark hard to track targets, e.g. because of occlusion.
+            # Thus, a tracker which does not find the target will not be penalized, whereas a tracker which finds the
+            # target won't be punished (with a false positive) either.
 
             # Count "recoverable" and "non-recoverable" mismatches
             # "recoverable" mismatches
-            if gt_id in self.gt_map_ and self.gt_map_[gt_id] != hypo_id and not groundtruths[gt_index].get("dco",False):
+            if gt_id in self.gt_map_ and self.gt_map_[gt_id] != hypo_id and not groundtruths[gt_index].get("dco",False): #get will return False by default if the key "dco" does not exist and its true value otherwise
                 LOG.info("Look ma! We got a recoverable mismatch over here! (%s-%s) -> (%s-%s)" % (gt_id, self.gt_map_[gt_id], gt_id, hypo_id))
                 self.recoverable_mismatches_ += 1
 
@@ -423,7 +476,10 @@ def getMOTP(self):
             write_stderr_red("Warning", "No correspondences found. MOTP calculation not possible")
 #            raise("No correspondence found. MOTP calculation not possible")
         else:
-            motp = self.total_overlap_ / self.total_correspondences_
+            if self.use3D:
+                motp = self.total_euclidian_distance_error / self.total_correspondences_
+            else:
+                motp = self.total_overlap_ / self.total_correspondences_
         return motp
 
 
@@ -432,6 +488,16 @@ def getAbsoluteStatistics(self):
         covered_ground_truths = self.groundtruth_ids_ & set(self.gt_map_.keys())
         lonely_hypotheses = self.hypothesis_ids_ - set(self.hypo_map_.keys())
 
+        distanceOrOverlapString = ""
+        distanceValue = 0.0
+        if self.use3D:
+            distanceOrOverlapString= "total distance error"
+            distanceValue = self.total_euclidian_distance_error
+        else:
+            distanceOrOverlapString = "total overlap"
+            distanceValue = self.total_overlap_
+
+
         return {
             "ground truths":   self.total_groundtruths_,
             "false positives": self.false_positives_,
@@ -440,7 +506,7 @@ def getAbsoluteStatistics(self):
             "recoverable mismatches": self.recoverable_mismatches_,
             "non-recoverable mismatches":  self.non_recoverable_mismatches_,
             "correspondences": self.total_correspondences_,
-            "total overlap":   self.total_overlap_,
+            distanceOrOverlapString:   distanceValue,
             "lonely ground truth tracks": len(lonely_ground_truths),
             "covered ground truth tracks": len(covered_ground_truths),
             "lonely hypothesis tracks":   len(lonely_hypotheses),
@@ -545,9 +611,16 @@ def resetStatistics(self):
         self.misses_ = 0
         self.false_positives_ = 0
         self.total_groundtruths_ = 0
-        self.total_overlap_ = 0.0
+
+        if self.use3D:
+            self.total_euclidian_distance_error = 0.0
+        else:
+            self.total_overlap_ = 0.0
+
         self.total_correspondences_ = 0       
 
+        # create two sets containing a unique collection of all the id that
+        # occur for every frame
         self.groundtruth_ids_ = set()
         self.hypothesis_ids_ = set()
 
@@ -561,6 +634,7 @@ def resetStatistics(self):
     parser.add_argument('-v', '--visual_debug_file')
     args = parser.parse_args()
 
+
     # Load ground truth according to format
     # Assume MOT format, if non-json
     gt = open(args.groundtruth) # gt file
@@ -578,22 +652,24 @@ def resetStatistics(self):
         hypotheses = MOT_hypo_import(hypo.readlines())
     hypo.close()
 
-
-    evaluator = MOTEvaluation(groundtruth, hypotheses)
-
     if(args.check_format):
         formatChecker = FormatChecker(groundtruth, hypotheses)
         success = formatChecker.checkForExistingIDs()
-        success |= formatChecker.checkForAmbiguousIDs()
-        success |= formatChecker.checkForCompleteness()
+        success &= formatChecker.checkForAmbiguousIDs()
+        success &= formatChecker.checkForCompleteness()
 
         if not success:
             write_stderr_red("Error:", "Stopping. Fix ids first. Evaluating with broken data does not make sense!\n    File: %s" % args.groundtruth)
             sys.exit()
 
+    # why would the evaluation start before checking format??
+    evaluator = MOTEvaluation(groundtruth, hypotheses, True)
+
     evaluator.evaluate()
     print "Track statistics"
     evaluator.printTrackStatistics()
+
+    evaluator.getAbsoluteStatistics()
     print 
     print "Results"
     evaluator.printResults()