CVPR 2021 Changes

Examples/OISTVisualizationTool/main.swift

@@ -44,9 +44,9 @@ struct ViewFrame: ParsableCommand {
 
     let image = dataset.loadFrame(frameRawId)!
 
-    plot(image, boxes: dataset.labels[frameId].enumerated().map {
-      (String($0), $1.location)
-    }, margin: 10.0, scale: 0.5).show()
+    // plot(image, boxes: dataset.labels[frameId].enumerated().map {
+    //   (String($0), $1.location)
+    // }, margin: 10.0, scale: 0.5).show()
   }
 }
 
@@ -118,7 +118,7 @@ struct RawTrack: ParsableCommand {
 
     var tracker = makeRawPixelTracker(frames: videos, target: startPatch)
 
-    if verbose { tracker.optimizer.verbosity = .SUMMARY }
+    // if verbose { tracker.optimizer.verbosity = .SUMMARY } For LM Optimizer
 
     let prediction = tracker.infer(knownStart: Tuple1(startPose))
 
@@ -150,9 +150,9 @@ struct RawTrack: ParsableCommand {
       print("Creating output plot")
     }
     startTimer("PLOTTING")
-    plot(image, boxes: bboxes.indices.map {
-      ("\($0)", bboxes[$0])
-    }, margin: 10.0, scale: 0.5).show()
+    // plot(image, boxes: bboxes.indices.map {
+    //   ("\($0)", bboxes[$0])
+    // }, margin: 10.0, scale: 0.5).show()
     stopTimer("PLOTTING")
 
     if verbose {
@@ -207,11 +207,11 @@ struct PpcaTrack: ParsableCommand {
     startTimer("MAKE_GRAPH")
     var tracker = makePPCATracker(model: ppca, statistics: statistics, frames: videos, targetSize: (40, 70))
     stopTimer("MAKE_GRAPH")
+    // For LM Optimizer
+    // if verbose { tracker.optimizer.verbosity = .SUMMARY }
 
-    if verbose { tracker.optimizer.verbosity = .SUMMARY }
-
-    tracker.optimizer.cgls_precision = 1e-6
-    tracker.optimizer.precision = 1e-2
+    // tracker.optimizer.cgls_precision = 1e-6
+    // tracker.optimizer.precision = 1e-2
 
     startTimer("GRAPH_INFER")
     let prediction = tracker.infer(knownStart: Tuple2(startPose, Vector10(flatTensor: startLatent)))
@@ -255,9 +255,9 @@ struct PpcaTrack: ParsableCommand {
       print("Creating output plot")
     }
     startTimer("PLOTTING")
-    plot(image, boxes: bboxes.indices.map {
-      ("\($0)", bboxes[$0])
-    }, margin: 10.0, scale: 0.5).show()
+    // plot(image, boxes: bboxes.indices.map {
+    //   ("\($0)", bboxes[$0])
+    // }, margin: 10.0, scale: 0.5).show()
     stopTimer("PLOTTING")
 
     if verbose {
@@ -358,11 +358,12 @@ struct NaiveRae: ParsableCommand {
     stopTimer("MAKE_GRAPH")
 
     if verbose { print("Starting Optimization...") }
-    if verbose { tracker.optimizer.verbosity = .SUMMARY }
+    // For LM Optimizer
+    // if verbose { tracker.optimizer.verbosity = .SUMMARY }
 
-    tracker.optimizer.cgls_precision = 1e-7
-    tracker.optimizer.precision = 1e-4
-    tracker.optimizer.max_iteration = 200
+    // tracker.optimizer.cgls_precision = 1e-7
+    // tracker.optimizer.precision = 1e-4
+    // tracker.optimizer.max_iteration = 200
 
     startTimer("GRAPH_INFER")
     let prediction = tracker.infer(knownStart: Tuple1(startPose))
@@ -402,9 +403,9 @@ struct NaiveRae: ParsableCommand {
       print("Creating output plot")
     }
     startTimer("PLOTTING")
-    plot(image, boxes: bboxes.indices.map {
-      ("\($0)", bboxes[$0])
-    }, margin: 10.0, scale: 0.5).show()
+    // plot(image, boxes: bboxes.indices.map {
+    //   ("\($0)", bboxes[$0])
+    // }, margin: 10.0, scale: 0.5).show()
     stopTimer("PLOTTING")
 
     if verbose {
@@ -578,11 +579,12 @@ struct NaivePca: ParsableCommand {
     stopTimer("MAKE_GRAPH")
 
     if verbose { print("Starting Optimization...") }
-    if verbose { tracker.optimizer.verbosity = .SUMMARY }
+    // For LM Optimizer
+    // if verbose { tracker.optimizer.verbosity = .SUMMARY }
 
-    tracker.optimizer.cgls_precision = 1e-7
-    tracker.optimizer.precision = 1e-4
-    tracker.optimizer.max_iteration = 200
+    // tracker.optimizer.cgls_precision = 1e-7
+    // tracker.optimizer.precision = 1e-4
+    // tracker.optimizer.max_iteration = 200
 
     startTimer("GRAPH_INFER")
     let prediction = tracker.infer(knownStart: Tuple1(startPose))
@@ -622,9 +624,9 @@ struct NaivePca: ParsableCommand {
       print("Creating output plot")
     }
     startTimer("PLOTTING")
-    plot(image, boxes: bboxes.indices.map {
-      ("\($0)", bboxes[$0])
-    }, margin: 10.0, scale: 0.5).show()
+    // plot(image, boxes: bboxes.indices.map {
+    //   ("\($0)", bboxes[$0])
+    // }, margin: 10.0, scale: 0.5).show()
     stopTimer("PLOTTING")
 
     if verbose {

Package.resolved

@@ -1,15 +1,6 @@
 {
   "object": {
     "pins": [
-      {
-        "package": "CSV.swift",
-        "repositoryURL": "https://github.com/yaslab/CSV.swift.git",
-        "state": {
-          "branch": null,
-          "revision": "81d2874c51db364d7e1d71b0d99018a294c87ac1",
-          "version": "2.4.3"
-        }
-      },
       {
         "package": "Penguin",
         "repositoryURL": "https://github.com/saeta/penguin.git",
@@ -19,15 +10,6 @@
           "version": null
         }
       },
-      {
-        "package": "Plotly",
-        "repositoryURL": "https://github.com/vojtamolda/Plotly.swift",
-        "state": {
-          "branch": null,
-          "revision": "6e80119ba37b913e5460459556e2bf58f02eba67",
-          "version": "0.4.0"
-        }
-      },
       {
         "package": "swift-argument-parser",
         "repositoryURL": "https://github.com/apple/swift-argument-parser.git",
@@ -46,24 +28,6 @@
           "version": "0.1.0"
         }
       },
-      {
-        "package": "swift-models",
-        "repositoryURL": "https://github.com/tensorflow/swift-models.git",
-        "state": {
-          "branch": null,
-          "revision": "b2fc0325bf9d476bf2d7a4cd0a09d36486c506e4",
-          "version": null
-        }
-      },
-      {
-        "package": "SwiftProtobuf",
-        "repositoryURL": "https://github.com/apple/swift-protobuf.git",
-        "state": {
-          "branch": null,
-          "revision": "da9a52be9cd36c63993291ce3f1b65dafcd1e826",
-          "version": "1.14.0"
-        }
-      },
       {
         "package": "swift-tools-support-core",
         "repositoryURL": "https://github.com/apple/swift-tools-support-core.git",

Package.swift

@@ -30,7 +30,6 @@ let package = Package(
     .package(name: "TensorBoardX", url: "https://github.com/ProfFan/tensorboardx-s4tf.git", from: "0.1.3"),
     .package(url: "https://github.com/apple/swift-tools-support-core.git", .branch("swift-5.2-branch")),
     .package(url: "https://github.com/apple/swift-argument-parser.git", from: "0.3.0"),
-    .package(name: "Plotly", url: "https://github.com/vojtamolda/Plotly.swift", from: "0.4.0"),
   ],
   targets: [
     // Targets are the basic building blocks of a package. A target can define a module or a test suite.
@@ -57,7 +56,6 @@ let package = Package(
       name: "BeeDataset",
       dependencies: [
         "SwiftFusion",
-        "Plotly",
         "ModelSupport",
       ]),
     .target(
@@ -86,7 +84,6 @@ let package = Package(
         "BeeTracking",
         .product(name: "PenguinParallelWithFoundation", package: "Penguin"),
         "SwiftFusion",
-        "Plotly",
         .product(name: "ArgumentParser", package: "swift-argument-parser"),
       ],
     path: "Examples/OISTVisualizationTool"),
@@ -97,7 +94,6 @@ let package = Package(
         "BeeTracking",
         .product(name: "PenguinParallelWithFoundation", package: "Penguin"),
         "SwiftFusion",
-        "Plotly",
         .product(name: "ArgumentParser", package: "swift-argument-parser"),
       ],
       path: "Scripts",

Scripts/Andrew01.swift

@@ -11,14 +11,11 @@ import PenguinStructures
 
 /// Andrew01: RAE Tracker
 struct Andrew01: ParsableCommand {
-  @Option(help: "Run on track number x")
-  var trackId: Int = 0
-
   @Option(help: "Run for number of frames")
   var trackLength: Int = 80
 
   @Option(help: "Size of feature space")
-  var featureSize: Int = 5
+  var featureSize: Int = 256
 
   @Option(help: "Pretrained weights")
   var weightsFile: String?
@@ -27,10 +24,12 @@ struct Andrew01: ParsableCommand {
   // Make sure you have a folder `Results/andrew01` before running
   func run() {
     let np = Python.import("numpy")
-    let kHiddenDimension = 100
+    let kHiddenDimension = 512
 
     let (imageHeight, imageWidth, imageChannels) =
       (40, 70, 1)
+
+
     var rae = DenseRAE(
       imageHeight: imageHeight, imageWidth: imageWidth, imageChannels: imageChannels,
       hiddenDimension: kHiddenDimension, latentDimension: featureSize
@@ -42,20 +41,25 @@ struct Andrew01: ParsableCommand {
       rae.load(weights: np.load("./oist_rae_weight_\(featureSize).npy", allow_pickle: true))
     }
 
+    // let (imageHeight, imageWidth, imageChannels) =
+    //   (40, 70, 1)
+
+    // let rp = RandomProjection(fromShape: TensorShape([imageHeight, imageWidth, imageChannels]), toFeatureSize: featureSize)
+
+    let trainingDatasetSize = 100
+
     let dataDir = URL(fileURLWithPath: "./OIST_Data")
-    let data = OISTBeeVideo(directory: dataDir, length: 100)!
-    let testData = OISTBeeVideo(directory: dataDir, afterIndex: 100, length: 80)!
+    let data = OISTBeeVideo(directory: dataDir, length: trainingDatasetSize)!
+    let testData = OISTBeeVideo(directory: dataDir, afterIndex: trainingDatasetSize, length: trackLength)!
 
     let trackerEvaluation = TrackerEvaluationDataset(testData)
-
     let evalTracker: Tracker = {frames, start in
-        let trainingDatasetSize = 100
         var tracker = trainProbabilisticTracker(
             trainingData: data,
             encoder: rae,
             frames: frames,
             boundingBoxSize: (40, 70),
-            withFeatureSize: 5,
+            withFeatureSize: featureSize,
             fgRandomFrameCount: trainingDatasetSize,
             bgRandomFrameCount: trainingDatasetSize
         )
@@ -64,32 +68,36 @@ struct Andrew01: ParsableCommand {
 
         return track
     }
+    let plt = Python.import("matplotlib.pyplot")
+    let sequenceCount = 1
+    var results = trackerEvaluation.evaluate(evalTracker, sequenceCount: sequenceCount, deltaAnchor: 175, outputFile: "andrew01")
 
-    var results = trackerEvaluation.evaluate(evalTracker, sequenceCount: 5, deltaAnchor: 175, outputFile: "andrew01")
-
-
-    for (index, value) in results.sequences.prefix(5).enumerated() {
+    for (index, value) in results.sequences.prefix(sequenceCount).enumerated() {
       var i: Int = 0
       zip(value.subsequences.first!.frames, zip(value.subsequences.first!.prediction, value.subsequences.first!.groundTruth)).map {
-        let (fig, axes) = plotPatchWithGT(frame: $0.0, actual: $0.1.0.center, expected: $0.1.1.center)
+        let (fig, axes) = plotFrameWithPatches(frame: $0.0, actual: $0.1.0.center, expected: $0.1.1.center, firstGroundTruth: value.subsequences.first!.groundTruth.first!.center)
         fig.savefig("Results/andrew01/sequence\(index)/andrew01_\(i).png", bbox_inches: "tight")
+        plt.close("all")
         i = i + 1
       }
-      let plt = Python.import("matplotlib.pyplot")
+
+
       let (fig, axes) = plt.subplots(1, 2, figsize: Python.tuple([20, 20])).tuple2
       fig.suptitle("Tracking positions and Subsequence Average Overlap with Accuracy \(String(format: "%.2f", value.subsequences.first!.metrics.accuracy)) and Robustness \(value.subsequences.first!.metrics.robustness).")
 
       value.subsequences.map {
-        plotTrajectory(
-          track: $0.prediction.map{$0.center}, withGroundTruth: $0.groundTruth.map{$0.center}, on: axes[0],
-          withTrackColors: plt.cm.jet, withGtColors: plt.cm.gray
+        plotPoseDifference(
+          track: $0.prediction.map{$0.center}, withGroundTruth: $0.groundTruth.map{$0.center}, on: axes[0]
         )
       }
       plotOverlap(
           metrics: value.subsequences.first!.metrics, on: axes[1]
       )
-      fig.savefig("Results/andrew01/andrew01_subsequence\(index).pdf", bbox_inches: "tight")
+      fig.savefig("Results/andrew01/andrew01_subsequence\(index).png", bbox_inches: "tight")
+      print("Accuracy for sequence is \(value.sequenceMetrics.accuracy) with Robustness of \(value.sequenceMetrics.robustness)")
     }
+
+    print("Accuracy for all sequences is \(results.trackerMetrics.accuracy) with Robustness of \(results.trackerMetrics.robustness)")
 
 
 
@@ -103,4 +111,4 @@ fileprivate func unpack<A, B>(_ t: Tuple2<A, B>) -> (A, B) {
 /// Returns `t` as a Swift tuple.
 fileprivate func unpack<A>(_ t: Tuple1<A>) -> (A) {
   return (t.head)
-}
+}