Releasing videocapture object in OpenCV takes more and more time.

[LiunxPaisley]

I'm running a opencv program with quartz, this program is to read a lots of videos from a dataset and get some frames from video. But with for loop going on, the videocapture object's release function takes more and more time. At the beginning, release() takes a few milliseconds, then takes hundreds of milliseconds, finally the program need to wait for release() for seconds.

Here is my program:

#include <fstream>
#include <iostream>
#include <string>
#include <cstdio>
#include <random>
#include <algorithm>

#include <opencv2/core/core.hpp>
#include <opencv2/core/version.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/highgui/highgui_c.h>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/opencv.hpp>

#include <sys/time.h>

#include "/home/liupai/hme-workspace/hme-opencv-test/quartz/src/lib/pmalloc.h"

using namespace std;

void ImageChannelToBuffer(const cv::Mat* img, char* buffer, int c)
{
    int idx = 0;
    for (int h = 0; h < img->rows; ++h) {
        for (int w = 0; w < img->cols; ++w) {
            buffer[idx++] = img->at<cv::Vec3b>(h, w)[c];
        }
    }
}
int data_size = 0;
int read_video_to_volume_datum(const char* filename, const int start_frm,
    const int label, const int length, const int height, const int width,
    const int sampling_rate, char** datum)
{
    cv::VideoCapture cap;
    cv::Mat img, img_origin;

    int offset = 0;
    int channel_size = 0;
    int image_size = 0;
    
    int use_start_frm = start_frm;

    cout << "\n#######Start!!!! cap.open file" << endl;
    cap.open(filename);
    if (!cap.isOpened()) {
        cout << "Cannot open " << filename << endl;
        return false;
    }

    int num_of_frames = cap.get(CV_CAP_PROP_FRAME_COUNT) + 1;
    if (num_of_frames < length * sampling_rate) {
        cerr << filename << " does not have enough frames; having "
             << num_of_frames << endl;
        return false;
    }

    offset = 0;
    if (use_start_frm < 0) {
        cerr << "start frame must be greater or equal to 0" << endl;
    }

    int end_frm = use_start_frm + length * sampling_rate - 1;
    if (end_frm > num_of_frames) {
        cerr << "end frame must be less or equal to num of frames, "
             << "filename: " << filename << endl;
    }

    if (use_start_frm) {
        cout << "\033[31m"
             << "use_start_frm: " << use_start_frm
             << ", end_frame: " << end_frm
             << ", num_of_frames: " << num_of_frames
             << ", filename: " << filename
             << "\033[0m" << endl;
        cap.set(CV_CAP_PROP_POS_FRAMES, use_start_frm - 1);
    }

    for (int i = use_start_frm; i <= end_frm; i += sampling_rate) {
        if (sampling_rate > 1) {
            cap.set(CV_CAP_PROP_POS_FRAMES, i);
        }

        if (height > 0 && width > 0) {
            cap.read(img_origin);
            if (!img_origin.data) {
                cerr << filename << " has no data at frame " << i << endl;
                if (*datum != NULL) {
                    pfree(datum, data_size);
                }
                cap.release();
                return false;
            }
            cout << "resize img_origin" << endl;
            cv::resize(img_origin, img, cv::Size(width, height));
        } else {
            cap.read(img);
        }

        if (!img.data) {
            cerr << "Could not open or find file " << filename << endl;
            if (*datum != NULL) {
                pfree(datum, data_size);
            }
            cap.release();
            return false;
        }

        if (i == use_start_frm) {
            image_size = img.rows * img.cols;
            channel_size = image_size * length;
            data_size = channel_size * 3;
            *datum = (char*)pmalloc(data_size*sizeof(char));
        }

        for (int c = 0; c < 3; c++) {
            ImageChannelToBuffer(&img, *datum + c * channel_size + offset, c);
        }
        cout << "offset = " << offset << endl;
        offset += image_size;
        img_origin.release();
    }
    cout << "\033[32mstart cap.release()\033[0m" << endl;
    struct timeval tv_begin, tv_end;
    gettimeofday(&tv_begin, NULL);
    cap.release();
    gettimeofday(&tv_end, NULL);
    cout << "cap.release(): " << 1000.0*(tv_end.tv_sec - tv_begin.tv_sec)
        + (tv_end.tv_usec - tv_begin.tv_usec)/1000.0 << " ms." << endl;
    cout << "\033[32mend cap.release()\033[0m" << endl;
    return true;
}

void shuffle_clips(vector<int>& shuffle_index){
    std::random_device rd;
    std::mt19937 g(rd());
    std::shuffle(shuffle_index.begin(), shuffle_index.end(), g);
}

int main()
{
    const string root_folder = "/home/liupai/hme-workspace/train-data/UCF-101/";
    const string list_file = "/home/liupai/hme-workspace/workspace/C3D/C3D-nvram/examples/c3d_ucf101_finetuning/train_02.lst";

    cout << "opening file: " << list_file << endl;
    std::ifstream list(list_file.c_str());

    vector<string> file_list_;
    vector<int> start_frm_list_;
    vector<int> label_list_;
    vector<int> shuffle_index_;

    int count = 0;
    string filename;
    int start_frm, label;
    while (list >> filename >> start_frm >> label) {
        file_list_.push_back(filename);
        start_frm_list_.push_back(start_frm);
        label_list_.push_back(label);
        shuffle_index_.push_back(count);
        count++;
    }
    shuffle_clips(shuffle_index_);

    const int dataset_size = shuffle_index_.size();
    const int batch_size = 30;
    const int new_length = 8;
    const int new_height = 128;
    const int new_width = 171;
    const int sampling_rate = 1;
    char* datum = NULL;
    int lines_id_ = 0;

    const int max_iter = 20000;
    for (int iter = 0; iter < max_iter; ++iter) {
        
        for (int item_id = 0; item_id < batch_size; ++item_id) {
            cout << "------> iter: " << iter << endl;
            bool read_status;
            int id = shuffle_index_[lines_id_];
            read_status = read_video_to_volume_datum((root_folder + file_list_[id]).c_str(), start_frm_list_[id],
                label_list_[id], new_length, new_height, new_width, sampling_rate, &datum);
            if (read_status) {
                pfree(datum, data_size);
            }

            lines_id_++;
            if (lines_id_ >= dataset_size) {
                // We have reached the end. Restart from the first.
                cout << "Restarting data prefetching from start." << endl;
                lines_id_ = 0;
            }
        }
    }
    cout << "$$$$$$$$$$$$$$ read file finish!!!!!!!!!!!!" << endl;
}

Here is a output:

# At the beginning

------> iter: 0
#######Start!!!! cap.open file
use_start_frm: 65, end_frame: 72, num_of_frames: 179, filename: /home/liupai/hme-workspace/train-data/UCF-101/PlayingViolin/v_PlayingViolin_g24_c02.avi
...
start cap.release()
cap.release(): 3.018 ms.
end cap.release()

------> iter: 0
#######Start!!!! cap.open file
use_start_frm: 1, end_frame: 8, num_of_frames: 202, filename: /home/liupai/hme-workspace/train-data/UCF-101/TrampolineJumping/v_TrampolineJumping_g18_c01.avi
...
start cap.release()
cap.release(): 3.062 ms.
end cap.release()

------> iter: 0
#######Start!!!! cap.open file
use_start_frm: 81, end_frame: 88, num_of_frames: 296, filename: /home/liupai/hme-workspace/train-data/UCF-101/PommelHorse/v_PommelHorse_g12_c03.avi
...
start cap.release()
cap.release(): 2.453 ms.
end cap.release()

------> iter: 0
#######Start!!!! cap.open file
use_start_frm: 49, end_frame: 56, num_of_frames: 272, filename: /home/liupai/hme-workspace/train-data/UCF-101/StillRings/v_StillRings_g22_c04.avi
...
start cap.release()
cap.release(): 2.146 ms.
end cap.release()

------> iter: 0
#######Start!!!! cap.open file
use_start_frm: 225, end_frame: 232, num_of_frames: 252, filename: /home/liupai/hme-workspace/train-data/UCF-101/HeadMassage/v_HeadMassage_g08_c03.avi
...
start cap.release()
cap.release(): 2.136 ms.
end cap.release()

------> iter: 0
#######Start!!!! cap.open file
use_start_frm: 49, end_frame: 56, num_of_frames: 106, filename: /home/liupai/hme-workspace/train-data/UCF-101/Bowling/v_Bowling_g19_c07.avi
...
start cap.release()
cap.release(): 3.315 ms.
end cap.release()

# After about 400 iterations

------> iter: 437
#######Start!!!! cap.open file
use_start_frm: 113, end_frame: 120, num_of_frames: 376, filename: /home/liupai/hme-workspace/train-data/UCF-101/Kayaking/v_Kayaking_g13_c04.avi
...
start cap.release()
cap.release(): 301.021 ms.
end cap.release()

------> iter: 437
#######Start!!!! cap.open file
use_start_frm: 49, end_frame: 56, num_of_frames: 141, filename: /home/liupai/hme-workspace/train-data/UCF-101/ApplyLipstick/v_ApplyLipstick_g20_c04.avi
...
start cap.release()
cap.release(): 301.74 ms.
end cap.release()

------> iter: 437
#######Start!!!! cap.open file
use_start_frm: 209, end_frame: 216, num_of_frames: 230, filename: /home/liupai/hme-workspace/train-data/UCF-101/BlowDryHair/v_BlowDryHair_g18_c03.avi
...
start cap.release()
cap.release(): 302.311 ms.
end cap.release()

------> iter: 438
#######Start!!!! cap.open file
use_start_frm: 177, end_frame: 184, num_of_frames: 307, filename: /home/liupai/hme-workspace/train-data/UCF-101/BoxingPunchingBag/v_BoxingPunchingBag_g08_c01.avi
....
start cap.release()
cap.release(): 351.546 ms.
end cap.release()

------> iter: 438
#######Start!!!! cap.open file
use_start_frm: 49, end_frame: 56, num_of_frames: 113, filename: /home/liupai/hme-workspace/train-data/UCF-101/FrontCrawl/v_FrontCrawl_g21_c06.avi
...
start cap.release()
cap.release(): 292.598 ms.
end cap.release()
------> iter: 438