C++ OpenCV Video Processing | Beginner's Tutorial

Video processing refers to processing and analyzing each frame in a video sequence.

OpenCV provides powerful tools for handling video data, including video reading, frame processing, video saving, and real-time video processing capabilities.

Application Scenarios of Video Processing

Video Surveillance:

• Using background subtraction to detect moving objects.
• Using optical flow to analyze object motion trajectories.

Video Analysis:

• Extracting key frames from video.
• Analyzing object behavior in video.

Real-time Processing:

• Real-time video filters (such as edge detection, blur, etc.).
• Real-time object detection and tracking.

---

Reading and Displaying Video

Reading Video Files (VideoCapture)

In OpenCV, the VideoCapture class is used to read video frames from video files or cameras.

To read a video file, you first need to create a VideoCapture object and specify the path to the video file.

Example

#include <opencv2/opencv.hpp>

using namespace cv;

int main(){
    // Create VideoCapture object and open video file
    VideoCapture cap("example.mp4");

    // Check if video was successfully opened
    if(!cap.isOpened()){
        std::cerr << "Error: Could not open video file." << std::endl;
        return -1;
    }

    // Video reading and display code will be introduced below

    return 0;
}

Displaying Video Frames

After reading the video file, you can read the video frame by frame through a loop and display each frame using the imshow function.

Example

Mat frame;
while(true){
    // Read next frame
    cap >> frame;

    // If frame is empty, video has ended
    if(frame.empty()){
        break;
    }

    // Display current frame
    imshow("Video", frame);

    // Wait 30 milliseconds, press ESC key to exit
    if(waitKey(30) == 27){
        break;
    }
}

// Release VideoCapture object
cap.release();

// Close all windows
destroyAllWindows();

Saving Video (VideoWriter)

If you want to save processed video to a file, you can use the VideoWriter class. You need to specify the output file name, encoding format, frame rate, and frame size.

Example

// Get video frame rate and frame size
double fps = cap.get(CAP_PROP_FPS);
Size frameSize(cap.get(CAP_PROP_FRAME_WIDTH), cap.get(CAP_PROP_FRAME_HEIGHT));

// Create VideoWriter object
VideoWriter writer("output.avi", VideoWriter::fourcc('M', 'J', 'P', 'G'), fps, frameSize);

while(true){
    cap >> frame;
    if(frame.empty()){
        break;
    }

    // Write frame to output video file
    writer.write(frame);
    imshow("Video", frame);

    if(waitKey(30) == 27){
        break;
    }
}

// Release VideoCapture and VideoWriter objects
cap.release();
writer.release();
destroyAllWindows();

---

Video Frame Processing

Processing Video Frame by Frame

In video processing, specific operations are usually required for each frame. For example, you can grayscale each frame, perform edge detection, etc.

Example

while(true){
    cap >> frame;
    if(frame.empty()){
        break;
    }

    // Convert frame to grayscale image
    Mat grayFrame;
    cvtColor(frame, grayFrame, COLOR_BGR2GRAY);

    // Display grayscale frame
    imshow("Gray Video", grayFrame);

    if(waitKey(30) == 27){
        break;
    }
}

Real-time Processing of Video Frames

When processing video frames in real-time, real-time processing algorithms are usually applied to each frame. For example, detecting moving objects in video in real-time.

Example

Mat prevFrame, nextFrame, diffFrame;
cap >> prevFrame;
cvtColor(prevFrame, prevFrame, COLOR_BGR2GRAY);

while(true){
    cap >> nextFrame;
    if(nextFrame.empty()){
        break;
    }
    cvtColor(nextFrame, nextFrame, COLOR_BGR2GRAY);

    // Calculate frame difference
    absdiff(prevFrame, nextFrame, diffFrame);

    // Display difference frame
    imshow("Motion Detection", diffFrame);

    // Update previous frame
    prevFrame = nextFrame.clone();

    if(waitKey(30) == 27){
        break;
    }
}

---

Camera Real-time Processing

Opening the Camera

The VideoCapture class can not only read video files but also open the camera for real-time video stream processing. To open the camera, simply set the VideoCapture parameter to 0 (representing the default camera).

Example

VideoCapture cap(0);
if(!cap.isOpened()){
    std::cerr << "Error: Could not open camera." << std::endl;
    return -1;
}

Processing Real-time Video Stream

After opening the camera, you can process the real-time video stream frame by frame just like processing video files. For example, you can apply edge detection algorithms to the real-time video stream.

Example

while(true){
    cap >> frame;
    if(frame.empty()){
        break;
    }

    // Apply Canny edge detection
    Mat edges;
    Canny(frame, edges, 100, 200);

    // Display edge detection results
    imshow("Edges", edges);

    if(waitKey(30) == 27){
        break;
    }
}

cap.release();
destroyAllWindows();

---

Advanced Video Processing Techniques

Video Background Subtraction

Background subtraction is used to extract foreground objects from video.

Example

#include <opencv2/opencv.hpp>
#include <iostream>

using namespace cv;
using namespace std;

int main(){
    // Open video file or camera
    VideoCapture cap("video.mp4");
    if(!cap.isOpened()){
        cout << "Error: Could not open video file or camera!" << endl;
        return -1;
    }

    // Create background subtractor
    Ptr<BackgroundSubtractor> bgSubtractor = createBackgroundSubtractorMOG2();

    // Process video frames
    Mat frame, fgMask;
    while(true){
        cap >> frame;
        if(frame.empty()) break;

        // Apply background subtraction
        bgSubtractor->apply(frame, fgMask);

        // Display results
        imshow("Frame", frame);
        imshow("Foreground Mask", fgMask);

        // Press ESC key to exit
        if(waitKey(30) == 27) break;
    }

    // Release resources
    cap.release();
    destroyAllWindows();

    return 0;
}

Optical Flow Calculation

Optical flow is used to calculate the motion of objects in video frames.

Sparse Optical Flow (Lucas-Kanade Method):

Example

#include <opencv2/opencv.hpp>
#include <iostream>

using namespace cv;
using namespace std;

int main(){
    // Open video file or camera
    VideoCapture cap("video.mp4");
    if(!cap.isOpened()){
        cout << "Error: Could not open video file or camera!" << endl;
        return -1;
    }

    // Read first frame
    Mat oldFrame, oldGray;
    cap >> oldFrame;
    cvtColor(oldFrame, oldGray, COLOR_BGR2GRAY);

    // Select feature points
    vector<Point2f> oldPoints;
    goodFeaturesToTrack(oldGray, oldPoints, 100, 0.3, 7);

    // Process video frames
    Mat frame, gray;
    while(true){
        cap >> frame;
        if(frame.empty()) break;

        // Convert to grayscale image
        cvtColor(frame, gray, COLOR_BGR2GRAY);

        // Calculate optical flow
        vector<Point2f> newPoints;
        vector<uchar> status;
        vector<float> err;
        calcOpticalFlowPyrLK(oldGray, gray, oldPoints, newPoints, status, err);

        // Draw optical flow trajectories
        for(size_t i = 0; i < oldPoints.size(); i++){
            if(status){
                line(frame, oldPoints, newPoints, Scalar(0, 255, 0), 2);
                circle(frame, newPoints, 3, Scalar(0, 0, 255), -1);
            }
        }

        // Update frame and feature points
        oldGray = gray.clone();
        oldPoints = newPoints;

        // Display results
        imshow("Optical Flow", frame);

        // Press ESC key to exit
        if(waitKey(30) == 27) break;
    }

    // Release resources
    cap.release();
    destroyAllWindows();

    return 0;
}