VuoFfmpegDecoder.h

Go to the documentation of this file.

 
#pragma once
 
#include "module.h"
 
#ifdef __cplusplus
extern "C"
{
#endif
#include "VuoVideoDecoder.h"
#include "VuoVideo.h"
#include "VuoImage.h"
#include "VuoAudioSamples.h"
#include "VuoAudioFrame.h"
#include "VuoList_VuoAudioSamples.h"
#include "VuoVideoFrame.h"
#include "VuoText.h"
#include "VuoUrl.h"
 
// FFMPEG
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdocumentation"
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libswscale/swscale.h>
#include <string.h>
#include <libswresample/swresample.h>
#include <libavutil/opt.h>
#pragma clang diagnostic pop
 
#include <sys/types.h> // for uint
#ifdef __cplusplus
}
#endif
 
class VuoFfmpegDecoder : public VuoVideoDecoder
{
public:
    static VuoFfmpegDecoder* Create(VuoUrl url);
 
    virtual ~VuoFfmpegDecoder();
    virtual bool NextVideoFrame(VuoVideoFrame* frame);
    virtual bool NextAudioFrame(VuoAudioFrame* audio);
    virtual bool SeekToSecond(double second, VuoVideoFrame *frame);
    bool ContainsAudio();
    virtual double GetDuration();
    virtual void SetPlaybackRate(double rate);
    virtual double GetLastDecodedVideoTimeStamp();
    virtual bool CanPlayAudio();
    virtual unsigned int GetAudioChannelCount();
    virtual bool IsReady() { return isReady; }
    virtual double GetFrameRate();
 
private:
 
    template<class T> struct LLNode
    {
        T data;
        LLNode* next;
        LLNode* previous;
 
        LLNode(T value)
        {
            data = value;
            next = NULL;
            previous = NULL;
        }
    };
 
    template<class T> struct LinkedList
    {
        LLNode<T>* first;   // pointer to the first node in this list.
        LLNode<T>* last;    // pointer to the last node in the list
 
        void (*destructor)(T*);
 
        unsigned int _size; // how many nodes are in this list
        unsigned int Count() { return _size; }
 
        LinkedList()
        {
            _size = 0;
            first = NULL;
            last = NULL;
            destructor = NULL;
        }
 
        ~LinkedList()
        {
            Clear();
        }
 
        void Clear()
        {
            LLNode<T>* cur = first;
 
            while(cur)
            {
                if(destructor != NULL)
                    destructor(&cur->data);
                LLNode<T>* tmp = cur;
                cur = tmp->next;
                delete(tmp);
            }
 
            _size = 0;
 
            first = NULL;
            last = NULL;
        }
 
        void Add(T value)
        {
            LLNode<T>* node = new LLNode<T>(value);
 
            if( last )
            {
                node->previous = last;
                last->next = node;
                last = node;
            }
            else
            {
                first = node;
                last = node;
            }
 
            _size++;
        }
 
        void Unshift(T value)
        {
            LLNode<T>* node = new LLNode<T>(value);
 
            if(first != NULL)
            {
                first->previous = node;
                node->next = first;
                first = node;
            }
            else
            {
                first = node;
                last = node;
            }
 
            _size++;
        }
 
        void Remove(LLNode<T>* node)
        {
            if(node == NULL)
                return;
 
            if(node->next)
            {
                if(node->previous)
                {
                    node->previous->next = node->next;
                    node->next->previous = node->previous;
                }
                else
                {
                    // was first node
                    node->next->previous = NULL;
                    first = node->next;
                }
            }
            else
            {
                // was last node in list
                if(node->previous)
                {
                    last = node->previous;
                    node->previous->next = NULL;
                }
                // was only node in list
                else
                {
                    first = NULL;
                    last = NULL;
                }
            }
 
            _size--;
            delete(node);
        }
 
        bool Shift(T* value)
        {
            if(first != NULL)
            {
                *value = first->data;
                LLNode<T>* tmp = first;
 
                if(tmp->next == NULL)
                {
                    first = NULL;
                    last = NULL;
                }
                else
                {
                    first = first->next;
                }
 
                delete(tmp);
                _size--;
 
                return true;
            }
            else
            {
                return false;
            }
        }
 
        bool Pop(T* value)
        {
            if(last != NULL)
            {
                *value = last->data;
 
                LLNode<T>* tmp = last;
 
                if(last->previous == NULL)
                {
                    first = NULL;
                    last = NULL;
                }
                else
                {
                    last = last->previous;
                    last->next = NULL;
                }
 
                delete(tmp);
                _size--;
 
                return true;
            }
 
            return false;
        }
 
        bool Peek(T* value)
        {
            if(first != NULL)
            {
                *value = first->data;
                return true;
            }
            return false;
        }
 
        bool PeekLast(T* value)
        {
            if(last != NULL)
            {
                *value = last->data;
                return true;
            }
            return false;
        }
    };
 
    struct AudioFrame
    {
        unsigned int size;
        unsigned int channels;
        int64_t pts;
        double timestamp;
        uint8_t** samples;
 
        static void Delete(AudioFrame* frame)
        {
            for(int i = 0; i < frame->channels; i++)
                free(frame->samples[i]);
            free(frame->samples);
        }
    };
 
    struct VideoFrame
    {
        VuoImage image;
        int64_t pts;
        double timestamp;
        double duration;
 
        static void Delete(VideoFrame* frame)
        {
            VuoRelease(frame->image);
        }
    };
 
    LinkedList<AudioFrame> audioFrames;
    LinkedList<VideoFrame> videoFrames;
    LinkedList<AVPacket> videoPackets;
    LinkedList<AVPacket> audioPackets;
 
    struct VideoInfo
    {
        int64_t first_pts;      // first pts value in video stream time
        int64_t last_pts;       // finish in video stream time
        int64_t max_pts;        // largest decoded pts in video stream time
        int64_t duration;       // duration in video stream time
    };
 
    struct AVContainer
    {
        AVFormatContext*    formatCtx;
        AVCodecContext*     videoCodecCtx;
        AVCodecContext*     audioCodecCtx;
 
        struct SwrContext *swr_ctx; // for resampling audio
        int bytesPerAudioSample;    // Size of an audio samples (eg, sizeof(float), sizeof(int), etc).
 
        // Stream contains information about the channel, ex time base and length.
        int videoStreamIndex, audioStreamIndex;
        AVStream* videoStream;
        AVStream* audioStream;
 
        VideoInfo videoInfo;
        VideoInfo audioInfo;
 
        bool seekUnavailable;
    };
 
    AVContainer container;
 
    VuoUrl mVideoPath;
 
    double mPlaybackRate;
 
    int64_t lastDecodedVideoPts;
    int64_t lastSentVideoPts;
    int64_t lastDecodedAudioPts;
    bool showedTimestampGapWarning;
    bool showedSeekIgnoredWarning;
 
    double lastVideoTimestamp;  // Last sent video timestamp in seconds, relative to 0 being start of movie (actual timestamp could be negative, or non-zero)
    double lastAudioTimestamp;  // Last sent audio timestamp in seconds.
 
    // Audio sync cumulative difference in last decoded timestamp.
    double audio_diff_cum;
    double audio_diff_avg_coef;
    // The threshold at which to begin modifying returned audio sample arrays to better sync with video.
    double audio_diff_threshold;
    int audio_diff_avg_count;
 
    // If the playback rate is 1, enable audio
    bool audioIsEnabled;
 
    // Holds decoded audio between extraction from frame and conversion to VuoAudioSamples.
    uint8_t** audio_buf;
 
    // The size of a single channel of audio in audio_buf, set by DecodeAudio().  ( sampleCount * sizeof(double) )
    unsigned int audio_buf_size;
 
    // the current index in audio_buff (per-channel) that is being copied to VuoAudioSamples
    unsigned int audio_buf_index;
 
    // The numbder of audio channels.
    uint audio_channels;
 
    // it can take more than one call to DecodeAudio() to extract all samples from an audio
    // packet, so store it locally til done.
    AVPacket audio_packet;
    uint8_t *audio_pkt_data;
    int audio_pkt_size;
 
    bool isReady = false;
 
    // Calls base-constructor, nothing else.
    VuoFfmpegDecoder(VuoUrl url);
 
    // Prepare internal stuff for decoding.
    bool Initialize() VuoWarnUnusedResult;
 
    // Initialize the video context
    bool InitializeVideo(VuoFfmpegDecoder::AVContainer& container) VuoWarnUnusedResult;
 
    // Initialize the audio context
    bool InitializeAudio(VuoFfmpegDecoder::AVContainer& container) VuoWarnUnusedResult;
 
    // Read information about starting and ending timestamp + duration for both container.videoInfo & container.audioInfo
    bool InitializeVideoInfo() VuoWarnUnusedResult;
 
    // Advance the next packet and place in either video or audio queue.
    bool NextPacket() VuoWarnUnusedResult;
 
    // Decode a frame of audio, and store the frames in audioFrames list
    bool DecodeAudioFrame() VuoWarnUnusedResult;
 
    // Decode a frame of video, and store the frames in videoFrames list
    bool DecodeVideoFrame() VuoWarnUnusedResult;
 
    // Decode a chunk of video frames prior to the current video timestamp.
    bool DecodePreceedingVideoFrames() VuoWarnUnusedResult;
 
    // Grab the next frame from audioFrames, calling NextPacket & DecodeAudio as necessary.
    bool FillAudioBuffer() VuoWarnUnusedResult;
 
    // Seek to presentation timestamp.
    void SeekToPts(int64_t pts, VuoVideoFrame *frame);
 
    bool seeking;
 
    // After av_seek, use this to step the decoded frames to a more exact position
    bool StepVideoFrame(int64_t pts, VuoVideoFrame *frame) VuoWarnUnusedResult;
    bool StepAudioFrame(int64_t pts) VuoWarnUnusedResult;
 
    void ClearAudioBuffer();
 
    const double MAX_AUDIO_LEAD = .022;
 
    const double MAX_AUDIO_LATENCY = -.045;
 
    const unsigned int MAX_FRAME_SKIP = 1;
 
    double AudioOffset();
};