feat: Implement audio playback in movies, refactor BikFile

.travis.yml

@@ -38,6 +38,7 @@ addons:
       - libavcodec-dev
       - libavformat-dev
       - libavutil-dev
+      - libswresample-dev
       - libswscale-dev
   coverity_scan:
     project:

CMakeLists.txt

@@ -42,7 +42,7 @@ if(ENABLE_VIDEO)
     if(WIN32)
         find_package(FFMPEG REQUIRED)
     else()
-        pkg_check_modules(FFMPEG REQUIRED libavcodec libavformat libavutil libswscale)
+        pkg_check_modules(FFMPEG REQUIRED libavcodec libavformat libavutil libswresample libswscale)
     endif()
     include_directories(${FFMPEG_INCLUDE_DIRS})
 endif()

src/game/game.cpp

@@ -123,6 +123,11 @@ void Game::playVideo(const string &name) {
     bik.load();
 
     _video = bik.video();
+
+    shared_ptr<AudioStream> audio(_video->audio());
+    if (audio) {
+        AudioPlayer::instance().play(audio, AudioType::Sound);
+    }
 }
 
 void Game::openMainMenu() {

src/video/bikfile.cpp

@@ -21,6 +21,7 @@
 
 #include <boost/filesystem.hpp>
 
+#include "../audio/stream.h"
 #include "../system/log.h"
 #include "../system/streamreader.h"
 
@@ -31,6 +32,7 @@
 extern "C" {
 
 #include "libavformat/avformat.h"
+#include "libswresample/swresample.h"
 #include "libswscale/swscale.h"
 
 }
@@ -41,12 +43,248 @@ namespace fs = boost::filesystem;
 
 using namespace std;
 
+using namespace reone::audio;
+
 namespace reone {
 
 namespace video {
 
 static const char kSignature[] = "BIKi";
 
+#if REONE_ENABLE_VIDEO
+
+class BinkVideoDecoder {
+public:
+    BinkVideoDecoder(const fs::path &path) : _path(path) {
+    }
+
+    ~BinkVideoDecoder() {
+        deinit();
+    }
+
+    void deinit() {
+        if (_videoBuffer) {
+            av_free(_videoBuffer);
+            _videoBuffer = nullptr;
+        }
+        if (_frameRgb) {
+            av_free(_frameRgb);
+            _frameRgb = nullptr;
+        }
+        if (_frame) {
+            av_free(_frame);
+            _frame = nullptr;
+        }
+        if (_swrContext) {
+            swr_free(&_swrContext);
+        }
+        if (_swsContext) {
+            sws_freeContext(_swsContext);
+            _swsContext = nullptr;
+        }
+        if (_audioCodecCtx) {
+            avcodec_close(_audioCodecCtx);
+            _audioCodecCtx = nullptr;
+        }
+        if (_videoCodecCtx) {
+            avcodec_close(_videoCodecCtx);
+            _videoCodecCtx = nullptr;
+        }
+        if (_formatCtx) {
+            avformat_close_input(&_formatCtx);
+        }
+    }
+
+    void load() {
+        openInput(_path);
+        findStreams();
+        openVideoCodec();
+
+        if (hasAudio()) {
+            openAudioCodec();
+        }
+        initConverters();
+        initFrames();
+        initVideo();
+        processStream();
+    }
+
+    shared_ptr<Video> video() const {
+        return _video;
+    }
+
+private:
+    fs::path _path;
+    AVFormatContext *_formatCtx { nullptr };
+    int _videoStreamIdx { -1 };
+    int _audioStreamIdx { -1 };
+    AVCodecContext *_videoCodecCtx { nullptr };
+    AVCodecContext *_audioCodecCtx { nullptr };
+    SwsContext *_swsContext { nullptr };
+    SwrContext *_swrContext { nullptr };
+    AVFrame *_frame { nullptr };
+    AVFrame *_frameRgb { nullptr };
+    uint8_t *_videoBuffer { nullptr };
+    shared_ptr<Video> _video;
+
+    void openInput(const fs::path &path) {
+        if (avformat_open_input(&_formatCtx, path.string().c_str(), nullptr, nullptr) != 0) {
+            throw runtime_error("BIK: failed to open");
+        }
+    }
+
+    void findStreams() {
+        if (avformat_find_stream_info(_formatCtx, nullptr) != 0) {
+            throw runtime_error("BIK: failed to find stream info");
+        }
+        for (uint32_t i = 0; i < _formatCtx->nb_streams; ++i) {
+            AVMediaType codecType = _formatCtx->streams[i]->codec->codec_type;
+            switch (codecType) {
+                case AVMEDIA_TYPE_VIDEO:
+                    _videoStreamIdx = i;
+                    break;
+                case AVMEDIA_TYPE_AUDIO:
+                    _audioStreamIdx = i;
+                    break;
+                default:
+                    break;
+            }
+        }
+        if (_videoStreamIdx == -1) {
+            throw runtime_error("BIK: video stream not found");
+        }
+    }
+
+    void openVideoCodec() {
+        AVCodecContext *codecCtx = _formatCtx->streams[_videoStreamIdx]->codec;
+        AVCodec *codec = avcodec_find_decoder(codecCtx->codec_id);
+        if (!codec) {
+            throw runtime_error("BIK: video codec not found");
+        }
+        _videoCodecCtx = avcodec_alloc_context3(codec);
+        if (avcodec_copy_context(_videoCodecCtx, codecCtx) != 0) {
+            throw runtime_error("BIK: failed to copy a video codec context");
+        }
+        if (avcodec_open2(_videoCodecCtx, codec, nullptr) != 0) {
+            throw runtime_error("BIK: failed to open a video codec");
+        }
+    }
+
+    bool hasAudio() const {
+        return _audioStreamIdx != -1;
+    }
+
+    void openAudioCodec() {
+        AVCodecContext *codecCtx = _formatCtx->streams[_audioStreamIdx]->codec;
+        AVCodec *codec = avcodec_find_decoder(codecCtx->codec_id);
+        if (!codec) {
+            throw runtime_error("BIK: audio codec not found");
+        }
+        _audioCodecCtx = avcodec_alloc_context3(codec);
+        if (avcodec_copy_context(_audioCodecCtx, codecCtx) != 0) {
+            throw runtime_error("BIK: failed to copy an audio codec context");
+        }
+        if (avcodec_open2(_audioCodecCtx, codec, nullptr) != 0) {
+            throw runtime_error("BIK: failed to open an audio codec");
+        }
+    }
+
+    void initConverters() {
+        _swsContext = sws_getContext(
+            _videoCodecCtx->width, _videoCodecCtx->height,
+            _videoCodecCtx->pix_fmt,
+            _videoCodecCtx->width, _videoCodecCtx->height,
+            AV_PIX_FMT_RGB24,
+            SWS_BILINEAR,
+            nullptr, nullptr, nullptr);
+
+        if (hasAudio()) {
+            _swrContext = swr_alloc_set_opts(
+                nullptr,
+                AV_CH_LAYOUT_MONO, AV_SAMPLE_FMT_S16, _audioCodecCtx->sample_rate,
+                _audioCodecCtx->channel_layout, _audioCodecCtx->sample_fmt, _audioCodecCtx->sample_rate,
+                0, nullptr);
+
+            swr_init(_swrContext);
+        }
+    }
+
+    void initFrames() {
+        _frame = av_frame_alloc();
+        _frameRgb = av_frame_alloc();
+
+        int bufSize = avpicture_get_size(AV_PIX_FMT_RGB24, _videoCodecCtx->width, _videoCodecCtx->height);
+        _videoBuffer = static_cast<uint8_t *>(av_malloc(bufSize));
+        avpicture_fill(reinterpret_cast<AVPicture *>(_frameRgb), _videoBuffer, AV_PIX_FMT_RGB24, _videoCodecCtx->width, _videoCodecCtx->height);
+    }
+
+    void initVideo() {
+        AVRational &frameRate = _formatCtx->streams[_videoStreamIdx]->r_frame_rate;
+
+        _video = make_shared<Video>();
+        _video->_width = _videoCodecCtx->width;
+        _video->_height = _videoCodecCtx->height;
+        _video->_fps = frameRate.num / static_cast<float>(frameRate.den);
+
+        if (hasAudio()) {
+            _video->_audio = make_shared<AudioStream>();
+        }
+    }
+
+    void processStream() {
+        AVPacket packet;
+
+        while (av_read_frame(_formatCtx, &packet) >= 0) {
+            int gotFrame = 0;
+
+            if (packet.stream_index == _videoStreamIdx) {
+                avcodec_decode_video2(_videoCodecCtx, _frame, &gotFrame, &packet);
+                if (gotFrame == 0) continue;
+
+                sws_scale(
+                    _swsContext,
+                    _frame->data, _frame->linesize, 0, _videoCodecCtx->height,
+                    _frameRgb->data, _frameRgb->linesize);
+
+                ByteArray data(3ll * _videoCodecCtx->width * _videoCodecCtx->height);
+                for (int y = 0; y < _videoCodecCtx->height; ++y) {
+                    int dstIdx = 3 * _videoCodecCtx->width * y;
+                    uint8_t *src = _frameRgb->data[0] + static_cast<long long>(y) * _frameRgb->linesize[0];
+                    memcpy(&data[dstIdx], src, 3ll * _videoCodecCtx->width);
+                }
+
+                Video::Frame frame;
+                frame.data = move(data);
+                _video->_frames.push_back(move(frame));
+
+            } else if (_audioStreamIdx != -1 && packet.stream_index == _audioStreamIdx) {
+                avcodec_send_packet(_audioCodecCtx, &packet);
+                avcodec_receive_frame(_audioCodecCtx, _frame);
+
+                int sampleCount = swr_get_out_samples(_swrContext, _frame->nb_samples);
+                int bufSize = av_samples_get_buffer_size(nullptr, 1, sampleCount, AV_SAMPLE_FMT_S16, 1);
+                ByteArray samples(bufSize);
+                uint8_t *samplesPtr = reinterpret_cast<uint8_t *>(&samples[0]);
+
+                swr_convert(
+                    _swrContext,
+                    &samplesPtr, sampleCount,
+                    const_cast<const uint8_t **>(&_frame->extended_data[0]), _frame->nb_samples);
+
+                AudioStream::Frame frame;
+                frame.format = AudioFormat::Mono16;
+                frame.sampleRate = _audioCodecCtx->sample_rate;
+                frame.samples = move(samples);
+                _video->_audio->add(move(frame));
+            }
+        }
+
+        av_free_packet(&packet);
+    }
+};
+
+#endif
+
 BikFile::BikFile(const fs::path &path) : _path(path) {
 }
 
@@ -55,92 +293,13 @@ void BikFile::load() {
     if (!fs::exists(_path)) {
         throw runtime_error("BIK: file not found: " + _path.string());
     }
-    AVFormatContext *formatCtx = nullptr;
-    if (avformat_open_input(&formatCtx, _path.string().c_str(), nullptr, nullptr) != 0) {
-        throw runtime_error("BIK: failed to open");
-    }
-    if (avformat_find_stream_info(formatCtx, nullptr) != 0) {
-        throw runtime_error("BIK: failed to find stream info");
-    }
-    int videoStream = -1;
-    for (uint32_t i = 0; i < formatCtx->nb_streams; ++i) {
-        if (formatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
-            videoStream = i;
-            break;
-        }
-    }
-    if (videoStream == -1) {
-        throw runtime_error("BIK: video stream not found");
-    }
-    AVCodecContext *codecCtx = formatCtx->streams[videoStream]->codec;
-    AVCodec *codec = avcodec_find_decoder(codecCtx->codec_id);
-    if (!codec) {
-        throw runtime_error("BIK: codec not found");
-    }
-    AVCodecContext *codecCtxOrig = codecCtx;
-    codecCtx = avcodec_alloc_context3(codec);
-    if (avcodec_copy_context(codecCtx, codecCtxOrig) != 0) {
-        throw runtime_error("BIK: failed to copy codec context");
-    }
-    if (avcodec_open2(codecCtx, codec, nullptr) != 0) {
-        throw runtime_error("BIK: failed to open codec");
-    }
-    AVFrame *frame = av_frame_alloc();
-    AVFrame *frameRGB = av_frame_alloc();
-    int numBytes = avpicture_get_size(AV_PIX_FMT_RGB24, codecCtx->width, codecCtx->height);
-    uint8_t *buffer = static_cast<uint8_t *>(av_malloc(numBytes));
-    avpicture_fill(reinterpret_cast<AVPicture *>(frameRGB), buffer, AV_PIX_FMT_RGB24, codecCtx->width, codecCtx->height);
-    AVPacket packet;
-    SwsContext *swsCtx = sws_getContext(
-        codecCtx->width, codecCtx->height,
-        codecCtx->pix_fmt,
-        codecCtx->width, codecCtx->height,
-        AV_PIX_FMT_RGB24,
-        SWS_BILINEAR,
-        nullptr, nullptr, nullptr);
 
-    AVRational &fps = formatCtx->streams[videoStream]->r_frame_rate;
-
-    _video = make_shared<Video>();
-    _video->_width = codecCtx->width;
-    _video->_height = codecCtx->height;
-    _video->_fps = fps.num / static_cast<float>(fps.den);
-
-    while (av_read_frame(formatCtx, &packet) >= 0) {
-        if (packet.stream_index != videoStream) continue;
-
-        int frameFinished = 0;
-        avcodec_decode_video2(codecCtx, frame, &frameFinished, &packet);
-
-        if (frameFinished == 0) continue;
-
-        sws_scale(
-            swsCtx,
-            frame->data, frame->linesize, 0, codecCtx->height,
-            frameRGB->data, frameRGB->linesize);
-
-        ByteArray data(3ll * codecCtx->width * codecCtx->height);
-        for (int y = 0; y < codecCtx->height; ++y) {
-            int dstIdx = 3 * codecCtx->width * y;
-            uint8_t *src = frameRGB->data[0] + static_cast<long long>(y) * frameRGB->linesize[0];
-            memcpy(&data[dstIdx], src, 3ll * codecCtx->width);
-        }
-
-        Video::Frame frame;
-        frame.data = move(data);
-        _video->_frames.push_back(move(frame));
-    }
+    BinkVideoDecoder decoder(_path);
+    decoder.load();
 
+    _video = decoder.video();
     _video->init();
-
-    av_free_packet(&packet);
-    av_free(buffer);
-    av_free(frameRGB);
-    av_free(frame);
-    avcodec_close(codecCtx);
-    avcodec_close(codecCtxOrig);
-    avformat_close_input(&formatCtx);
-#endif
+#endif // REONE_ENABLE_VIDEO
 }
 
 shared_ptr<Video> BikFile::video() const {

src/video/bikfile.h

@@ -19,12 +19,19 @@
 
 #include <boost/filesystem/path.hpp>
 
+#include "../audio/types.h"
+
 namespace reone {
 
 namespace video {
 
 class Video;
 
+/**
+ * Represents a Bink Video file. Uses FFmpeg for loading.
+ *
+ * http://dranger.com/ffmpeg/ffmpeg.html
+ */
 class BikFile {
 public:
     BikFile(const boost::filesystem::path &path);

src/video/video.cpp

@@ -30,6 +30,7 @@
 
 using namespace std;
 
+using namespace reone::audio;
 using namespace reone::render;
 
 namespace reone {
@@ -99,6 +100,10 @@ bool Video::isFinished() const {
     return _finished;
 }
 
+shared_ptr<AudioStream> Video::audio() const {
+    return _audio;
+}
+
 } // namespace video
 
 } // namespace reone

src/video/video.h

@@ -18,12 +18,19 @@
 #pragma once
 
 #include <cstdint>
+#include <memory>
 #include <vector>
 
 #include "../system/types.h"
 
 namespace reone {
 
+namespace audio {
+
+class AudioStream;
+
+}
+
 namespace video {
 
 class BikFile;
@@ -38,6 +45,8 @@ public:
 
     bool isFinished() const;
 
+    std::shared_ptr<audio::AudioStream> audio() const;
+
 private:
     struct Frame {
         ByteArray data;
@@ -51,8 +60,9 @@ private:
     uint32_t _textureId { 0 };
     float _time { 0.0f };
     bool _finished { false };
+    std::shared_ptr<audio::AudioStream> _audio;
 
-    friend class BikFile;
+    friend class BinkVideoDecoder;
 };
 
 } // namespace video