#include "AudioStream.h" #include "WallpaperEngine/Logging/Log.h" #include #include #include // maximum size of the queue to prevent reading too much data using namespace WallpaperEngine::Audio; int audio_read_thread (void* arg) { SDL_mutex* waitMutex = SDL_CreateMutex (); auto* stream = static_cast (arg); AVPacket* packet = av_packet_alloc (); int ret = 0; if (waitMutex == nullptr) sLog.exception ("Cannot create mutex for audio playback waiting"); while (ret >= 0 && stream->getAudioContext ().getApplicationContext ().state.general.keepRunning) { // give the cpu some time to play the queued frames if there's enough info there if (stream->getQueueSize () >= MAX_QUEUE_SIZE || (stream->getQueuePacketCount () > MIN_FRAMES && (av_q2d (stream->getTimeBase ()) * stream->getQueueDuration () > 1.0))) { SDL_LockMutex (waitMutex); SDL_CondWaitTimeout (stream->getWaitCondition (), waitMutex, 10); SDL_UnlockMutex (waitMutex); continue; } ret = av_read_frame (stream->getFormatContext (), packet); if (ret == AVERROR_EOF) { // seek to the beginning of the file again avformat_seek_file (stream->getFormatContext (), stream->getAudioStream (), 0, 0, 0, ~AVSEEK_FLAG_FRAME); avcodec_flush_buffers (stream->getContext ()); // ensure the thread is not killed if audio has to be looped if (stream->isRepeat ()) ret = 0; continue; } // TODO: PROPERLY IMPLEMENT THIS if (packet->stream_index == stream->getAudioStream ()) stream->queuePacket (packet); else av_packet_unref (packet); if (!stream->isInitialized ()) break; } // stop the audio too just in case stream->stop (); SDL_DestroyMutex (waitMutex); return 0; } static int audio_read_data_callback (void* streamarg, uint8_t* buffer, int buffer_size) { const auto stream = static_cast (streamarg); // check if we're at eof and return the right value if (stream->getBuffer ()->eof ()) { return AVERROR_EOF; } stream->getBuffer ()->read (reinterpret_cast (buffer), buffer_size); if (stream->getBuffer ()->fail () && !stream->getBuffer ()->eof ()) { return AVERROR_INVALIDDATA; } // return read bytes only return stream->getBuffer ()->gcount (); } int64_t audio_seek_data_callback (void* streamarg, int64_t offset, int whence) { const auto stream = static_cast (streamarg); // reset error state stream->getBuffer ()->clear (); if (whence & AVSEEK_SIZE) { const auto current = stream->getBuffer ()->tellg (); stream->getBuffer ()->seekg (0, std::ios_base::end); const auto end = stream->getBuffer ()->tellg (); stream->getBuffer ()->seekg (current, std::ios_base::beg); return end; } switch (whence) { case SEEK_CUR: stream->getBuffer ()->seekg (offset, std::ios_base::cur); break; case SEEK_SET: stream->getBuffer ()->seekg (offset, std::ios_base::beg); break; case SEEK_END: stream->getBuffer ()->seekg (offset, std::ios_base::end); break; } return 0; } AudioStream::AudioStream (AudioContext& context, const std::string& filename) : m_audioContext (context) { this->loadCustomContent (filename.c_str ()); } AudioStream::AudioStream (AudioContext& context, const ReadStreamSharedPtr& buffer) : m_audioContext (context) { // setup a custom context first this->m_formatContext = avformat_alloc_context (); if (this->m_formatContext == nullptr) sLog.exception ("Cannot allocate ffmpeg format context"); this->m_buffer = buffer; // setup custom io for it this->m_formatContext->pb = avio_alloc_context (static_cast (av_malloc (4096)), 4096, 0, this, &audio_read_data_callback, nullptr, &audio_seek_data_callback); if (this->m_formatContext->pb == nullptr) sLog.exception ("Cannot create avio context"); // continue the normal load procedure this->loadCustomContent (); } AudioStream::AudioStream (AudioContext& audioContext, AVCodecContext* context) : m_audioContext (audioContext), m_context (context), m_queue (new PacketQueue) { this->initialize (); } AudioStream::~AudioStream () { if (this->m_swrctx != nullptr && swr_is_initialized (this->m_swrctx) == true) swr_close (this->m_swrctx); if (this->m_swrctx != nullptr) swr_free (&this->m_swrctx); // TODO: FREE EVERYTHING ELSE THAT THIS CLASS HOLDS! } void AudioStream::loadCustomContent (const char* filename) { if (avformat_open_input (&this->m_formatContext, filename, nullptr, nullptr) != 0) sLog.exception ("Cannot open audio file: ", filename); if (avformat_find_stream_info (this->m_formatContext, nullptr) < 0) sLog.exception ("Cannot determine file format: ", filename); // find the audio stream for (unsigned int i = 0; i < this->m_formatContext->nb_streams; i++) { if (this->m_formatContext->streams [i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && this->m_audioStream == NO_AUDIO_STREAM) { this->m_audioStream = i; } } if (this->m_audioStream == NO_AUDIO_STREAM) sLog.exception ("Cannot find an audio stream in file ", filename); // get the decoder for it and alloc the required context const AVCodec* aCodec = avcodec_find_decoder (this->m_formatContext->streams [this->m_audioStream]->codecpar->codec_id); if (aCodec == nullptr) sLog.exception ("Cannot initialize audio decoder for file: ", filename); // alocate context AVCodecContext* avCodecContext = avcodec_alloc_context3 (aCodec); if (avcodec_parameters_to_context (avCodecContext, this->m_formatContext->streams [this->m_audioStream]->codecpar) != 0) sLog.exception ("Cannot initialize audio decoder parameters"); // finally open avcodec_open2 (avCodecContext, aCodec, nullptr); // initialize default data this->m_context = avCodecContext; this->m_queue = new PacketQueue; this->initialize (); // initialize an SDL thread to read the file SDL_CreateThread (audio_read_thread, filename, this); } void AudioStream::initialize () { // allocate the FIFO buffer #if FF_API_FIFO_OLD_API this->m_queue->packetList = av_fifo_alloc (sizeof (MyAVPacketList)); #else this->m_queue->packetList = av_fifo_alloc2 (1, sizeof (MyAVPacketList), AV_FIFO_FLAG_AUTO_GROW); #endif #if FF_API_OLD_CHANNEL_LAYOUT int64_t out_channel_layout; // set output audio channels based on the input audio channels switch (this->m_audioContext.getChannels ()) { case 1: out_channel_layout = AV_CH_LAYOUT_MONO; break; case 2: out_channel_layout = AV_CH_LAYOUT_STEREO; break; default: out_channel_layout = AV_CH_LAYOUT_SURROUND; break; } // initialize swrctx this->m_swrctx = swr_alloc_set_opts (nullptr, out_channel_layout, this->m_audioContext.getFormat (), this->m_audioContext.getSampleRate (), this->getContext ()->channel_layout, this->getContext ()->sample_fmt, this->getContext ()->sample_rate, 0, nullptr); #else AVChannelLayout out_channel_layout; int64_t out_channel_mask; // set output audio channels based on the input audio channels switch (this->m_audioContext.getChannels ()) { case 1: out_channel_mask = AV_CH_LAYOUT_MONO; break; case 2: out_channel_mask = AV_CH_LAYOUT_STEREO; break; default: out_channel_mask = AV_CH_LAYOUT_SURROUND; break; } if (av_channel_layout_from_mask (&out_channel_layout, out_channel_mask) != 0) sLog.exception ("Cannot get channel layout from mask"); swr_alloc_set_opts2 (&this->m_swrctx, &out_channel_layout, this->m_audioContext.getFormat (), this->m_audioContext.getSampleRate (), &this->m_context->ch_layout, this->m_context->sample_fmt, this->m_context->sample_rate, 0, nullptr); #endif if (this->m_swrctx == nullptr) sLog.exception ("Cannot initialize swrctx for audio resampling"); // initialize the context if (swr_init (this->m_swrctx) < 0) sLog.exception ("Failed to initialize the resampling context."); // setup the queue information this->m_queue->mutex = SDL_CreateMutex (); this->m_queue->cond = SDL_CreateCond (); this->m_queue->wait = SDL_CreateCond (); this->m_initialized = true; } void AudioStream::queuePacket (AVPacket* pkt) { // clone the packet AVPacket* clone = av_packet_alloc (); if (clone == nullptr) { av_packet_unref (clone); return; } av_packet_move_ref (clone, pkt); SDL_LockMutex (this->m_queue->mutex); const bool gotQueued = this->doQueue (clone); SDL_UnlockMutex (this->m_queue->mutex); if (!gotQueued) av_packet_free (&pkt); } bool AudioStream::doQueue (AVPacket* pkt) { MyAVPacketList entry {pkt}; #if FF_API_FIFO_OLD_API if (av_fifo_space (this->m_queue->packetList) < static_cast (sizeof (entry))) if (av_fifo_grow (this->m_queue->packetList, sizeof (entry)) < 0) return false; av_fifo_generic_write (this->m_queue->packetList, &entry, sizeof (entry), nullptr); #else // write the entry if possible if (av_fifo_write (this->m_queue->packetList, &entry, 1) < 0) return false; #endif this->m_queue->nb_packets++; this->m_queue->size += entry.packet->size + sizeof (entry); this->m_queue->duration += entry.packet->duration; SDL_CondSignal (this->m_queue->cond); return true; } void AudioStream::dequeuePacket (AVPacket* output) { MyAVPacketList entry{}; SDL_LockMutex (this->m_queue->mutex); while (this->m_audioContext.getApplicationContext ().state.general.keepRunning) { #if FF_API_FIFO_OLD_API int ret = -1; if (av_fifo_size (this->m_queue->packetList) >= static_cast (sizeof (entry))) ret = av_fifo_generic_read (this->m_queue->packetList, &entry, sizeof (entry), nullptr); #else const int ret = av_fifo_read (this->m_queue->packetList, &entry, 1); #endif // enough data available, read it if (ret >= 0) { this->m_queue->nb_packets--; this->m_queue->size -= entry.packet->size + sizeof (entry); this->m_queue->duration -= entry.packet->duration; // move the reference and free the old one av_packet_move_ref (output, entry.packet); av_packet_free (&entry.packet); break; } // make the thread wait if nothing was available SDL_CondWait (this->m_queue->cond, this->m_queue->mutex); } SDL_UnlockMutex (this->m_queue->mutex); } AVCodecContext* AudioStream::getContext () const { return this->m_context; } AVFormatContext* AudioStream::getFormatContext () const { return this->m_formatContext; } int AudioStream::getAudioStream () const { return this->m_audioStream; } bool AudioStream::isInitialized () const { return this->m_initialized; } void AudioStream::setRepeat (const bool newRepeat) { this->m_repeat = newRepeat; } bool AudioStream::isRepeat () const { return this->m_repeat; } ReadStreamSharedPtr& AudioStream::getBuffer () { return this->m_buffer; } SDL_cond* AudioStream::getWaitCondition () const { return this->m_queue->wait; } int AudioStream::getQueueSize () const { return this->m_queue->size; } int AudioStream::getQueuePacketCount () const { return this->m_queue->nb_packets; } AVRational AudioStream::getTimeBase () const { if (this->m_audioStream == NO_AUDIO_STREAM) { return {0, 0}; } return this->m_formatContext->streams [this->m_audioStream]->time_base; } int64_t AudioStream::getQueueDuration () const { return this->m_queue->duration; } bool AudioStream::isQueueEmpty () const { return this->m_queue->nb_packets == 0; } SDL_mutex* AudioStream::getMutex () const { return this->m_queue->mutex; } void AudioStream::stop () { if (!this->isInitialized ()) return; // stop the threads running this->m_initialized = false; } int AudioStream::resampleAudio (const AVFrame* decoded_audio_frame, uint8_t* out_buf) { int out_linesize = 0; int ret; int out_nb_channels; int out_nb_samples; uint8_t** resampled_data = nullptr; int resampled_data_size; // retrieve number of audio samples (per channel) const int in_nb_samples = decoded_audio_frame->nb_samples; if (in_nb_samples <= 0) { sLog.error ("in_nb_samples error."); return -1; } int max_out_nb_samples = out_nb_samples = av_rescale_rnd (in_nb_samples, this->m_audioContext.getSampleRate (), this->getContext ()->sample_rate, AV_ROUND_UP); // check rescaling was successful if (max_out_nb_samples <= 0) { sLog.error ("av_rescale_rnd error."); return -1; } // get number of output audio channels #if FF_API_OLD_CHANNEL_LAYOUT int64_t out_channel_layout; // set output audio channels based on the input audio channels switch (this->m_audioContext.getChannels ()) { case 1: out_channel_layout = AV_CH_LAYOUT_MONO; break; case 2: out_channel_layout = AV_CH_LAYOUT_STEREO; break; default: out_channel_layout = AV_CH_LAYOUT_SURROUND; break; } out_nb_channels = av_get_channel_layout_nb_channels (out_channel_layout); #else out_nb_channels = this->getContext ()->ch_layout.nb_channels; #endif ret = av_samples_alloc_array_and_samples (&resampled_data, &out_linesize, out_nb_channels, out_nb_samples, this->m_audioContext.getFormat (), 0); if (ret < 0) { sLog.error ("av_samples_alloc_array_and_samples() error: Could not allocate destination samples."); return -1; } // retrieve output samples number taking into account the progressive delay out_nb_samples = av_rescale_rnd (swr_get_delay (this->m_swrctx, this->getContext ()->sample_rate) + in_nb_samples, this->m_audioContext.getSampleRate (), this->getContext ()->sample_rate, AV_ROUND_UP); // check output samples number was correctly retrieved if (out_nb_samples <= 0) { sLog.error ("av_rescale_rnd error"); return -1; } if (out_nb_samples > max_out_nb_samples) { // free memory block and set pointer to NULL av_freep (&resampled_data [0]); // Allocate a samples buffer for out_nb_samples samples ret = av_samples_alloc (resampled_data, &out_linesize, out_nb_channels, out_nb_samples, this->m_audioContext.getFormat (), 1); // check samples buffer correctly allocated if (ret < 0) { sLog.error ("av_samples_alloc failed."); return -1; } max_out_nb_samples = out_nb_samples; } // do the actual audio data resampling ret = swr_convert (this->m_swrctx, resampled_data, max_out_nb_samples, decoded_audio_frame->data, decoded_audio_frame->nb_samples); // check audio conversion was successful if (ret < 0) { sLog.error ("swr_convert_error."); return -1; } // Get the required buffer size for the given audio parameters resampled_data_size = av_samples_get_buffer_size (&out_linesize, out_nb_channels, ret, this->m_audioContext.getFormat (), 1); // check audio buffer size if (resampled_data_size < 0) { sLog.error ("av_samples_get_buffer_size error."); return -1; } // copy the resampled data to the output buffer memcpy (out_buf, resampled_data [0], resampled_data_size); /* * Memory Cleanup. */ if (resampled_data) { // free memory block and set pointer to NULL av_freep (&resampled_data [0]); } av_freep (&resampled_data); return resampled_data_size; } int AudioStream::decodeFrame (uint8_t* audioBuffer, const int bufferSize) { static AVPacket* pkt = av_packet_alloc (); static uint8_t* audio_pkt_data = nullptr; static int audio_pkt_size = 0; int len1, data_size; // allocate a new frame, used to decode audio packets static AVFrame* avFrame = av_frame_alloc (); if (!avFrame) { sLog.error ("Could not allocate AVFrame.\n"); return -1; } // block until there's any data in the buffers while (this->m_audioContext.getApplicationContext ().state.general.keepRunning) { while (audio_pkt_size > 0 && this->m_audioContext.getApplicationContext ().state.general.keepRunning) { int got_frame = 0; int ret = avcodec_receive_frame (this->getContext (), avFrame); if (ret == 0) got_frame = 1; if (ret == AVERROR (EAGAIN)) ret = 0; if (ret == 0) ret = avcodec_send_packet (this->getContext (), pkt); if (ret < 0 && ret != AVERROR (EAGAIN)) return -1; len1 = pkt->size; if (len1 < 0) { // if error, skip frame audio_pkt_size = 0; break; } audio_pkt_data += len1; audio_pkt_size -= len1; data_size = 0; if (got_frame) { // audio resampling data_size = this->resampleAudio (avFrame, audioBuffer); assert (data_size <= bufferSize); } if (data_size <= 0) { // no data found, keep waiting continue; } // some data was found return data_size; } if (pkt->data) av_packet_unref (pkt); this->dequeuePacket (pkt); audio_pkt_data = pkt->data; audio_pkt_size = pkt->size; } return 0; } AudioContext& AudioStream::getAudioContext () const { return this->m_audioContext; }