Added basic support for audio processing so backgrounds can react to sound playing in the system

Signed-off-by: Alexis Maiquez <almamu@almamu.com>

CMakeLists.txt

@@ -11,6 +11,8 @@ if(NOT ERRORONLY)
     set(ERRORONLY 0)
 endif()
 
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing")
+
 # if you're developing you might find this debug option useful for shader output, although RenderDoc is encouraged
 add_compile_definitions(ERRORONLY=${ERRORONLY})
 add_compile_definitions(DATADIR="${DATADIR}")
@@ -45,6 +47,9 @@ add_executable(
     linux-wallpaperengine
     main.cpp
 
+    src/External/Android/fft.cpp
+    src/External/Android/fft.h
+
     src/Steam/FileSystem/FileSystem.h
     src/Steam/FileSystem/FileSystem.cpp
 
@@ -77,6 +82,11 @@ add_executable(
     src/WallpaperEngine/Core/Core.h
     src/WallpaperEngine/Core/Core.cpp
 
+    src/WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.cpp
+    src/WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.h
+    src/WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.cpp
+    src/WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.h
+
     src/WallpaperEngine/Audio/Drivers/Detectors/CPulseAudioPlayingDetector.cpp
     src/WallpaperEngine/Audio/Drivers/Detectors/CPulseAudioPlayingDetector.h
     src/WallpaperEngine/Audio/Drivers/Detectors/CAudioPlayingDetector.cpp
@@ -288,7 +298,5 @@ if(HAVE_XSETIOERROREXITHANDLER)
 endif()
 
 # set some install parameters if not in debug mode
-if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
-    install(TARGETS linux-wallpaperengine)
-    install(DIRECTORY share/ DESTINATION share/${PROJECT_NAME})
-endif()
+install(TARGETS linux-wallpaperengine)
+install(DIRECTORY share/ DESTINATION share/${PROJECT_NAME})

src/External/Android/fft.cpp

@@ -0,0 +1,187 @@
+#include "fft.h"
+
+namespace External::Android
+{
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* A Fixed point implementation of Fast Fourier Transform (FFT). Complex numbers
+ * are represented by 32-bit integers, where higher 16 bits are real part and
+ * lower ones are imaginary part. Few compromises are made between efficiency,
+ * accuracy, and maintainability. To make it fast, arithmetic shifts are used
+ * instead of divisions, and bitwise inverses are used instead of negates. To
+ * keep it small, only radix-2 Cooley-Tukey algorithm is implemented, and only
+ * half of the twiddle factors are stored. Although there are still ways to make
+ * it even faster or smaller, it costs too much on one of the aspects.
+ */
+#include <stdio.h>
+#include <stdint.h>
+#ifdef __arm__
+#include <machine/cpu-features.h>
+#endif
+#define LOG_FFT_SIZE 10
+#define MAX_FFT_SIZE (1 << LOG_FFT_SIZE)
+    static const int32_t twiddle[MAX_FFT_SIZE / 4] = {
+        0x00008000, 0xff378001, 0xfe6e8002, 0xfda58006, 0xfcdc800a, 0xfc13800f,
+        0xfb4a8016, 0xfa81801e, 0xf9b88027, 0xf8ef8032, 0xf827803e, 0xf75e804b,
+        0xf6958059, 0xf5cd8068, 0xf5058079, 0xf43c808b, 0xf374809e, 0xf2ac80b2,
+        0xf1e480c8, 0xf11c80de, 0xf05580f6, 0xef8d8110, 0xeec6812a, 0xedff8146,
+        0xed388163, 0xec718181, 0xebab81a0, 0xeae481c1, 0xea1e81e2, 0xe9588205,
+        0xe892822a, 0xe7cd824f, 0xe7078276, 0xe642829d, 0xe57d82c6, 0xe4b982f1,
+        0xe3f4831c, 0xe3308349, 0xe26d8377, 0xe1a983a6, 0xe0e683d6, 0xe0238407,
+        0xdf61843a, 0xde9e846e, 0xdddc84a3, 0xdd1b84d9, 0xdc598511, 0xdb998549,
+        0xdad88583, 0xda1885be, 0xd95885fa, 0xd8988637, 0xd7d98676, 0xd71b86b6,
+        0xd65c86f6, 0xd59e8738, 0xd4e1877b, 0xd42487c0, 0xd3678805, 0xd2ab884c,
+        0xd1ef8894, 0xd13488dd, 0xd0798927, 0xcfbe8972, 0xcf0489be, 0xce4b8a0c,
+        0xcd928a5a, 0xccd98aaa, 0xcc218afb, 0xcb698b4d, 0xcab28ba0, 0xc9fc8bf5,
+        0xc9468c4a, 0xc8908ca1, 0xc7db8cf8, 0xc7278d51, 0xc6738dab, 0xc5c08e06,
+        0xc50d8e62, 0xc45b8ebf, 0xc3a98f1d, 0xc2f88f7d, 0xc2488fdd, 0xc198903e,
+        0xc0e990a1, 0xc03a9105, 0xbf8c9169, 0xbedf91cf, 0xbe329236, 0xbd86929e,
+        0xbcda9307, 0xbc2f9371, 0xbb8593dc, 0xbadc9448, 0xba3394b5, 0xb98b9523,
+        0xb8e39592, 0xb83c9603, 0xb7969674, 0xb6f196e6, 0xb64c9759, 0xb5a897ce,
+        0xb5059843, 0xb46298b9, 0xb3c09930, 0xb31f99a9, 0xb27f9a22, 0xb1df9a9c,
+        0xb1409b17, 0xb0a29b94, 0xb0059c11, 0xaf689c8f, 0xaecc9d0e, 0xae319d8e,
+        0xad979e0f, 0xacfd9e91, 0xac659f14, 0xabcd9f98, 0xab36a01c, 0xaaa0a0a2,
+        0xaa0aa129, 0xa976a1b0, 0xa8e2a238, 0xa84fa2c2, 0xa7bda34c, 0xa72ca3d7,
+        0xa69ca463, 0xa60ca4f0, 0xa57ea57e, 0xa4f0a60c, 0xa463a69c, 0xa3d7a72c,
+        0xa34ca7bd, 0xa2c2a84f, 0xa238a8e2, 0xa1b0a976, 0xa129aa0a, 0xa0a2aaa0,
+        0xa01cab36, 0x9f98abcd, 0x9f14ac65, 0x9e91acfd, 0x9e0fad97, 0x9d8eae31,
+        0x9d0eaecc, 0x9c8faf68, 0x9c11b005, 0x9b94b0a2, 0x9b17b140, 0x9a9cb1df,
+        0x9a22b27f, 0x99a9b31f, 0x9930b3c0, 0x98b9b462, 0x9843b505, 0x97ceb5a8,
+        0x9759b64c, 0x96e6b6f1, 0x9674b796, 0x9603b83c, 0x9592b8e3, 0x9523b98b,
+        0x94b5ba33, 0x9448badc, 0x93dcbb85, 0x9371bc2f, 0x9307bcda, 0x929ebd86,
+        0x9236be32, 0x91cfbedf, 0x9169bf8c, 0x9105c03a, 0x90a1c0e9, 0x903ec198,
+        0x8fddc248, 0x8f7dc2f8, 0x8f1dc3a9, 0x8ebfc45b, 0x8e62c50d, 0x8e06c5c0,
+        0x8dabc673, 0x8d51c727, 0x8cf8c7db, 0x8ca1c890, 0x8c4ac946, 0x8bf5c9fc,
+        0x8ba0cab2, 0x8b4dcb69, 0x8afbcc21, 0x8aaaccd9, 0x8a5acd92, 0x8a0cce4b,
+        0x89becf04, 0x8972cfbe, 0x8927d079, 0x88ddd134, 0x8894d1ef, 0x884cd2ab,
+        0x8805d367, 0x87c0d424, 0x877bd4e1, 0x8738d59e, 0x86f6d65c, 0x86b6d71b,
+        0x8676d7d9, 0x8637d898, 0x85fad958, 0x85beda18, 0x8583dad8, 0x8549db99,
+        0x8511dc59, 0x84d9dd1b, 0x84a3dddc, 0x846ede9e, 0x843adf61, 0x8407e023,
+        0x83d6e0e6, 0x83a6e1a9, 0x8377e26d, 0x8349e330, 0x831ce3f4, 0x82f1e4b9,
+        0x82c6e57d, 0x829de642, 0x8276e707, 0x824fe7cd, 0x822ae892, 0x8205e958,
+        0x81e2ea1e, 0x81c1eae4, 0x81a0ebab, 0x8181ec71, 0x8163ed38, 0x8146edff,
+        0x812aeec6, 0x8110ef8d, 0x80f6f055, 0x80def11c, 0x80c8f1e4, 0x80b2f2ac,
+        0x809ef374, 0x808bf43c, 0x8079f505, 0x8068f5cd, 0x8059f695, 0x804bf75e,
+        0x803ef827, 0x8032f8ef, 0x8027f9b8, 0x801efa81, 0x8016fb4a, 0x800ffc13,
+        0x800afcdc, 0x8006fda5, 0x8002fe6e, 0x8001ff37,
+    };
+/* Returns the multiplication of \conj{a} and {b}. */
+    static inline int32_t mult (int32_t a, int32_t b)
+    {
+#if __ARM_ARCH__ >= 6
+        int32_t t = b;
+    __asm__("smuad  %0, %0, %1"          : "+r" (t) : "r" (a));
+    __asm__("smusdx %0, %0, %1"          : "+r" (b) : "r" (a));
+    __asm__("pkhtb  %0, %0, %1, ASR #16" : "+r" (t) : "r" (b));
+    return t;
+#else
+        return (((a >> 16) * (b >> 16) + (int16_t) a * (int16_t) b) & ~0xFFFF) |
+            ((((a >> 16) * (int16_t) b - (int16_t) a * (b >> 16)) >> 16) & 0xFFFF);
+#endif
+    }
+    static inline int32_t half (int32_t a)
+    {
+#if __ARM_ARCH__ >= 6
+        __asm__("shadd16 %0, %0, %1" : "+r" (a) : "r" (0));
+    return a;
+#else
+        return ((a >> 1) & ~0x8000) | (a & 0x8000);
+#endif
+    }
+    void fixed_fft (int n, int32_t* v)
+    {
+        int scale = LOG_FFT_SIZE, i, p, r;
+        for (r = 0, i = 1; i < n; ++i)
+        {
+            for (p = n; !(p & r); p >>= 1, r ^= p);
+            if (i < r)
+            {
+                int32_t t = v[i];
+                v[i] = v[r];
+                v[r] = t;
+            }
+        }
+        for (p = 1; p < n; p <<= 1)
+        {
+            --scale;
+            for (i = 0; i < n; i += p << 1)
+            {
+                int32_t x = half (v[i]);
+                int32_t y = half (v[i + p]);
+                v[i] = x + y;
+                v[i + p] = x - y;
+            }
+            for (r = 1; r < p; ++r)
+            {
+                int32_t w = MAX_FFT_SIZE / 4 - (r << scale);
+                i = w >> 31;
+                w = twiddle[(w ^ i) - i] ^ (i << 16);
+                for (i = r; i < n; i += p << 1)
+                {
+                    int32_t x = half (v[i]);
+                    int32_t y = mult (w, v[i + p]);
+                    v[i] = x - y;
+                    v[i + p] = x + y;
+                }
+            }
+        }
+    }
+    void fixed_fft_real (int n, int32_t* v)
+    {
+        int scale = LOG_FFT_SIZE, m = n >> 1, i;
+        fixed_fft (n, v);
+        for (i = 1; i <= n; i <<= 1, --scale);
+        v[0] = mult (~v[0], 0x80008000);
+        v[m] = half (v[m]);
+        for (i = 1; i < n >> 1; ++i)
+        {
+            int32_t x = half (v[i]);
+            int32_t z = half (v[n - i]);
+            int32_t y = z - (x ^ 0xFFFF);
+            x = half (x + (z ^ 0xFFFF));
+            y = mult (y, twiddle[i << scale]);
+            v[i] = x - y;
+            v[n - i] = (x + y) ^ 0xFFFF;
+        }
+    }
+
+    bool doFft (uint8_t* fft, uint8_t* waveform)
+    {
+        int32_t workspace[WAVE_BUFFER_SIZE >> 1];
+        int32_t nonzero = 0;
+        for (uint32_t i = 0; i < WAVE_BUFFER_SIZE; i += 2)
+        {
+            workspace[i >> 1] =
+                ((waveform[i] ^ 0x80) << 24) | ((waveform[i + 1] ^ 0x80) << 8);
+            nonzero |= workspace[i >> 1];
+        }
+        if (nonzero)
+        {
+            fixed_fft_real (WAVE_BUFFER_SIZE >> 1, workspace);
+        }
+        for (uint32_t i = 0; i < WAVE_BUFFER_SIZE; i += 2)
+        {
+            short tmp = workspace[i >> 1] >> 21;
+            while (tmp > 127 || tmp < -128) tmp >>= 1;
+            fft[i] = tmp;
+            tmp = workspace[i >> 1];
+            tmp >>= 5;
+            while (tmp > 127 || tmp < -128) tmp >>= 1;
+            fft[i + 1] = tmp;
+        }
+        return true;
+    }
+}

src/External/Android/fft.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <iostream>
+
+#define WAVE_BUFFER_SIZE 1024
+
+namespace External::Android
+{
+    bool doFft (uint8_t* fft, uint8_t* waveform);
+}

src/WallpaperEngine/Application/CWallpaperApplication.cpp

@@ -274,10 +274,12 @@ namespace WallpaperEngine::Application
         WallpaperEngine::Render::Drivers::Output::COutput* output;
         // fullscreen detector is common for the different render modes
         WallpaperEngine::Render::Drivers::Detectors::CX11FullScreenDetector fullscreenDetector (videoDriver);
+        // stereo mix recorder for audio processing
+        WallpaperEngine::Audio::Drivers::Recorders::CPulseAudioPlaybackRecorder audioRecorder;
         // audio playing detector
         WallpaperEngine::Audio::Drivers::Detectors::CPulseAudioPlayingDetector audioDetector (this->m_context, fullscreenDetector);
         // initialize sdl audio driver
-        WallpaperEngine::Audio::Drivers::CSDLAudioDriver audioDriver (this->m_context, audioDetector);
+        WallpaperEngine::Audio::Drivers::CSDLAudioDriver audioDriver (this->m_context, audioDetector, audioRecorder);
         // initialize audio context
         WallpaperEngine::Audio::CAudioContext audioContext (audioDriver);
 
@@ -314,6 +316,8 @@ namespace WallpaperEngine::Application
 
         while (!videoDriver.closeRequested () && this->m_context.state.general.keepRunning)
         {
+            // update audio recorder
+            audioDriver.update ();
             // update input information
             inputContext.update ();
             // keep track of the previous frame's time

src/WallpaperEngine/Audio/CAudioContext.cpp

@@ -32,4 +32,9 @@ namespace WallpaperEngine::Audio
     {
         return this->m_driver.getApplicationContext ();
     }
+
+    Drivers::Recorders::CPlaybackRecorder& CAudioContext::getRecorder ()
+    {
+        return this->m_driver.getRecorder ();
+    }
 }

src/WallpaperEngine/Audio/CAudioContext.h

@@ -3,6 +3,7 @@
 #include <libavutil/samplefmt.h>
 #include <vector>
 
+#include "WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.h"
 #include "WallpaperEngine/Application/CApplicationContext.h"
 
 namespace WallpaperEngine
@@ -17,6 +18,11 @@ namespace WallpaperEngine
         namespace Drivers
         {
             class CAudioDriver;
+
+            namespace Recorders
+            {
+                class CPulseAudioPlaybackRecorder;
+            }
         }
 
         class CAudioStream;
@@ -51,6 +57,10 @@ namespace WallpaperEngine
              * @return The application context under which the audio driver is initialized
              */
             Application::CApplicationContext& getApplicationContext ();
+            /**
+             * @return The audio recorder to use to capture stereo mix data
+             */
+            [[nodiscard]] Drivers::Recorders::CPlaybackRecorder& getRecorder ();
 
         private:
             /** The audio driver in use */

src/WallpaperEngine/Audio/Drivers/CAudioDriver.cpp

@@ -2,12 +2,19 @@
 
 namespace WallpaperEngine::Audio::Drivers
 {
-    CAudioDriver::CAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector) :
+    CAudioDriver::CAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector, Recorders::CPlaybackRecorder& recorder) :
         m_applicationContext (applicationContext),
-        m_detector (detector)
+        m_detector (detector),
+        m_recorder (recorder)
     {
     }
 
+    void CAudioDriver::update ()
+    {
+        this->m_recorder.update ();
+        this->m_detector.update ();
+    }
+
     Application::CApplicationContext& CAudioDriver::getApplicationContext ()
     {
         return this->m_applicationContext;
@@ -16,4 +23,9 @@ namespace WallpaperEngine::Audio::Drivers
     {
         return this->m_detector;
     }
+
+    Recorders::CPlaybackRecorder& CAudioDriver::getRecorder ()
+    {
+        return this->m_recorder;
+    }
 }

src/WallpaperEngine/Audio/Drivers/CAudioDriver.h

@@ -3,6 +3,7 @@
 #include <vector>
 
 #include "WallpaperEngine/Audio/Drivers/Detectors/CAudioPlayingDetector.h"
+#include "WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.h"
 #include "WallpaperEngine/Application/CApplicationContext.h"
 #include "WallpaperEngine/Audio/CAudioStream.h"
 
@@ -17,20 +18,25 @@ namespace WallpaperEngine
     {
         class CAudioStream;
 
-        namespace Detectors
-        {
-            class CAudioPlayingDetector;
-        }
-
         namespace Drivers
         {
+            namespace Detectors
+            {
+                class CAudioPlayingDetector;
+            }
+
+            namespace Recorders
+            {
+                class CPulseAudioPlaybackRecorder;
+            }
+
             /**
              * Base class for audio driver implementations
              */
             class CAudioDriver
             {
             public:
-                explicit CAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector);
+                explicit CAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector, Recorders::CPlaybackRecorder& recorder);
 
                 /**
                  * Registers the given stream in the driver for playing
@@ -39,6 +45,11 @@ namespace WallpaperEngine
                  */
                 virtual void addStream (CAudioStream* stream) = 0;
 
+                /**
+                 * Updates status of the different audio settings
+                 */
+                virtual void update ();
+
                 /**
                  * TODO: MAYBE THIS SHOULD BE OUR OWN DEFINITIONS INSTEAD OF LIBRARY SPECIFIC ONES?
                  *
@@ -61,10 +72,15 @@ namespace WallpaperEngine
                  * @return The audio playing detector to use to stop playing sound when something else starts playing
                  */
                 [[nodiscard]] Detectors::CAudioPlayingDetector& getAudioDetector ();
+                /**
+                 * @return The audio recorder to use to capture stereo mix data
+                 */
+                [[nodiscard]] Recorders::CPlaybackRecorder& getRecorder ();
 
             private:
                 Application::CApplicationContext& m_applicationContext;
                 Detectors::CAudioPlayingDetector& m_detector;
+                Recorders::CPlaybackRecorder& m_recorder;
             };
         }
     }

src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.cpp

@@ -74,8 +74,8 @@ void audio_callback (void* userdata, uint8_t* streamData, int length)
     }
 }
 
-CSDLAudioDriver::CSDLAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector) :
-    CAudioDriver (applicationContext, detector),
+CSDLAudioDriver::CSDLAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector, Recorders::CPlaybackRecorder& recorder) :
+    CAudioDriver (applicationContext, detector, recorder),
     m_initialized (false),
     m_audioSpec ()
 {

src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h

@@ -29,7 +29,7 @@ namespace WallpaperEngine::Audio::Drivers
     class CSDLAudioDriver : public CAudioDriver
     {
     public:
-        CSDLAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector);
+        CSDLAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector, Recorders::CPlaybackRecorder& recorder);
         ~CSDLAudioDriver ();
 
         /** @inheritdoc */

src/WallpaperEngine/Audio/Drivers/Detectors/CAudioPlayingDetector.cpp

@@ -6,10 +6,16 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
         Application::CApplicationContext& appContext,
         Render::Drivers::Detectors::CFullScreenDetector& fullscreenDetector) :
         m_applicationContext (appContext),
-        m_fullscreenDetector (fullscreenDetector)
+        m_fullscreenDetector (fullscreenDetector),
+        m_isPlaying (false)
     {
     }
 
+    bool CAudioPlayingDetector::anythingPlaying () const
+    {
+        return this->m_isPlaying;
+    }
+
     Application::CApplicationContext& CAudioPlayingDetector::getApplicationContext ()
     {
         return this->m_applicationContext;
@@ -19,4 +25,9 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
     {
         return this->m_fullscreenDetector;
     }
+
+    void CAudioPlayingDetector::setIsPlaying (bool newState)
+    {
+        this->m_isPlaying = newState;
+    }
 }

src/WallpaperEngine/Audio/Drivers/Detectors/CAudioPlayingDetector.h

@@ -17,6 +17,9 @@ namespace WallpaperEngine
 
     namespace Audio::Drivers::Detectors
     {
+        /**
+         * Base class for any implementation of audio playing detection
+         */
         class CAudioPlayingDetector
         {
         public:
@@ -25,7 +28,19 @@ namespace WallpaperEngine
             /**
              * @return If any kind of sound is currently playing on the default audio device
              */
-            virtual bool anythingPlaying () = 0;
+            [[nodiscard]] bool anythingPlaying () const;
+
+            /**
+             * Updates the playing status to the specified value
+             *
+             * @param newState
+             */
+            void setIsPlaying (bool newState);
+
+            /**
+             * Checks if any audio is playing and updates state accordingly
+             */
+            virtual void update () = 0;
             /**
              * @return The application context using this detector
              */
@@ -36,6 +51,8 @@ namespace WallpaperEngine
             [[nodiscard]] Render::Drivers::Detectors::CFullScreenDetector& getFullscreenDetector ();
 
         private:
+            bool m_isPlaying;
+
             Application::CApplicationContext& m_applicationContext;
             Render::Drivers::Detectors::CFullScreenDetector& m_fullscreenDetector;
         };

src/WallpaperEngine/Audio/Drivers/Detectors/CPulseAudioPlayingDetector.cpp

@@ -36,8 +36,7 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
         CAudioPlayingDetector (appContext, fullscreenDetector),
         m_mainloop (nullptr),
         m_mainloopApi (nullptr),
-        m_context (nullptr),
-        m_isPlaying (false)
+        m_context (nullptr)
     {
         this->m_mainloop = pa_mainloop_new ();
         this->m_mainloopApi = pa_mainloop_get_api (this->m_mainloop);
@@ -62,17 +61,13 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
             pa_mainloop_free (this->m_mainloop);
     }
 
-    void CPulseAudioPlayingDetector::setIsPlaying (bool newState)
-    {
-        this->m_isPlaying = newState;
-    }
 
-    bool CPulseAudioPlayingDetector::anythingPlaying ()
+    void CPulseAudioPlayingDetector::update ()
     {
         if (!this->getApplicationContext ().settings.audio.automute)
-            return false;
+            return this->setIsPlaying (false);
         if (this->getFullscreenDetector ().anythingFullscreen ())
-            return true;
+            return this->setIsPlaying (true);
 
         // reset playing state
         this->setIsPlaying (false);
@@ -85,7 +80,5 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
             pa_mainloop_iterate (this->m_mainloop, 1, nullptr);
 
         pa_operation_unref (op);
-
-        return this->m_isPlaying;
     }
 }

src/WallpaperEngine/Audio/Drivers/Detectors/CPulseAudioPlayingDetector.h

@@ -13,15 +13,11 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
         explicit CPulseAudioPlayingDetector (Application::CApplicationContext& appContext, Render::Drivers::Detectors::CFullScreenDetector&);
         ~CPulseAudioPlayingDetector ();
 
-        [[nodiscard]] bool anythingPlaying () override;
-        void setIsPlaying (bool newState);
+        void update () override;
 
     private:
-        bool m_isPlaying;
-
         pa_mainloop* m_mainloop;
         pa_mainloop_api* m_mainloopApi;
         pa_context* m_context;
-        pa_operation* m_operation;
     };
 }

src/WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.cpp

@@ -0,0 +1 @@
+#include "CPlaybackRecorder.h"

src/WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.h

@@ -0,0 +1,14 @@
+#pragma once
+
+namespace WallpaperEngine::Audio::Drivers::Recorders
+{
+    class CPlaybackRecorder
+    {
+    public:
+        virtual void update () = 0;
+
+        float audio16[16] = {0};
+        float audio32[32] = {0};
+        float audio64[64] = {0};
+    };
+}

src/WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.cpp

@@ -0,0 +1,216 @@
+#include <cstring>
+#include <cmath>
+#include <glm/common.hpp>
+#include "CPulseAudioPlaybackRecorder.h"
+#include "WallpaperEngine/Logging/CLog.h"
+#include "External/Android/fft.h"
+
+namespace WallpaperEngine::Audio::Drivers::Recorders
+{
+    float movetowards(float current, float target, float maxDelta)
+    {
+        if (abs(target - current) <= maxDelta)
+            return target;
+
+        return current + glm::sign(target - current) * maxDelta;
+    }
+
+    void pa_stream_notify_cb(pa_stream *stream, void* /*userdata*/)
+    {
+        const pa_stream_state state = pa_stream_get_state(stream);
+        switch (state) {
+            case PA_STREAM_FAILED:
+                sLog.error ("Cannot open stream for capture. Audio processing is disabled");
+                break;
+            case PA_STREAM_READY:
+                sLog.debug ("Capture stream ready");
+                break;
+        }
+    }
+
+    void pa_stream_read_cb(pa_stream *stream, const size_t /*nbytes*/, void* userdata)
+    {
+        auto* recorder = reinterpret_cast<CPulseAudioPlaybackRecorder*>(userdata);
+
+        // Careful when to pa_stream_peek() and pa_stream_drop()!
+        // c.f. https://www.freedesktop.org/software/pulseaudio/doxygen/stream_8h.html#ac2838c449cde56e169224d7fe3d00824
+        uint8_t *data = nullptr;
+        size_t currentSize;
+        if (pa_stream_peek(stream, (const void**)&data, &currentSize) != 0) {
+            sLog.error ("Failed to peek at stream data...");
+            return;
+        }
+
+        if (data == nullptr && currentSize == 0) {
+            // No data in the buffer, ignore.
+            return;
+        } else if (data == nullptr && currentSize > 0) {
+            // Hole in the buffer. We must drop it.
+            if (pa_stream_drop(stream) != 0) {
+                sLog.error ("Failed to drop a hole while capturing!");
+                return;
+            }
+        } else if (currentSize > 0 && data) {
+            size_t dataToCopy = std::min (currentSize, WAVE_BUFFER_SIZE - recorder->currentWritePointer);
+
+            memcpy (&recorder->audio_buffer_tmp [recorder->currentWritePointer], data, dataToCopy * sizeof (uint8_t));
+
+            recorder->currentWritePointer += dataToCopy;
+
+            if (recorder->currentWritePointer == WAVE_BUFFER_SIZE) {
+                // copy to the final buffer
+                memcpy (recorder->audio_buffer, recorder->audio_buffer_tmp, WAVE_BUFFER_SIZE * sizeof (uint8_t));
+                // reset the write pointer
+                recorder->currentWritePointer = 0;
+                recorder->fullframeReady = true;
+            }
+
+            // any data read left?
+            if (dataToCopy < currentSize) {
+                while ((currentSize - dataToCopy) > WAVE_BUFFER_SIZE)
+                    dataToCopy += WAVE_BUFFER_SIZE; // there's more than one full frame available, skip it entirely
+
+                // data pending, keep it in the buffer
+                memcpy (recorder->audio_buffer_tmp, data + dataToCopy, (currentSize - dataToCopy) * sizeof (uint8_t));
+
+                recorder->currentWritePointer = currentSize - dataToCopy;
+            }
+        }
+
+        if (pa_stream_drop(stream) != 0) {
+            sLog.error ("Failed to drop data after peeking");
+        }
+    }
+
+    void pa_server_info_cb(pa_context *ctx, const pa_server_info *info, void* userdata)
+    {
+        auto* recorder = reinterpret_cast<CPulseAudioPlaybackRecorder*>(userdata);
+
+        pa_sample_spec spec;
+        spec.format = PA_SAMPLE_U8;
+        spec.rate = 44100;
+        spec.channels = 1;
+
+        if (recorder->getCaptureStream ())
+            pa_stream_unref (recorder->getCaptureStream ());
+
+        pa_stream* captureStream = pa_stream_new(ctx, "output monitor", &spec, nullptr);
+
+        pa_stream_set_state_callback(captureStream, &pa_stream_notify_cb, userdata);
+        pa_stream_set_read_callback(captureStream, &pa_stream_read_cb, userdata);
+
+        std::string monitor_name(info->default_sink_name);
+        monitor_name += ".monitor";
+        if (pa_stream_connect_record(captureStream, monitor_name.c_str(), nullptr, PA_STREAM_NOFLAGS) != 0) {
+            sLog.error ("Failed to connect to input for recording");
+            return;
+        }
+
+        recorder->setCaptureStream (captureStream);
+    }
+
+    void pa_context_subscribe_cb (pa_context *ctx, pa_subscription_event_type_t t, uint32_t idx, void *userdata)
+    {
+        // sink changes mean re-take the stream
+        pa_context_get_server_info(ctx, &pa_server_info_cb, userdata);
+    }
+
+    void pa_context_notify_cb(pa_context *ctx, void* userdata)
+    {
+        const pa_context_state state = pa_context_get_state(ctx);
+        switch (state) {
+            case PA_CONTEXT_READY:
+            {
+                //set callback
+                pa_context_set_subscribe_callback (ctx, pa_context_subscribe_cb, userdata);
+                //set events mask and enable event callback.
+                pa_operation* o = pa_context_subscribe (
+                    ctx, static_cast<pa_subscription_mask_t>(PA_SUBSCRIPTION_MASK_SINK | PA_SUBSCRIPTION_MASK_SOURCE),
+                    NULL, NULL
+                );
+
+                if (o)
+                    pa_operation_unref (o);
+
+                // context being ready means to fetch the sink too
+                pa_context_get_server_info(ctx, &pa_server_info_cb, userdata);
+
+                break;
+            }
+            case PA_CONTEXT_FAILED:
+                sLog.error ("PulseAudio context initialization failed. Audio processing is disabled");
+                break;
+        }
+    }
+
+    CPulseAudioPlaybackRecorder::CPulseAudioPlaybackRecorder ()
+    {
+        this->m_mainloop = pa_mainloop_new ();
+        this->m_mainloopApi = pa_mainloop_get_api (this->m_mainloop);
+        this->m_context = pa_context_new (this->m_mainloopApi, "wallpaperengine-audioprocessing");
+
+        pa_context_set_state_callback (this->m_context, &pa_context_notify_cb, this);
+
+        if (pa_context_connect(this->m_context, nullptr, PA_CONTEXT_NOFLAGS, nullptr) < 0) {
+            sLog.error ("PulseAudio connection failed! Audio processing is disabled");
+            return;
+        }
+
+        // wait until the context is ready
+        while (pa_context_get_state (this->m_context) != PA_CONTEXT_READY)
+            pa_mainloop_iterate (this->m_mainloop, 1, nullptr);
+    }
+
+    CPulseAudioPlaybackRecorder::~CPulseAudioPlaybackRecorder ()
+    {
+        pa_context_disconnect(this->m_context);
+        pa_mainloop_free(this->m_mainloop);
+    }
+
+    pa_stream* CPulseAudioPlaybackRecorder::getCaptureStream ()
+    {
+        return this->m_captureStream;
+    }
+
+    void CPulseAudioPlaybackRecorder::setCaptureStream (pa_stream* stream)
+    {
+        this->m_captureStream = stream;
+    }
+
+    void CPulseAudioPlaybackRecorder::update ()
+    {
+        pa_mainloop_iterate (this->m_mainloop, 1, nullptr);
+
+        // interpolate current values to the destination
+        for (int i = 0; i < 64; i ++) {
+            this->audio64 [i] = movetowards (this->audio64[i], fft_destination64[i], 0.1f);
+            if (i >= 32)
+                continue;
+            this->audio32 [i] = movetowards (this->audio32[i], fft_destination32[i], 0.1f);
+            if (i >= 16)
+                continue;
+            this->audio16 [i] = movetowards (this->audio16[i], fft_destination16[i], 0.1f);
+        }
+
+        if (this->fullframeReady == false)
+            return;
+
+        this->fullframeReady = false;
+
+        External::Android::doFft (audio_fft, audio_buffer);
+
+        for (int i = 0; i < 64; i ++) {
+            int paramInt = (i + 2) * 2;
+            float f1 = audio_fft[paramInt];
+            float f2 = audio_fft[paramInt + 1];
+            f2 = f1 * f1 + f2 * f2;
+            f1 = 0.0F;
+            if (f2 > 0.0F)
+                f1 = 0.35F * (float)log10(f2);
+
+            this->fft_destination64[i] = fmin(1.0F, f1 * (float)(2.0f - pow(M_E, (1.0F - i / 63.0F) * 1.0f - 0.5f)));
+            this->fft_destination32[i >> 1] = fmin(1.0F, f1 * (float)(2.0f - pow(M_E, (1.0F - i / 31.0F) * 1.0f - 0.5f)));
+            this->fft_destination16[i >> 2] = fmin(1.0F, f1 * (float)(2.0f - pow(M_E, (1.0F - i / 15.0F) * 1.0f - 0.5f)));
+        }
+    }
+}

src/WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <pulse/pulseaudio.h>
+#include "External/Android/fft.h"
+#include "CPlaybackRecorder.h"
+
+namespace WallpaperEngine::Audio::Drivers::Recorders
+{
+    class CPlaybackRecorder;
+
+    class CPulseAudioPlaybackRecorder : public CPlaybackRecorder
+    {
+    public:
+        CPulseAudioPlaybackRecorder ();
+        ~CPulseAudioPlaybackRecorder ();
+
+        void update () override;
+
+        /**
+         * @return The current stream we're capturing from
+         */
+        [[nodiscard]] pa_stream* getCaptureStream ();
+
+        /**
+         * @param stream The new stream to be capturing off from
+         */
+        void setCaptureStream (pa_stream* stream);
+
+        uint8_t audio_buffer [WAVE_BUFFER_SIZE] = {0x80};
+        uint8_t audio_buffer_tmp [WAVE_BUFFER_SIZE] = {0x80};
+        uint8_t audio_fft [WAVE_BUFFER_SIZE] = {0};
+        size_t currentWritePointer = 0;
+        bool fullframeReady = false;
+
+    private:
+        pa_mainloop* m_mainloop;
+        pa_mainloop_api* m_mainloopApi;
+        pa_context* m_context;
+        pa_stream* m_captureStream;
+
+        float fft_destination64[64];
+        float fft_destination32[32];
+        float fft_destination16[16];
+    };
+}

src/WallpaperEngine/Render/Objects/Effects/CPass.cpp

@@ -180,14 +180,15 @@ void CPass::render ()
         switch (entry->type)
         {
             case Double:
-                glUniform1d (entry->id, *reinterpret_cast <const double*> (entry->value));
+                glUniform1dv (entry->id, entry->count, reinterpret_cast <const double*> (entry->value));
                 break;
             case Float:
-                glUniform1f (entry->id, *reinterpret_cast <const float*> (entry->value));
+                glUniform1fv (entry->id, entry->count, reinterpret_cast <const float*> (entry->value));
                 break;
             case Integer:
-                glUniform1i (entry->id, *reinterpret_cast <const int*> (entry->value));
+                glUniform1iv (entry->id, entry->count, reinterpret_cast <const int*> (entry->value));
                 break;
+            // TODO: THESE MIGHT NEED SPECIAL TREATMENT? IDK ONLY SUPPORT 1 FOR NOW
             case Vector4:
                 glUniform4fv (entry->id, 1, glm::value_ptr (*reinterpret_cast <const glm::vec4*> (entry->value)));
                 break;
@@ -595,6 +596,12 @@ void CPass::setupUniforms ()
     this->addUniform ("g_EffectTextureProjectionMatrixInverse", glm::mat4(1.0));
     this->addUniform ("g_TexelSize", glm::vec2 (1.0 / projection->getWidth (), 1.0 / projection->getHeight ()));
     this->addUniform ("g_TexelSizeHalf", glm::vec2 (0.5 / projection->getWidth (), 0.5 / projection->getHeight ()));
+    this->addUniform ("g_AudioSpectrum16Left", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio16, 16);
+    this->addUniform ("g_AudioSpectrum16Right", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio16, 16);
+    this->addUniform ("g_AudioSpectrum32Left", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio32, 32);
+    this->addUniform ("g_AudioSpectrum32Right", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio32, 32);
+    this->addUniform ("g_AudioSpectrum64Left", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio64, 64);
+    this->addUniform ("g_AudioSpectrum64Right", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio64, 64);
 }
 
 void CPass::addAttribute (const std::string& name, GLint type, GLint elements, const GLuint* value)
@@ -623,12 +630,12 @@ void CPass::addUniform (const std::string& name, UniformType type, T value)
 
     // uniform found, add it to the list
     this->m_uniforms.insert (
-        std::make_pair (name, new UniformEntry (id, name, type, newValue))
+        std::make_pair (name, new UniformEntry (id, name, type, newValue, 1))
     );
 }
 
 template <typename T>
-void CPass::addUniform (const std::string& name, UniformType type, T* value)
+void CPass::addUniform (const std::string& name, UniformType type, T* value, int count)
 {
     // this version is used to reference to system variables so things like g_Time works fine
     GLint id = glGetUniformLocation (this->m_programID, name.c_str ());
@@ -639,7 +646,7 @@ void CPass::addUniform (const std::string& name, UniformType type, T* value)
 
     // uniform found, add it to the list
     this->m_uniforms.insert (
-        std::make_pair (name, new UniformEntry (id, name, type, value))
+        std::make_pair (name, new UniformEntry (id, name, type, value, count))
     );
 }
 
@@ -804,9 +811,9 @@ void CPass::addUniform (const std::string& name, int value)
     this->addUniform (name, UniformType::Integer, value);
 }
 
-void CPass::addUniform (const std::string& name, const int* value)
+void CPass::addUniform (const std::string& name, const int* value, int count)
 {
-    this->addUniform (name, UniformType::Integer, value);
+    this->addUniform (name, UniformType::Integer, value, count);
 }
 
 void CPass::addUniform (const std::string& name, const int** value)
@@ -819,9 +826,9 @@ void CPass::addUniform (const std::string& name, double value)
     this->addUniform (name, UniformType::Double, value);
 }
 
-void CPass::addUniform (const std::string& name, const double* value)
+void CPass::addUniform (const std::string& name, const double* value, int count)
 {
-    this->addUniform (name, UniformType::Double, value);
+    this->addUniform (name, UniformType::Double, value, count);
 }
 
 void CPass::addUniform (const std::string& name, const double** value)
@@ -834,9 +841,9 @@ void CPass::addUniform (const std::string& name, float value)
     this->addUniform (name, UniformType::Float, value);
 }
 
-void CPass::addUniform (const std::string& name, const float* value)
+void CPass::addUniform (const std::string& name, const float* value, int count)
 {
-    this->addUniform (name, UniformType::Float, value);
+    this->addUniform (name, UniformType::Float, value, count);
 }
 
 void CPass::addUniform (const std::string& name, const float** value)
@@ -852,12 +859,12 @@ void CPass::addUniform (const std::string& name, glm::vec2 value)
 
 void CPass::addUniform (const std::string& name, const glm::vec2* value)
 {
-    this->addUniform (name, UniformType::Vector2, value);
+    this->addUniform (name, UniformType::Vector2, value, 1);
 }
 
 void CPass::addUniform (const std::string& name, const glm::vec2** value)
 {
-    this->addUniform (name, UniformType::Vector2, value);
+    this->addUniform (name, UniformType::Vector2, value, 1);
 }
 
 
@@ -868,7 +875,7 @@ void CPass::addUniform (const std::string& name, glm::vec3 value)
 
 void CPass::addUniform (const std::string& name, const glm::vec3* value)
 {
-    this->addUniform (name, UniformType::Vector3, value);
+    this->addUniform (name, UniformType::Vector3, value, 1);
 }
 
 void CPass::addUniform (const std::string& name, const glm::vec3** value)
@@ -884,7 +891,7 @@ void CPass::addUniform (const std::string& name, glm::vec4 value)
 
 void CPass::addUniform (const std::string& name, const glm::vec4* value)
 {
-    this->addUniform (name, UniformType::Vector4, value);
+    this->addUniform (name, UniformType::Vector4, value, 1);
 }
 
 void CPass::addUniform (const std::string& name, const glm::vec4** value)
@@ -899,7 +906,7 @@ void CPass::addUniform (const std::string& name, glm::mat4 value)
 
 void CPass::addUniform (const std::string& name, const glm::mat4* value)
 {
-    this->addUniform (name, UniformType::Matrix4, value);
+    this->addUniform (name, UniformType::Matrix4, value, 1);
 }
 
 void CPass::addUniform (const std::string& name, const glm::mat4** value)

src/WallpaperEngine/Render/Objects/Effects/CPass.h

@@ -52,13 +52,14 @@ namespace WallpaperEngine::Render::Objects::Effects
         class UniformEntry
         {
         public:
-            UniformEntry (GLint id, std::string name, UniformType type, const void* value) :
-                id (id), name (std::move (name)), type (type), value (value) { }
+            UniformEntry (GLint id, std::string name, UniformType type, const void* value, int count) :
+                id (id), name (std::move (name)), type (type), value (value), count (count) { }
 
             GLint id;
             std::string name;
             UniformType type;
             const void* value;
+            int count;
         };
 
         class ReferenceUniformEntry
@@ -102,9 +103,9 @@ namespace WallpaperEngine::Render::Objects::Effects
         void addUniform (const std::string& name, glm::vec3 value);
         void addUniform (const std::string& name, glm::vec4 value);
         void addUniform (const std::string& name, glm::mat4 value);
-        void addUniform (const std::string& name, const int* value);
-        void addUniform (const std::string& name, const double* value);
-        void addUniform (const std::string& name, const float* value);
+        void addUniform (const std::string& name, const int* value, int count = 1);
+        void addUniform (const std::string& name, const double* value, int count = 1);
+        void addUniform (const std::string& name, const float* value, int count = 1);
         void addUniform (const std::string& name, const glm::vec2* value);
         void addUniform (const std::string& name, const glm::vec3* value);
         void addUniform (const std::string& name, const glm::vec4* value);
@@ -117,7 +118,7 @@ namespace WallpaperEngine::Render::Objects::Effects
         void addUniform (const std::string& name, const glm::vec4** value);
         void addUniform (const std::string& name, const glm::mat4** value);
         template <typename T> void addUniform (const std::string& name, UniformType type, T value);
-        template <typename T> void addUniform (const std::string& name, UniformType type, T* value);
+        template <typename T> void addUniform (const std::string& name, UniformType type, T* value, int count = 1);
         template <typename T> void addUniform (const std::string& name, UniformType type, T** value);
 
         const ITexture* resolveTexture (const ITexture* expected, int index, const ITexture* previous = nullptr);

src/WallpaperEngine/Render/Shaders/Compiler.cpp

@@ -701,38 +701,30 @@ namespace WallpaperEngine::Render::Shaders
         // so only define the ones that are not already defined
         if (entry == this->m_combos->end ())
         {
-            if (type != data.end () && (*type) == "audioprocessingoptions")
+            if (type != data.end ())
+                sLog.error ("Resorting to default value as type ", *type, " is unknown");
+
+            // if no combo is defined just load the default settings
+            if (defvalue == data.end ())
+            {
+                // TODO: PROPERLY SUPPORT EMPTY COMBOS
+                this->m_combos->insert (std::make_pair <std::string, int> (*combo, (int) defaultValue));
+            }
+            else if ((*defvalue).is_number_float ())
+            {
+                sLog.exception ("float combos are not supported in shader ", this->m_file, ". ", *combo);
+            }
+            else if ((*defvalue).is_number_integer ())
             {
-                sLog.out ("Found audioprocessing value, nothing working yet");
-                this->m_combos->insert (std::make_pair <std::string, int> (*combo, 1));
+                this->m_combos->insert (std::make_pair <std::string, int> (*combo, (*defvalue).get <int> ()));
+            }
+            else if ((*defvalue).is_string ())
+            {
+                sLog.exception ("string combos are not supported in shader ", this->m_file, ". ", *combo);
             }
             else
             {
-                if (type != data.end ())
-                    sLog.error ("Resorting to default value as type ", *type, " is unknown");
-
-                // if no combo is defined just load the default settings
-                if (defvalue == data.end ())
-                {
-                    // TODO: PROPERLY SUPPORT EMPTY COMBOS
-                    this->m_combos->insert (std::make_pair <std::string, int> (*combo, (int) defaultValue));
-                }
-                else if ((*defvalue).is_number_float ())
-                {
-                    sLog.exception ("float combos are not supported in shader ", this->m_file, ". ", *combo);
-                }
-                else if ((*defvalue).is_number_integer ())
-                {
-                    this->m_combos->insert (std::make_pair <std::string, int> (*combo, (*defvalue).get <int> ()));
-                }
-                else if ((*defvalue).is_string ())
-                {
-                    sLog.exception ("string combos are not supported in shader ", this->m_file, ". ", *combo);
-                }
-                else
-                {
-                    sLog.exception ("cannot parse combo information ", *combo, ". unknown type for ", defvalue->dump ());
-                }
+                sLog.exception ("cannot parse combo information ", *combo, ". unknown type for ", defvalue->dump ());
             }
         }
     }