fft effects for opus

Author: ArdenButterfield

source/CodecControllers/OpusBendFlags.h

@@ -0,0 +1,49 @@
+//
+// Created by arden on 7/31/26.
+//
+
+#ifndef MAIM_OPUSBENDFLAGS_H
+#define MAIM_OPUSBENDFLAGS_H
+
+#include <array>
+
+// Mirrors the role of BendFlagsAndData (lame) / blade_bend_flags (blade): a single struct
+// of "bend" parameters that gets threaded through the encoder to warp the encoding process
+// away from the codec's intended behaviour. Opus has no exposed MDCT stage of its own (libopus
+// is used unmodified), so these bends are applied to our own FFT of the PCM signal that Opus
+// consumes/produces, standing in for the frequency-domain hooks lame/blade have natively.
+struct OpusBendFlags
+{
+    // Frequency reassignment: mdctBandReassignment[destBand] = sourceBand, same 32-band
+    // remapping idea as BendFlagsAndData::mdct_band_reassignments.
+    static constexpr int numReassignmentBands = 32;
+    std::array<int, numReassignmentBands> mdctBandReassignment = identityReassignment();
+
+    // Spectral effects: horizontal (frequency) and vertical (amplitude) shift, matching
+    // BendFlagsAndData::mdct_post_h_shift / mdct_post_v_shift.
+    int mdctPostHShift = 0;
+    float mdctPostVShift = 0.0f;
+
+    // Spectral effects: blends each frame's spectrum with the previous frame's, matching
+    // BendFlagsAndData::mdct_feedback.
+    float mdctFeedback = 0.0f;
+
+    static std::array<int, numReassignmentBands> identityReassignment()
+    {
+        std::array<int, numReassignmentBands> order{};
+        for (int i = 0; i < numReassignmentBands; ++i) {
+            order[static_cast<size_t>(i)] = i;
+        }
+        return order;
+    }
+
+    bool isIdentity() const
+    {
+        return mdctBandReassignment == identityReassignment()
+            && mdctPostHShift == 0
+            && mdctPostVShift >= 0.0f && mdctPostVShift <= 0.0f
+            && mdctFeedback >= 0.0f && mdctFeedback <= 0.0f;
+    }
+};
+
+#endif //MAIM_OPUSBENDFLAGS_H

source/CodecControllers/OpusController.cpp

@@ -5,6 +5,7 @@
 #include "OpusController.h"
 #include "../parameterIds.h"
 #include "../Mp3ControllerManager.h"
+#include "../SpectrumRescale.h"
 OpusController::OpusController (juce::AudioProcessorValueTreeState& _parameters)
     : bInitialized(false),
       opusEncoder(nullptr),
@@ -23,7 +24,14 @@ OpusController::OpusController (juce::AudioProcessorValueTreeState& _parameters)
     parameters.addParameterListener(PACKET_LOSS_STICK_PARAM_ID, this);
     parameters.addParameterListener(PACKET_LOSS_MODE_PARAM_ID, this);
     parameters.addParameterListener(ERROR_PARAM_ID, this);
+    parameters.addParameterListener(MDCT_PITCH_SHIFT_PARAM_ID, this);
+    parameters.addParameterListener(MDCT_AMPLITUDE_SHIFT_PARAM_ID, this);
+    parameters.addParameterListener(MDCT_FEEDBACK_PARAM_ID, this);
+    for (const auto& bandOrderParamId : BAND_ORDER_PARAM_IDS) {
+        parameters.addParameterListener(bandOrderParamId, this);
+    }
     parametersNeedUpdating = true;
+    startTimerHz(30);
 }
 
 OpusController::~OpusController()
@@ -38,6 +46,12 @@ OpusController::~OpusController()
     parameters.removeParameterListener(PACKET_LOSS_STICK_PARAM_ID, this);
     parameters.removeParameterListener(PACKET_LOSS_MODE_PARAM_ID, this);
     parameters.removeParameterListener(ERROR_PARAM_ID, this);
+    parameters.removeParameterListener(MDCT_PITCH_SHIFT_PARAM_ID, this);
+    parameters.removeParameterListener(MDCT_AMPLITUDE_SHIFT_PARAM_ID, this);
+    parameters.removeParameterListener(MDCT_FEEDBACK_PARAM_ID, this);
+    for (const auto& bandOrderParamId : BAND_ORDER_PARAM_IDS) {
+        parameters.removeParameterListener(bandOrderParamId, this);
+    }
 }
 
 bool OpusController::init (int sampleRate, int, int)
@@ -64,6 +78,20 @@ bool OpusController::init (int sampleRate, int, int)
 
     packetLossModel = std::make_unique<PacketLossModel>(sampleRate);
 
+    fftOrder = 1;
+    while ((1 << fftOrder) < samplesPerFrame) {
+        ++fftOrder;
+    }
+    fftSize = 1 << fftOrder;
+    fft = std::make_unique<juce::dsp::FFT>(fftOrder);
+
+    const auto numBins = fftSize / 2 + 1;
+    for (auto channel = 0; channel < 2; ++channel) {
+        fftBuffers[static_cast<size_t>(channel)].assign(static_cast<size_t>(fftSize) * 2, 0.0f);
+        feedbackSpectrum[static_cast<size_t>(channel)].assign(static_cast<size_t>(numBins), std::complex<float>(0.0f, 0.0f));
+    }
+    reassignmentScratch.assign(static_cast<size_t>(numBins), std::complex<float>(0.0f, 0.0f));
+
     return true;
 }
 
@@ -89,6 +117,7 @@ void OpusController::processBlock (juce::AudioBuffer<float>& buffer)
         output[static_cast<size_t>(sampleCounter * 2 + 1)] = 0;
         sampleCounter++;
         if (sampleCounter == samplesPerFrame) {
+            applyFrequencyBends();
             int framesizeDownscaleFactor = std::round(random.nextFloat() * turbo * 3);
             framesizeDownscaleFactor = 1 << std::min(std::max(0, framesizeDownscaleFactor), 2);
             auto subframeSize = samplesPerFrame / framesizeDownscaleFactor;
@@ -164,4 +193,184 @@ void OpusController::updateParameters()
 
     error = *((juce::AudioParameterFloat*)parameters.getParameter(ERROR_PARAM_ID));
     error = std::pow(error, 3);
+
+    setMDCTpostshiftBends(
+        ((juce::AudioParameterInt*) parameters.getParameter(MDCT_PITCH_SHIFT_PARAM_ID))->get(),
+        ((juce::AudioParameterFloat*) parameters.getParameter(MDCT_AMPLITUDE_SHIFT_PARAM_ID))->get()
+    );
+
+    setMDCTfeedback(
+        ((juce::AudioParameterFloat*) parameters.getParameter(MDCT_FEEDBACK_PARAM_ID))->get()
+    );
+
+    int bandReassign[OpusBendFlags::numReassignmentBands];
+    int i;
+    for (i = 0; i < NUM_REASSIGNMENT_BANDS; ++i) {
+        bandReassign[i] = ((juce::AudioParameterInt*) parameters.getParameter(BAND_ORDER_PARAM_IDS[static_cast<size_t>(i)]))->get();
+    }
+    for (; i < OpusBendFlags::numReassignmentBands; ++i) {
+        bandReassign[i] = i;
+    }
+    setMDCTBandReassignmentBends(bandReassign);
+}
+
+void OpusController::applyFrequencyBends()
+{
+    const auto numBins = fftSize / 2 + 1;
+    const bool identity = bendFlags.isIdentity();
+
+    // Accumulated across both channels, for the live "before/after" spectrum display - this
+    // runs whether or not any bend is active, mirroring how lame/blade always snapshot their
+    // MDCT spectrum for the graph regardless of bend settings.
+    std::array<float, mdctDisplaySize> preSnapshot{};
+    std::array<float, mdctDisplaySize> postSnapshot{};
+
+    for (auto channel = 0; channel < 2; ++channel) {
+        auto& fftBuffer = fftBuffers[static_cast<size_t>(channel)];
+        std::fill(fftBuffer.begin(), fftBuffer.end(), 0.0f);
+        for (auto s = 0; s < samplesPerFrame; ++s) {
+            fftBuffer[static_cast<size_t>(s)] = input[static_cast<size_t>(s * 2 + channel)];
+        }
+
+        fft->performRealOnlyForwardTransform(fftBuffer.data(), true);
+
+        auto* bins = reinterpret_cast<std::complex<float>*>(fftBuffer.data());
+
+        accumulateSpectrumSnapshot(bins, numBins, preSnapshot);
+
+        // When every bend is at its neutral value, skip mutating the spectrum and skip the
+        // inverse transform entirely, so the encoded audio is bit-for-bit identical to before
+        // this FFT stage existed - only the (otherwise unused) snapshot above is computed.
+        if (!identity) {
+            applyBandReassignment(bins, numBins);
+            applyPostShift(bins, numBins);
+            applyFeedback(bins, numBins, channel);
+        }
+
+        accumulateSpectrumSnapshot(bins, numBins, postSnapshot);
+
+        if (!identity) {
+            fft->performRealOnlyInverseTransform(fftBuffer.data());
+
+            for (auto s = 0; s < samplesPerFrame; ++s) {
+                input[static_cast<size_t>(s * 2 + channel)] = fftBuffer[static_cast<size_t>(s)];
+            }
+        }
+    }
+
+    for (auto i = 0; i < mdctDisplaySize; ++i) {
+        mdctPreBend[static_cast<size_t>(i)] = preSnapshot[static_cast<size_t>(i)] / 2.0f;
+        mdctPostBend[static_cast<size_t>(i)] = postSnapshot[static_cast<size_t>(i)] / 2.0f;
+    }
+}
+
+void OpusController::accumulateSpectrumSnapshot(const std::complex<float>* bins, int numBins, std::array<float, mdctDisplaySize>& target) const
+{
+    // Normalizes FFT bin magnitude to roughly the 0-1(+) range rescaleMDCT() expects, and
+    // resamples the (sample-rate-dependent) bin count down to the graph's fixed display width.
+    const auto normalization = static_cast<float>(fftSize) / 2.0f;
+    for (auto i = 0; i < mdctDisplaySize; ++i) {
+        auto sourceBin = std::min(numBins - 1, (i * numBins) / mdctDisplaySize);
+        target[static_cast<size_t>(i)] += std::abs(bins[static_cast<size_t>(sourceBin)]) / normalization;
+    }
+}
+
+void OpusController::applyBandReassignment(std::complex<float>* bins, int numBins)
+{
+    const auto numBands = OpusBendFlags::numReassignmentBands;
+    if (bendFlags.mdctBandReassignment == OpusBendFlags::identityReassignment()) {
+        return;
+    }
+
+    std::copy(bins, bins + numBins, reassignmentScratch.begin());
+
+    auto bandBoundary = [numBins, numBands] (int band) {
+        return (band * numBins) / numBands;
+    };
+
+    for (auto band = 0; band < numBands; ++band) {
+        auto destStart = bandBoundary(band);
+        auto destEnd = bandBoundary(band + 1);
+        auto srcBand = juce::jlimit(0, numBands - 1, bendFlags.mdctBandReassignment[static_cast<size_t>(band)]);
+        auto srcStart = bandBoundary(srcBand);
+        auto srcEnd = bandBoundary(srcBand + 1);
+        auto count = std::min(destEnd - destStart, srcEnd - srcStart);
+        for (auto i = 0; i < count; ++i) {
+            bins[destStart + i] = reassignmentScratch[static_cast<size_t>(srcStart + i)];
+        }
+    }
+}
+
+void OpusController::applyPostShift(std::complex<float>* bins, int numBins)
+{
+    const auto hShift = bendFlags.mdctPostHShift;
+    if (hShift != 0) {
+        std::copy(bins, bins + numBins, reassignmentScratch.begin());
+        for (auto i = 0; i < numBins; ++i) {
+            auto srcIndex = i - hShift;
+            bins[i] = (srcIndex >= 0 && srcIndex < numBins)
+                ? reassignmentScratch[static_cast<size_t>(srcIndex)]
+                : std::complex<float>(0.0f, 0.0f);
+        }
+    }
+
+    const auto vShift = bendFlags.mdctPostVShift;
+    if (vShift < 0.0f || vShift > 0.0f) {
+        auto peak = 0.0f;
+        for (auto i = 0; i < numBins; ++i) {
+            peak = std::max(peak, std::abs(bins[i]));
+        }
+        if (peak > 0.0f) {
+            for (auto i = 0; i < numBins; ++i) {
+                auto mag = std::abs(bins[i]);
+                float newMag;
+                if (vShift > 0.0f) {
+                    newMag = mag + vShift * (peak - mag);
+                } else {
+                    auto attenuation = std::max(0.0f, 1.0f + vShift * (1.0f - mag / peak));
+                    newMag = mag * attenuation;
+                }
+                if (mag > 1.0e-8f) {
+                    bins[i] *= (newMag / mag);
+                } else if (vShift > 0.0f) {
+                    bins[i] = std::complex<float>(newMag, 0.0f);
+                }
+            }
+        }
+    }
+}
+
+void OpusController::applyFeedback(std::complex<float>* bins, int numBins, int channel)
+{
+    auto& previous = feedbackSpectrum[static_cast<size_t>(channel)];
+    const auto feedback = bendFlags.mdctFeedback;
+    if (feedback > 0.0f) {
+        for (auto i = 0; i < numBins; ++i) {
+            bins[i] = bins[i] * (1.0f - feedback) + previous[static_cast<size_t>(i)] * feedback;
+        }
+    }
+    std::copy(bins, bins + numBins, previous.begin());
+}
+
+void OpusController::timerCallback()
+{
+    // Mp3ControllerManager (lame/blade) owns this same "mdct" ValueTree node while it's the
+    // active encoder - only push while Opus is actually selected, so the two don't fight.
+    bool isOpus = (((juce::AudioParameterChoice*)parameters.getParameter(ENCODER_PARAM_ID))->getIndex() == 2);
+    if (!isOpus) {
+        return;
+    }
+
+    auto mdctSamples = parameters.state.getChildWithName("mdct");
+    if (!mdctSamples.isValid()) {
+        return;
+    }
+
+    juce::var preBendV, postBendV;
+    for (auto i = 0; i < mdctDisplaySize; ++i) {
+        preBendV.append(rescaleMDCT(mdctPreBend[static_cast<size_t>(i)]));
+        postBendV.append(rescaleMDCT(mdctPostBend[static_cast<size_t>(i)]));
+    }
+    mdctSamples.setProperty("pre", preBendV, nullptr);
+    mdctSamples.setProperty("post", postBendV, nullptr);
 }

source/CodecControllers/OpusController.h

@@ -8,13 +8,16 @@
 #include "CodecController.h"
 #include "juce_audio_basics/juce_audio_basics.h"
 #include "juce_audio_processors/juce_audio_processors.h"
+#include "juce_dsp/juce_dsp.h"
 #include "opus.h"
 #include <array>
+#include <complex>
 #include <vector>
 #include "../QueueBuffer.h"
 #include "PacketLossModel.h"
+#include "OpusBendFlags.h"
 
-class OpusController : public CodecController, public juce::AudioProcessorValueTreeState::Listener {
+class OpusController : public CodecController, public juce::AudioProcessorValueTreeState::Listener, public juce::Timer {
 public:
     explicit OpusController(juce::AudioProcessorValueTreeState& _parameters);
 
@@ -33,12 +36,27 @@ class OpusController : public CodecController, public juce::AudioProcessorValueT
     void setError(float) override {}
     void setButterflyBends(float, float, float, float) override {}
     void setMDCTbandstepBends(bool, int) override {}
-    void setMDCTpostshiftBends(int, float) override {}
     void setMDCTwindowincrBends(int) override {}
-    void setMDCTBandReassignmentBends(int*) override {}
     void setBitrateSquishBends(float) override {}
     void setThresholdBias(float) override {}
-    void setMDCTfeedback(float) override {}
+
+    void setMDCTpostshiftBends(int h_shift, float v_shift) override
+    {
+        bendFlags.mdctPostHShift = h_shift;
+        bendFlags.mdctPostVShift = v_shift;
+    }
+
+    void setMDCTBandReassignmentBends(int* order) override
+    {
+        for (int i = 0; i < OpusBendFlags::numReassignmentBands; ++i) {
+            bendFlags.mdctBandReassignment[static_cast<size_t>(i)] = order[i];
+        }
+    }
+
+    void setMDCTfeedback(float feedback) override
+    {
+        bendFlags.mdctFeedback = feedback;
+    }
 
     float* getPsychoanalThreshold() override {
         return nullptr;
@@ -47,10 +65,10 @@ class OpusController : public CodecController, public juce::AudioProcessorValueT
         return nullptr;
     }
     float* getMDCTpreBend() override {
-        return nullptr;
+        return mdctPreBend.data();
     }
     float* getMDCTpostBend() override {
-        return nullptr;
+        return mdctPostBend.data();
     }
     int getShortBlockStatus() override {
         return 0;
@@ -73,6 +91,14 @@ class OpusController : public CodecController, public juce::AudioProcessorValueT
         return false;
     }
 
+    static constexpr int mdctDisplaySize = 576;
+
+    void applyFrequencyBends();
+    void applyBandReassignment(std::complex<float>* bins, int numBins);
+    void applyPostShift(std::complex<float>* bins, int numBins);
+    void applyFeedback(std::complex<float>* bins, int numBins, int channel);
+    void accumulateSpectrumSnapshot(const std::complex<float>* bins, int numBins, std::array<float, mdctDisplaySize>& target) const;
+    void timerCallback() override;
 
     bool bInitialized;
     OpusEncoder* opusEncoder;
@@ -106,6 +132,16 @@ class OpusController : public CodecController, public juce::AudioProcessorValueT
     PacketLossMode packetLossMode{PacketLossMode::pulse};
     int encodeResult{};
     float error{};
+
+    OpusBendFlags bendFlags;
+    std::unique_ptr<juce::dsp::FFT> fft;
+    int fftOrder{};
+    int fftSize{};
+    std::array<std::vector<float>, 2> fftBuffers;
+    std::vector<std::complex<float>> reassignmentScratch;
+    std::array<std::vector<std::complex<float>>, 2> feedbackSpectrum;
+    std::array<float, mdctDisplaySize> mdctPreBend{};
+    std::array<float, mdctDisplaySize> mdctPostBend{};
 };
 
 #endif //MAIM_OPUSCONTROLLER_H