From 72822d03aa6a4cf60938e1debb03433b6d5c4848 Mon Sep 17 00:00:00 2001 From: paxcut <53811119+paxcut@users.noreply.github.com> Date: Fri, 29 Nov 2024 09:22:22 -0700 Subject: [PATCH] fix: Stereo sound visualizations (#1970) Even tough the sound visualizer has `channels` as one of its parameters it wasn't using it properly. ### Problem description The biggest problem is that at each frame the index was being advanced per channel frame_count increments. The number of channels also determines how many graph will be needed to display the graphs of the visualized sound files. Besides these two problems there were many others like incorrect playback time, cracking audio, etc. which will not be mentioned. ### Implementation description To sample the signal a channel sampler was created based on the one used previously that returns as many sampled signals as there are channels. This PR aims hopefully at fixing all the problems encountered, and it has been tested extensively using `Audacity` exported samples to ensure the visualizer fidelity on playback and graph appearance. ### Screenshots ![image](https://github.com/user-attachments/assets/03453860-693f-4af4-b6c6-e828a102c389) --- lib/libimhex/include/hex/helpers/utils.hpp | 23 ++++ .../source/content/pl_visualizers/sound.cpp | 126 ++++++++++-------- 2 files changed, 95 insertions(+), 54 deletions(-) diff --git a/lib/libimhex/include/hex/helpers/utils.hpp b/lib/libimhex/include/hex/helpers/utils.hpp index 067f0a3b7..eee48a012 100644 --- a/lib/libimhex/include/hex/helpers/utils.hpp +++ b/lib/libimhex/include/hex/helpers/utils.hpp @@ -34,6 +34,29 @@ namespace hex { class Provider; } + template + [[nodiscard]] std::vector> sampleChannels(const std::vector &data, size_t count, size_t channels) { + if (channels == 0) return {}; + size_t signalLength = std::max(1.0, double(data.size()) / channels); + + size_t stride = std::max(1.0, double(signalLength) / count); + + std::vector> result; + result.resize(channels); + for (size_t i = 0; i < channels; i++) { + result[i].reserve(count); + } + result.reserve(count); + + for (size_t i = 0; i < data.size(); i += stride) { + for (size_t j = 0; j < channels; j++) { + result[j].push_back(data[i + j]); + } + } + + return result; + } + template [[nodiscard]] std::vector sampleData(const std::vector &data, size_t count) { size_t stride = std::max(1.0, double(data.size()) / count); diff --git a/plugins/visualizers/source/content/pl_visualizers/sound.cpp b/plugins/visualizers/source/content/pl_visualizers/sound.cpp index de163496f..2c5b6f324 100644 --- a/plugins/visualizers/source/content/pl_visualizers/sound.cpp +++ b/plugins/visualizers/source/content/pl_visualizers/sound.cpp @@ -16,8 +16,11 @@ namespace hex::plugin::visualizers { auto wavePattern = arguments[0].toPattern(); auto channels = arguments[1].toUnsigned(); auto sampleRate = arguments[2].toUnsigned(); + u32 downSampling = wavePattern->getSize() / 300_scaled / 8 / channels; - static std::vector waveData, sampledData; + static std::vector waveData; + static std::vector> sampledData; + sampledData.resize(channels); static ma_device audioDevice; static ma_device_config deviceConfig; static bool shouldStop = false; @@ -28,14 +31,16 @@ namespace hex::plugin::visualizers { throw std::logic_error(hex::format("Invalid sample rate: {}", sampleRate)); else if (channels == 0) throw std::logic_error(hex::format("Invalid channel count: {}", channels)); - + u64 sampledIndex; if (shouldReset) { waveData.clear(); resetTask = TaskManager::createTask("hex.visualizers.pl_visualizer.task.visualizing"_lang, TaskManager::NoProgress, [=](Task &) { ma_device_stop(&audioDevice); waveData = patternToArray(wavePattern.get()); - sampledData = sampleData(waveData, 300_scaled * 4); + if (waveData.empty()) + return; + sampledData = sampleChannels(waveData, 300_scaled * 4, channels); index = 0; deviceConfig = ma_device_config_init(ma_device_type_playback); @@ -51,74 +56,87 @@ namespace hex::plugin::visualizers { } ma_copy_pcm_frames(pOutput, waveData.data() + index, frameCount, device->playback.format, device->playback.channels); - index += frameCount; + index += frameCount * device->playback.channels; }; ma_device_init(nullptr, &deviceConfig, &audioDevice); }); } - + sampledIndex = index / downSampling; ImGui::BeginDisabled(resetTask.isRunning()); - + u32 waveDataSize = waveData.size(); + u32 sampledDataSize = sampledData[0].size(); + auto subplotFlags = ImPlotSubplotFlags_LinkAllX | ImPlotSubplotFlags_LinkCols | ImPlotSubplotFlags_NoResize; + auto plotFlags = ImPlotFlags_CanvasOnly | ImPlotFlags_NoFrame | ImPlotFlags_NoInputs; + auto axisFlags = ImPlotAxisFlags_NoDecorations | ImPlotAxisFlags_NoMenus | ImPlotAxisFlags_AutoFit; ImPlot::PushStyleVar(ImPlotStyleVar_PlotPadding, ImVec2(0, 0)); - if (ImPlot::BeginPlot("##amplitude_plot", scaled(ImVec2(300, 80)), ImPlotFlags_CanvasOnly | ImPlotFlags_NoFrame | ImPlotFlags_NoInputs)) { - ImPlot::SetupAxes("##time", "##amplitude", ImPlotAxisFlags_NoDecorations | ImPlotAxisFlags_NoMenus, ImPlotAxisFlags_NoDecorations | ImPlotAxisFlags_NoMenus); - ImPlot::SetupAxesLimits(0, waveData.size(), std::numeric_limits::min(), std::numeric_limits::max(), ImGuiCond_Always); - double dragPos = index; - if (ImPlot::DragLineX(1, &dragPos, ImGui::GetStyleColorVec4(ImGuiCol_Text))) { - if (dragPos < 0) dragPos = 0; - if (dragPos >= waveData.size()) dragPos = waveData.size() - 1; + if (ImPlot::BeginSubplots("##AxisLinking", channels, 1, scaled(ImVec2(300, 80 * channels)), subplotFlags)) { + for (u32 i = 0; i < channels; i++) { + if (ImPlot::BeginPlot("##amplitude_plot", scaled(ImVec2(300, 80)), plotFlags)) { - index = dragPos; + ImPlot::SetupAxes("##time", "##amplitude", axisFlags, axisFlags); + double dragPos = sampledIndex; + if (ImPlot::DragLineX(1, &dragPos, ImGui::GetStyleColorVec4(ImGuiCol_Text))) { + if (dragPos < 0) dragPos = 0; + if (dragPos >= sampledDataSize) dragPos = sampledDataSize - 1; + + sampledIndex = dragPos; + } + ImPlot::PlotLine("##audio", sampledData[i].data(), sampledDataSize); + + ImPlot::EndPlot(); + } + } + ImPlot::PopStyleVar(); + + index = sampledIndex * downSampling; + { + const u64 min = 0, max = sampledDataSize-1; + ImGui::PushItemWidth(300_scaled); + ImGui::PushStyleVar(ImGuiStyleVar_FramePadding, ImVec2(0, 0)); + ImGui::SliderScalar("##index", ImGuiDataType_U64, &sampledIndex, &min, &max, ""); + ImGui::PopStyleVar(); + ImGui::PopItemWidth(); + } + index = sampledIndex * downSampling; + + + if (shouldStop) { + shouldStop = false; + ma_device_stop(&audioDevice); } - ImPlot::PlotLine("##audio", sampledData.data(), sampledData.size()); + bool playing = ma_device_is_started(&audioDevice); - ImPlot::EndPlot(); - } - ImPlot::PopStyleVar(); + if (ImGuiExt::IconButton(playing ? ICON_VS_DEBUG_PAUSE : ICON_VS_PLAY, ImGuiExt::GetCustomColorVec4(ImGuiCustomCol_ToolbarGreen))) { + if (playing) + ma_device_stop(&audioDevice); + else + ma_device_start(&audioDevice); + } - { - const u64 min = 0, max = waveData.size(); - ImGui::PushItemWidth(300_scaled); - ImGui::PushStyleVar(ImGuiStyleVar_FramePadding, ImVec2(0, 0)); - ImGui::SliderScalar("##index", ImGuiDataType_U64, &index, &min, &max, ""); - ImGui::PopStyleVar(); - ImGui::PopItemWidth(); - } + ImGui::SameLine(); - if (shouldStop) { - shouldStop = false; - ma_device_stop(&audioDevice); - } - - bool playing = ma_device_is_started(&audioDevice); - - if (ImGuiExt::IconButton(playing ? ICON_VS_DEBUG_PAUSE : ICON_VS_PLAY, ImGuiExt::GetCustomColorVec4(ImGuiCustomCol_ToolbarGreen))) { - if (playing) + if (ImGuiExt::IconButton(ICON_VS_DEBUG_STOP, ImGuiExt::GetCustomColorVec4(ImGuiCustomCol_ToolbarRed))) { + index = 0; + sampledIndex = 0; ma_device_stop(&audioDevice); + } + + ImGui::EndDisabled(); + + ImGui::SameLine(); + index = sampledIndex * downSampling; + + if (resetTask.isRunning()) + ImGuiExt::TextSpinner(""); else - ma_device_start(&audioDevice); + ImGuiExt::TextFormatted("{:02d}:{:02d}:{:03d} / {:02d}:{:02d}:{:03d}", + (index / sampleRate / channels) / 60, (index / sampleRate / channels) % 60, (index * 1000 / sampleRate / channels ) % 1000, + ((waveDataSize-1) / sampleRate / channels) / 60, ((waveDataSize-1) / sampleRate / channels) % 60, ((waveDataSize-1) * 1000 / sampleRate / channels) % 1000); + ImPlot::EndSubplots(); } - - ImGui::SameLine(); - - if (ImGuiExt::IconButton(ICON_VS_DEBUG_STOP, ImGuiExt::GetCustomColorVec4(ImGuiCustomCol_ToolbarRed))) { - index = 0; - ma_device_stop(&audioDevice); - } - - ImGui::EndDisabled(); - - ImGui::SameLine(); - - if (resetTask.isRunning()) - ImGuiExt::TextSpinner(""); - else - ImGuiExt::TextFormatted("{:02d}:{:02d} / {:02d}:{:02d}", - (index / sampleRate) / 60, (index / sampleRate) % 60, - (waveData.size() / sampleRate) / 60, (waveData.size() / sampleRate) % 60); } } \ No newline at end of file