WIP making note claps and beat ticks pitch-invariant

src/custom_sfml_audio/beat_ticks.cpp

@@ -8,15 +8,30 @@
 
 BeatTicks::BeatTicks(
     const better::Timing* timing_,
-    const std::filesystem::path& assets
+    const std::filesystem::path& assets,
+    float pitch_
 ) :
+    pitch(pitch_),
     timing(timing_),
-    beat_tick()
+    beat_tick(std::make_shared<sf::SoundBuffer>())
 {
-    if (not beat_tick.loadFromFile(assets / "sounds" / "beat.wav")) {
+    if (not beat_tick->loadFromFile(assets / "sounds" / "beat.wav")) {
         throw std::runtime_error("Could not load beat tick audio file");
     }
-    sf::SoundStream::initialize(beat_tick.getChannelCount(), beat_tick.getSampleRate());
+    sf::SoundStream::initialize(beat_tick->getChannelCount(), beat_tick->getSampleRate());
+    samples.resize(timeToSamples(sf::seconds(1)), 0);
+}
+
+BeatTicks::BeatTicks(
+    const better::Timing* timing_,
+    std::shared_ptr<sf::SoundBuffer> beat_tick_,
+    float pitch_
+) :
+    pitch(pitch_),
+    timing(timing_),
+    beat_tick(beat_tick_)
+{
+    sf::SoundStream::initialize(beat_tick_->getChannelCount(), beat_tick_->getSampleRate());
     samples.resize(timeToSamples(sf::seconds(1)), 0);
 }
 
@@ -24,6 +39,14 @@ void BeatTicks::set_timing(const better::Timing* timing_) {
     timing = timing_;
 }
 
+std::shared_ptr<BeatTicks> BeatTicks::with_pitch(float pitch) {
+    return std::make_shared<BeatTicks>(
+        timing,
+        beat_tick,
+        pitch
+    );
+}
+
 bool BeatTicks::onGetData(sf::SoundStream::Chunk& data) {
     samples.assign(samples.size(), 0);
     if (timing != nullptr) {
@@ -47,13 +70,13 @@ bool BeatTicks::onGetData(sf::SoundStream::Chunk& data) {
         for (auto it = beat_at_sample.begin(); it != beat_at_sample.end();) {
             // Should we still be playing the clap ?
             const auto next = std::next(it);
-            const auto last_audible_start = start_sample - static_cast<std::int64_t>(beat_tick.getSampleCount());
+            const auto last_audible_start = start_sample - static_cast<std::int64_t>(beat_tick->getSampleCount());
             if (*it <= last_audible_start) {
                 it = beat_at_sample.erase(it);
             } else {
                 const auto full_tick_start_in_buffer = *it - static_cast<std::int64_t>(start_sample);
                 const auto slice_start_in_buffer = std::max(std::int64_t(0), full_tick_start_in_buffer);
-                const auto full_tick_end_in_buffer = full_tick_start_in_buffer + static_cast<std::int64_t>(beat_tick.getSampleCount());
+                const auto full_tick_end_in_buffer = full_tick_start_in_buffer + static_cast<std::int64_t>(beat_tick->getSampleCount());
                 auto slice_end_in_buffer = full_tick_end_in_buffer;
                 bool tick_finished_playing_in_current_buffer = true;
                 if (next != beat_at_sample.end()) {
@@ -68,9 +91,9 @@ bool BeatTicks::onGetData(sf::SoundStream::Chunk& data) {
                 auto slice_start_in_tick = slice_start_in_buffer - full_tick_start_in_buffer;
                 auto slice_size = std::min(
                     slice_end_in_buffer - slice_start_in_buffer,
-                    static_cast<std::int64_t>(beat_tick.getSampleCount()) - slice_start_in_tick
+                    static_cast<std::int64_t>(beat_tick->getSampleCount()) - slice_start_in_tick
                 );
-                const auto tick_pointer = beat_tick.getSamples() + slice_start_in_tick;
+                const auto tick_pointer = beat_tick->getSamples() + slice_start_in_tick;
                 std::copy(
                     tick_pointer,
                     tick_pointer + slice_size,
@@ -102,15 +125,15 @@ std::int64_t BeatTicks::timeToSamples(sf::Time position) const {
     // This avoids most precision errors arising from "samples => Time => samples" conversions
     // Original rounding calculation is ((Micros * Freq * Channels) / 1000000) + 0.5
     // We refactor it to keep Int64 as the data type throughout the whole operation.
-    return ((static_cast<std::int64_t>(position.asMicroseconds()) * beat_tick.getSampleRate() * beat_tick.getChannelCount()) + 500000) / 1000000;
+    return ((static_cast<std::int64_t>((position / pitch).asMicroseconds()) * beat_tick->getSampleRate() * beat_tick->getChannelCount()) + 500000) / 1000000;
 }
 
 sf::Time BeatTicks::samplesToTime(std::int64_t samples) const {
     sf::Time position = sf::Time::Zero;
 
     // Make sure we don't divide by 0
-    if (beat_tick.getSampleRate() != 0 && beat_tick.getChannelCount() != 0)
+    if (beat_tick->getSampleRate() != 0 && beat_tick->getChannelCount() != 0)
-        position = sf::microseconds((samples * 1000000) / (beat_tick.getChannelCount() * beat_tick.getSampleRate()));
+        position = sf::microseconds((samples * 1000000) / (beat_tick->getChannelCount() * beat_tick->getSampleRate()));
 
-    return position;
+    return position * pitch;
 }

src/custom_sfml_audio/beat_ticks.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <memory>
 #include <set>
 
 #include <SFML/Audio/SoundBuffer.hpp>
@@ -11,17 +12,26 @@ class BeatTicks: public PreciseSoundStream {
 public:
     BeatTicks(
         const better::Timing* timing_,
-        const std::filesystem::path& assets
+        const std::filesystem::path& assets,
+        float pitch_
+    );
+
+    BeatTicks(
+        const better::Timing* timing_,
+        std::shared_ptr<sf::SoundBuffer> beat_tick_,
+        float pitch_
     );
 
     void set_timing(const better::Timing* timing);
-    std::atomic<bool> play_chords = true;
+    
+    std::shared_ptr<BeatTicks> with_pitch(float pitch);
 
 protected:
     bool onGetData(Chunk& data) override;
     void onSeek(sf::Time timeOffset) override;
 
 private:
+    float pitch = 1.f;
     std::vector<sf::Int16> samples;
     std::int64_t current_sample = 0;
     std::int64_t timeToSamples(sf::Time position) const;
@@ -30,5 +40,5 @@ private:
     std::set<std::int64_t> beat_at_sample;
 
     const better::Timing* timing;
-    sf::SoundBuffer beat_tick;
+    std::shared_ptr<sf::SoundBuffer> beat_tick;
 };

src/custom_sfml_audio/note_claps.cpp

@@ -10,8 +10,10 @@
 NoteClaps::NoteClaps(
     const better::Notes* notes_,
     const better::Timing* timing_,
-    const std::filesystem::path& assets
+    const std::filesystem::path& assets,
+    float pitch_
 ) :
+    pitch(pitch_),
     notes(notes_),
     timing(timing_),
     note_clap(std::make_shared<sf::SoundBuffer>())
@@ -26,8 +28,10 @@ NoteClaps::NoteClaps(
 NoteClaps::NoteClaps(
     const better::Notes* notes_,
     const better::Timing* timing_,
-    std::shared_ptr<sf::SoundBuffer> note_clap_
+    std::shared_ptr<sf::SoundBuffer> note_clap_,
+    float pitch_
 ) :
+    pitch(pitch_),
     notes(notes_),
     timing(timing_),
     note_clap(note_clap_)
@@ -41,6 +45,15 @@ void NoteClaps::set_notes_and_timing(const better::Notes* notes_, const better::
     timing = timing_;
 }
 
+std::shared_ptr<NoteClaps> NoteClaps::with_pitch(float pitch) {
+    return std::make_shared<NoteClaps>(
+        notes,
+        timing,
+        note_clap,
+        pitch
+    );
+}
+
 bool NoteClaps::onGetData(sf::SoundStream::Chunk& data) {
     samples.assign(samples.size(), 0);
     if (timing != nullptr and notes != nullptr) {
@@ -116,7 +129,7 @@ std::int64_t NoteClaps::timeToSamples(sf::Time position) const {
     // This avoids most precision errors arising from "samples => Time => samples" conversions
     // Original rounding calculation is ((Micros * Freq * Channels) / 1000000) + 0.5
     // We refactor it to keep Int64 as the data type throughout the whole operation.
-    return ((static_cast<std::int64_t>(position.asMicroseconds()) * note_clap->getSampleRate() * note_clap->getChannelCount()) + 500000) / 1000000;
+    return ((static_cast<std::int64_t>((position / pitch).asMicroseconds()) * note_clap->getSampleRate() * note_clap->getChannelCount()) + 500000) / 1000000;
 }
 
 sf::Time NoteClaps::samplesToTime(std::int64_t samples) const {
@@ -126,5 +139,5 @@ sf::Time NoteClaps::samplesToTime(std::int64_t samples) const {
     if (note_clap->getSampleRate() != 0 && note_clap->getChannelCount() != 0)
         position = sf::microseconds((samples * 1000000) / (note_clap->getChannelCount() * note_clap->getSampleRate()));
 
-    return position;
+    return position * pitch;
 }

src/custom_sfml_audio/note_claps.hpp

@@ -15,22 +15,27 @@ public:
     NoteClaps(
         const better::Notes* notes_,
         const better::Timing* timing_,
-        const std::filesystem::path& assets
+        const std::filesystem::path& assets,
+        float pitch_
     );
 
     NoteClaps(
         const better::Notes* notes_,
         const better::Timing* timing_,
-        std::shared_ptr<sf::SoundBuffer> note_clap_
+        std::shared_ptr<sf::SoundBuffer> note_clap_,
+        float time_factor_
     );
 
     void set_notes_and_timing(const better::Notes* notes, const better::Timing* timing);
 
+    std::shared_ptr<NoteClaps> with_pitch(float pitch);
+
 protected:
     bool onGetData(Chunk& data) override;
     void onSeek(sf::Time timeOffset) override;
 
 private:
+    float pitch = 1.f;
     std::vector<sf::Int16> samples;
     std::int64_t current_sample = 0;
     std::int64_t timeToSamples(sf::Time position) const;

src/custom_sfml_audio/synced_sound_streams.cpp

@@ -3,6 +3,7 @@
 #include <boost/math/constants/constants.hpp>
 #include <cassert>
 #include <iostream>
+#include <memory>
 #include <mutex>
 #include <ostream>
 
@@ -68,31 +69,51 @@ void SyncedSoundStreams::change_streams(std::function<void()> callback) {
 
     reload_sources();
     setPlayingOffset(position);
+    setPitch(pitch);
     if (oldStatus == sf::SoundSource::Playing) {
         play();
     }
 }
 
-
+void SyncedSoundStreams::update_streams(std::map<std::string, NewStream> new_streams) {
-void SyncedSoundStreams::add_stream(const std::string& name, std::shared_ptr<PreciseSoundStream> s) {
     change_streams([&](){
-        InternalStream internal_stream{s, {}};
+        for (const auto& [name, new_stream] : new_streams) {
-        internal_stream.buffers.m_channelCount = s->getChannelCount();
+            if (contains_stream(name)) {
-        internal_stream.buffers.m_sampleRate = s->getSampleRate();
+                remove_stream_internal(name);
-        internal_stream.buffers.m_format = AudioDevice::getFormatFromChannelCount(s->getChannelCount());
+            }
-        streams.emplace(name, internal_stream);
+            add_stream_internal(name, new_stream);
+        }
     });
 }
 
+
+void SyncedSoundStreams::add_stream(const std::string& name, NewStream s) {
+    change_streams([&](){
+        add_stream_internal(name, s);
+    });
+}
+
+void SyncedSoundStreams::add_stream_internal(const std::string& name, NewStream s) {
+    InternalStream internal_stream{s.stream, {}, s.reconstruct_on_pitch_change};
+    internal_stream.buffers.m_channelCount = s.stream->getChannelCount();
+    internal_stream.buffers.m_sampleRate = s.stream->getSampleRate();
+    internal_stream.buffers.m_format = AudioDevice::getFormatFromChannelCount(s.stream->getChannelCount());
+    streams.emplace(name, internal_stream);
+}
+
 void SyncedSoundStreams::remove_stream(const std::string& name) {
     change_streams([&](){
-        if (streams.contains(name)) {
+        remove_stream_internal(name);
-            streams.at(name).clear_queue();
-        }
-        streams.erase(name);
     });
 }
 
+void SyncedSoundStreams::remove_stream_internal(const std::string& name) {
+    if (streams.contains(name)) {
+        streams.at(name).clear_queue();
+    }
+    streams.erase(name);
+}
+
 bool SyncedSoundStreams::contains_stream(const std::string& name) {
     return streams.contains(name);
 }
@@ -196,7 +217,11 @@ void SyncedSoundStreams::setPlayingOffset(sf::Time timeOffset) {
 
     // Let the derived class update the current position
     for (auto& [_, s]: streams) {
-        s.stream->public_seek_callback(timeOffset);
+        auto stream_pitch = 1.f;
+        if (s.reconstruct_on_pitch_change) {
+            stream_pitch = pitch;
+        }
+        s.stream->public_seek_callback(timeOffset * stream_pitch);
         // Restart streaming
         s.buffers.m_samplesProcessed = timeToSamples(timeOffset, s.buffers.m_sampleRate, s.buffers.m_channelCount);
     }
@@ -220,12 +245,17 @@ sf::Time SyncedSoundStreams::getPlayingOffset() const {
 
     ALfloat secs = 0.f;
     alCheck(alGetSourcef(s.stream->get_source(), AL_SEC_OFFSET, &secs));
-    return sf::seconds(
+    const auto unpitched_seconds = sf::seconds(
         secs
         + static_cast<float>(s.buffers.m_samplesProcessed)
         / static_cast<float>(s.buffers.m_sampleRate)
         / static_cast<float>(s.buffers.m_channelCount)
     );
+    if (s.reconstruct_on_pitch_change) {
+        return unpitched_seconds * pitch;
+    } else {
+        return unpitched_seconds;
+    }
 }
 
 sf::Time SyncedSoundStreams::getPrecisePlayingOffset() const {
@@ -237,16 +267,23 @@ sf::Time SyncedSoundStreams::getPrecisePlayingOffset() const {
     if (not (s.buffers.m_sampleRate && s.buffers.m_channelCount)) {
         return base;
     }
-    auto correction = (
+    auto stream_pitch = s.stream->getPitch();
-        (s.stream->alSecOffsetLatencySoft()[1] * s.stream->getPitch())
+    if (s.reconstruct_on_pitch_change) {
-        - (s.stream->lag * s.stream->getPitch())
+        stream_pitch = 1.f;
+    }
+    const auto correction = (
+        (s.stream->alSecOffsetLatencySoft()[1] * stream_pitch)
+        - (s.stream->lag * stream_pitch)
     );
     return base - correction;
 }
 
-void SyncedSoundStreams::setPitch(float pitch) {
+void SyncedSoundStreams::setPitch(float new_pitch) {
+    pitch = new_pitch;
     for (auto& [_, s] : streams) {
-        s.stream->setPitch(pitch);
+        if (not s.reconstruct_on_pitch_change) {
+            s.stream->setPitch(new_pitch);
+        }
     }
 }
 

src/custom_sfml_audio/synced_sound_streams.hpp

@@ -39,6 +39,11 @@ struct Buffers {
     std::array<sf::Int64, BufferCount> m_bufferSeeks = {0, 0, 0};
 };
 
+struct NewStream {
+    std::shared_ptr<PreciseSoundStream> stream;
+    bool reconstruct_on_pitch_change;
+};
+
 struct InternalStream {
     std::shared_ptr<PreciseSoundStream> stream;
     Buffers buffers;
@@ -52,7 +57,8 @@ public:
     SyncedSoundStreams();
     ~SyncedSoundStreams();
 
-    void add_stream(const std::string& name, std::shared_ptr<PreciseSoundStream> s);
+    void update_streams(std::map<std::string, NewStream> new_streams);
+    void add_stream(const std::string& name, NewStream s);
     void remove_stream(const std::string& name);
     bool contains_stream(const std::string& name);
 
@@ -76,6 +82,8 @@ protected:
 
 private:
     void change_streams(std::function<void()> callback);
+    void add_stream_internal(const std::string& name, NewStream s);
+    void remove_stream_internal(const std::string& name);
     void streamData();
     [[nodiscard]] bool fillAndPushBuffer(InternalStream& stream, unsigned int bufferNum, bool immediateLoop = false);
     [[nodiscard]] bool fillQueues();
@@ -86,6 +94,7 @@ private:
     void unsafe_update_streams();
     void reload_sources();
 
+    float pitch = 1.f;
     std::thread m_thread; // Thread running the background tasks
     mutable std::recursive_mutex m_threadMutex; // Thread mutex
     sf::SoundSource::Status m_threadStartState; // State the thread starts in (Playing, Paused, Stopped)

src/editor_state.cpp

@@ -35,8 +35,8 @@
 #include "variant_visitor.hpp"
 
 EditorState::EditorState(const std::filesystem::path& assets_) : 
-    note_claps(std::make_shared<NoteClaps>(nullptr, nullptr, assets_)),
+    note_claps(std::make_shared<NoteClaps>(nullptr, nullptr, assets_, 1.f)),
-    beat_ticks(std::make_shared<BeatTicks>(nullptr, assets_)),
+    beat_ticks(std::make_shared<BeatTicks>(nullptr, assets_, 1.f)),
     playfield(assets_),
     linear_view(assets_),
     applicable_timing(song.timing),
@@ -44,7 +44,7 @@ EditorState::EditorState(const std::filesystem::path& assets_) :
 {
     reload_music();
     reload_jacket();
-    audio.add_stream(note_clap_stream, note_claps);
+    audio.add_stream(note_clap_stream, {note_claps, true});
 };
 
 EditorState::EditorState(
@@ -54,8 +54,8 @@ EditorState::EditorState(
 ) : 
     song(song_),
     song_path(song_path),
-    note_claps(std::make_shared<NoteClaps>(nullptr, nullptr, assets_)),
+    note_claps(std::make_shared<NoteClaps>(nullptr, nullptr, assets_, 1.f)),
-    beat_ticks(std::make_shared<BeatTicks>(nullptr, assets_)),
+    beat_ticks(std::make_shared<BeatTicks>(nullptr, assets_, 1.f)),
     playfield(assets_),
     linear_view(assets_),
     applicable_timing(song.timing),
@@ -67,7 +67,7 @@ EditorState::EditorState(
     }
     reload_music();
     reload_jacket();
-    audio.add_stream(note_clap_stream, note_claps);
+    audio.add_stream(note_clap_stream, {note_claps, true});
 };
 
 int EditorState::get_volume() const {
@@ -124,7 +124,7 @@ void EditorState::toggle_beat_ticks() {
     if (audio.contains_stream(beat_tick_stream)) {
         audio.remove_stream(beat_tick_stream);
     } else {
-        audio.add_stream(beat_tick_stream, beat_ticks);
+        audio.add_stream(beat_tick_stream, {beat_ticks, true});
     }
 }
 
@@ -145,6 +145,16 @@ sf::SoundSource::Status EditorState::get_status() {
 }
 
 void EditorState::set_pitch(float pitch) {
+    std::map<std::string, NewStream> update;
+    if (audio.contains_stream(note_clap_stream)) {
+        note_claps = note_claps->with_pitch(pitch);
+        update[note_clap_stream] = {note_claps, true};
+    }
+    if (audio.contains_stream(beat_tick_stream)) {
+        beat_ticks = beat_ticks->with_pitch(pitch);
+        update[beat_tick_stream] = {beat_ticks, true};
+    }
+    audio.update_streams(update);
     audio.setPitch(pitch);
 }
 
@@ -902,7 +912,7 @@ void EditorState::reload_music() {
     previous_playback_position = playback_position;
     set_speed(speed);
     if (music.has_value()) {
-        audio.add_stream(music_stream, *music);
+        audio.add_stream(music_stream, {*music, false});
     } else {
         audio.remove_stream(music_stream);
     }