diff --git a/samplebrain/qt/qtmain.cpp b/samplebrain/qt/qtmain.cpp index 44fa52e..5a27f90 100644 --- a/samplebrain/qt/qtmain.cpp +++ b/samplebrain/qt/qtmain.cpp @@ -23,6 +23,7 @@ #include "process_thread.h" #include "audio_thread.h" +#include "pitchshift.h" using namespace std; @@ -31,6 +32,8 @@ int main( int argc , char *argv[] ){ MainWindow mainWin; mainWin.show(); + pitchshift::init(44100); + process_thread pt; audio_thread at(pt); pt.register_renderer(at.m_renderer); diff --git a/samplebrain/qt/samplebrain.pro b/samplebrain/qt/samplebrain.pro index ba3cbe9..fa10264 100644 --- a/samplebrain/qt/samplebrain.pro +++ b/samplebrain/qt/samplebrain.pro @@ -25,6 +25,7 @@ SOURCES += MainWindow.cpp \ ../src/renderer.cpp \ ../src/status.cpp \ ../src/window.cpp \ + ../src/pitchshift.cpp \ ../src/aquila/filter/MelFilterBank.cpp \ ../src/aquila/filter/MelFilter.cpp \ ../src/aquila/transform/Dct.cpp \ @@ -38,7 +39,7 @@ SOURCES += MainWindow.cpp \ ../../../jellyfish/src/core/stream.cpp INCLUDEPATH += ../src -LIBS += -L.. -lportaudio -lfftw3 -lsndfile -llo -ldl -lpthread -lm +LIBS += -L.. -lrubberband -lportaudio -lfftw3 -lsndfile -llo -ldl -lpthread -lm #CONFIG+=debug QMAKE_CXXFLAGS += -Wall -Wno-unused -std=c++11 -DDONT_USE_FLUXA_GRAPH diff --git a/samplebrain/src/block.cpp b/samplebrain/src/block.cpp index ca39155..a611a18 100644 --- a/samplebrain/src/block.cpp +++ b/samplebrain/src/block.cpp @@ -26,274 +26,279 @@ FFT *block::m_fftw; Aquila::Mfcc *block::m_mfcc_proc; static const int MFCC_FILTERS=12; +static const int FFT_BIAS=200; double blend(double a, double b, double t) { - return a*(1-t)+b*t; + return a*(1-t)+b*t; } double square(double a) { - return a*a; + return a*a; } void normalise(sample &in) { - // find min/max - float max = 0; - float min = FLT_MAX; - for (u32 i=0; imax) max=in[i]; - } + // find min/max + float max = 0; + float min = FLT_MAX; + for (u32 i=0; imax) max=in[i]; + } - float mid = min+(max-min)/2.0f; + float mid = min+(max-min)/2.0f; - // remove dc - for (u32 i=0; im_length!=block_size) { - if (m_fftw == NULL) delete m_fftw; - m_fftw = new FFT(block_size,100); - if (m_mfcc_proc == NULL) delete m_mfcc_proc; - m_mfcc_proc = new Aquila::Mfcc(block_size); - } + if (m_fftw == NULL || m_fftw->m_length!=block_size) { + if (m_fftw == NULL) delete m_fftw; + m_fftw = new FFT(block_size,100); + if (m_mfcc_proc == NULL) delete m_mfcc_proc; + m_mfcc_proc = new Aquila::Mfcc(block_size); + } } -void block::process(const sample &pcm, sample &fft, sample &mfcc) { - m_fftw->impulse2freq(pcm.get_buffer()); - m_fftw->calculate_bins(); +void block::process(const sample &pcm, sample &fft, sample &mfcc, float &freq) { + m_fftw->impulse2freq(pcm.get_buffer()); + m_fftw->calculate_bins(); - // calculate fft - std::vector > mfspec; - for (u32 i=0; i(m_fftw->m_spectrum[i][0], - m_fftw->m_spectrum[i][1])); - } + // calculate fft + std::vector > mfspec; + for (u32 i=0; i(m_fftw->m_spectrum[i][0], + m_fftw->m_spectrum[i][1])); + } - u32 fft_size = m_block_size; - if (fft_size>100) { - fft.crop_to(100); - fft_size=100; - } + freq = m_fftw->calculate_dominant_freq(); - for (u32 i=0; im_bin[i]; - } + u32 fft_size = m_block_size; + if (fft_size>100) { + fft.crop_to(100); + fft_size=100; + } + + for (u32 i=0; im_bin[i]; + } - // calculate mfcc - std::vector m = m_mfcc_proc->calculate(mfspec,MFCC_FILTERS); - for (u32 i=0; i m = m_mfcc_proc->calculate(mfspec,MFCC_FILTERS); + for (u32 i=0; i1) s||b.m_id; + if (version>1) s||b.m_id; + if (version>2) { + s||b.m_dominant_freq; + s||b.m_n_dominant_freq; + } - s||b.m_pcm||b.m_fft||b.m_mfcc; - s||b.m_n_pcm||b.m_n_fft||b.m_n_mfcc; + s||b.m_pcm||b.m_fft||b.m_mfcc; + s||b.m_n_pcm||b.m_n_fft||b.m_n_mfcc; - s||b.m_block_size||b.m_rate||b.m_orig_filename; - stream_vector(s,b.m_synapse); - return s; + s||b.m_block_size||b.m_rate||b.m_orig_filename; + stream_vector(s,b.m_synapse); + return s; } bool block::unit_test() { - stream_unit_test(); + stream_unit_test(); - sample ntest(3); - u32 idx=0; - ntest[idx++]=-1; - ntest[idx++]=1; - ntest[idx++]=-1; - idx=0; - normalise(ntest); - assert(feq(ntest[idx++],-1)); - assert(feq(ntest[idx++],1)); - assert(feq(ntest[idx++],-1)); - idx=0; - ntest[idx++]=-2; - ntest[idx++]=2; - ntest[idx++]=-2; - normalise(ntest); - idx=0; - assert(feq(ntest[idx++],-1)); - assert(feq(ntest[idx++],1)); - assert(feq(ntest[idx++],-1)); - idx=0; - ntest[idx++]=19; - ntest[idx++]=20; - ntest[idx++]=19; - normalise(ntest); - idx=0; - assert(feq(ntest[idx++],-1)); - assert(feq(ntest[idx++],1)); - assert(feq(ntest[idx++],-1)); + sample ntest(3); + u32 idx=0; + ntest[idx++]=-1; + ntest[idx++]=1; + ntest[idx++]=-1; + idx=0; + normalise(ntest); + assert(feq(ntest[idx++],-1)); + assert(feq(ntest[idx++],1)); + assert(feq(ntest[idx++],-1)); + idx=0; + ntest[idx++]=-2; + ntest[idx++]=2; + ntest[idx++]=-2; + normalise(ntest); + idx=0; + assert(feq(ntest[idx++],-1)); + assert(feq(ntest[idx++],1)); + assert(feq(ntest[idx++],-1)); + idx=0; + ntest[idx++]=19; + ntest[idx++]=20; + ntest[idx++]=19; + normalise(ntest); + idx=0; + assert(feq(ntest[idx++],-1)); + assert(feq(ntest[idx++],1)); + assert(feq(ntest[idx++],-1)); - sample data(200); - for (u32 i=0; i &get_synapse() { return m_synapse; } const std::vector &get_synapse_const() const { return m_synapse; } float &get_usage() { return m_usage; } + float get_freq() const { return m_dominant_freq; } + float get_n_freq() const { return m_n_dominant_freq; } -private: + private: - void process(const sample &pcm, sample &fft, sample &mfcc); + void process(const sample &pcm, sample &fft, sample &mfcc, float &freq); double _compare(const sample &fft_a, const sample &mfcc_a, const sample &fft_b, const sample &mfcc_b, @@ -75,11 +77,14 @@ private: std::vector m_synapse; float m_usage; + float m_dominant_freq; + float m_n_dominant_freq; + friend ios &operator||(ios &s, block &b); -}; + }; -ios &operator||(ios &s, block &b); + ios &operator||(ios &s, block &b); } diff --git a/samplebrain/src/brain.cpp b/samplebrain/src/brain.cpp index d77a1fd..a992e01 100644 --- a/samplebrain/src/brain.cpp +++ b/samplebrain/src/brain.cpp @@ -71,19 +71,19 @@ void brain::init(u32 block_size, u32 overlap, window::type t, bool ditchpcm) { u32 count=0; for (std::list::iterator i=m_samples.begin(); i!=m_samples.end(); ++i) { count++; - chop_and_add(i->m_sample, count, ditchpcm); + chop_and_add(*i, count, ditchpcm); } status::update("all samples processed"); } -void brain::chop_and_add(const sample &s, u32 count, bool ditchpcm) { +void brain::chop_and_add(const sound &s, u32 count, bool ditchpcm) { u32 pos=0; if (m_overlap>=m_block_size) m_overlap=0; - while (pos+m_block_size-1 m_blocks; std::list m_samples; + vector m_active_sounds; u32 m_block_size; u32 m_overlap; diff --git a/samplebrain/src/fft.cpp b/samplebrain/src/fft.cpp index 57d6f6c..f83930d 100644 --- a/samplebrain/src/fft.cpp +++ b/samplebrain/src/fft.cpp @@ -25,58 +25,73 @@ using namespace std; static const int MAX_FFT_LENGTH = 4096; FFT::FFT(u32 length, u32 bins) : - m_length(length), - m_num_bins(bins), - m_in(new double[length]), - m_spectrum(new fftw_complex[length]), - m_bin(new float[bins]) + m_length(length), + m_num_bins(bins), + m_in(new double[length]), + m_spectrum(new fftw_complex[length]), + m_bin(new float[bins]) { memset(m_spectrum,0,sizeof(fftw_complex)*length); - m_plan = fftw_plan_dft_r2c_1d(m_length, m_in, m_spectrum, FFTW_ESTIMATE); + m_plan = fftw_plan_dft_r2c_1d(m_length, m_in, m_spectrum, FFTW_ESTIMATE); } FFT::~FFT() { - delete[] m_in; - fftw_destroy_plan(m_plan); + delete[] m_in; + fftw_destroy_plan(m_plan); } void FFT::impulse2freq(const float *imp) { - unsigned int i; + unsigned int i; - for (i=0; ihighest) { + index=i; + highest=t; + } + } + return index * (SRATE/(float)m_length); } void FFT::calculate_bins() { - float useful_area = m_length/2; + float useful_area = m_length/2; - for (unsigned int n=0; n + +using namespace spiralcore; +using namespace std; +using namespace RubberBand; + +RubberBandStretcher *pitchshift::m_stretcher = NULL; + +void pitchshift::init(u32 srate) { + if (m_stretcher!=NULL) delete m_stretcher; + m_stretcher = new RubberBandStretcher(srate,1, + RubberBandStretcher::OptionProcessRealTime); +} + +void pitchshift::process(const sample &in, float freq_scale, sample &out) { + if (freq_scale<1/256) freq_scale=1; + if (freq_scale>255) freq_scale=255; + m_stretcher->setPitchScale(freq_scale); + const float *in_ptr = in.get_buffer(); + m_stretcher->process(&in_ptr, in.get_length(), false); + float *out_ptr = out.get_non_const_buffer(); + m_stretcher->retrieve(&out_ptr, out.get_length()); +} diff --git a/samplebrain/src/pitchshift.h b/samplebrain/src/pitchshift.h new file mode 100644 index 0000000..dd26883 --- /dev/null +++ b/samplebrain/src/pitchshift.h @@ -0,0 +1,35 @@ +// Copyright (C) 2015 Foam Kernow +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +#ifndef SPIRALCORE_PITCHSHIFT +#define SPIRALCORE_PITCHSHIFT + +#include +#include +#include + +namespace spiralcore { + + class pitchshift { + public: + static void init(u32 srate); + static void process(const sample &in, float freq_change, sample &out); + + static RubberBand::RubberBandStretcher *m_stretcher; + }; + +} +#endif diff --git a/samplebrain/src/renderer.cpp b/samplebrain/src/renderer.cpp index a381e34..bad3688 100644 --- a/samplebrain/src/renderer.cpp +++ b/samplebrain/src/renderer.cpp @@ -16,76 +16,78 @@ #include "renderer.h" #include +#include "pitchshift.h" using namespace spiralcore; using namespace std; void renderer::init(brain &source, brain &target) { - m_volume=1; - m_playing=false; - m_source=source; - m_target=target; - m_target_time=0; - m_render_time=0; - m_n_mix=0; - m_target_mix=0; - m_render_blocks.clear(); - m_search_algo=BASIC; - m_slide_error=1; - m_target_index=0; - m_render_index=0; - m_stretch=1; - m_last_tgt_shift=0; + m_volume=1; + m_playing=false; + m_source=source; + m_target=target; + m_target_time=0; + m_render_time=0; + m_n_mix=0; + m_target_mix=0; + m_render_blocks.clear(); + m_search_algo=BASIC; + m_slide_error=1; + m_target_index=0; + m_render_index=0; + m_stretch=1; + m_last_tgt_shift=0; + m_autotune=0; } static int ratio_time = 0; void renderer::reset() { - m_target_time=0; - m_render_time=0; - m_target_index=0; - m_render_index=0; - m_render_blocks.clear(); - m_source.jiggle(); + m_target_time=0; + m_render_time=0; + m_target_index=0; + m_render_index=0; + m_render_blocks.clear(); + m_source.jiggle(); } void renderer::process(u32 nframes, float *buf) { - if (!m_playing) return; - if (!find_render_blocks(nframes)) return; + if (!m_playing) return; + if (!find_render_blocks(nframes)) return; - render(nframes,buf); + render(nframes,buf); - clean_up(); + clean_up(); - m_render_time+=nframes; - //m_target_time+=nframes/(float)m_stretch; + m_render_time+=nframes; + //m_target_time+=nframes/(float)m_stretch; } // target_time = samples time into target stream // render_time = position in output stream, updated per process - used for offsets bool renderer::find_render_blocks(u32 nframes) { - // get new blocks from source for the current buffer + // get new blocks from source for the current buffer - // where are we phase? - u32 tgt_shift = m_target.get_block_size()-m_target.get_overlap(); - m_target_index = m_target_time/(float)tgt_shift; - u32 tgt_end = (m_target_time+nframes)/(float)tgt_shift; - u32 rnd_end = (m_render_time+nframes)/(float)tgt_shift; + // where are we phase? + u32 tgt_shift = m_target.get_block_size()-m_target.get_overlap(); + m_target_index = m_target_time/(float)tgt_shift; + u32 tgt_end = (m_target_time+nframes)/(float)tgt_shift; + u32 rnd_end = (m_render_time+nframes)/(float)tgt_shift; - // stuff has changed - recompute and abort - if (tgt_shift!=m_last_tgt_shift || - tgt_end>=m_target.get_num_blocks() || - m_source.get_num_blocks()==0) { - reset(); - m_last_tgt_shift = tgt_shift; - // next time... - return false; - } + // stuff has changed - recompute and abort + if (tgt_shift!=m_last_tgt_shift || + tgt_end>=m_target.get_num_blocks() || + m_source.get_num_blocks()==0) { + reset(); + m_last_tgt_shift = tgt_shift; + // next time... + return false; + } -/* + /* cerr<<"-----------------"<m_finished) { - // mix in - u32 buffer_pos = buffer_start; - u32 block_pos = block_start; - u32 block_end = pcm.get_length(); + // render phase + // render all blocks in list + for (std::list::iterator i=m_render_blocks.begin(); i!=m_render_blocks.end(); ++i) { + const sample &pcm=m_source.get_block(i->m_index).get_pcm(); + const sample &n_pcm=m_source.get_block(i->m_index).get_n_pcm(); + const sample &target_pcm=m_target.get_block(i->m_tgt_index).get_pcm(); + // get the sample offset into the buffer + s32 offset = i->m_time-m_render_time; - while (block_pos=pcm.get_length() || + i->m_position>=pcm.get_length()) i->m_finished=true; + } else { // block is midway through buffer + block_start=0; + buffer_start=offset; } + + // cerr<<"-----------------"<m_tgt_index).get_freq() / + m_source.get_block(i->m_index).get_freq(); + + // fade in/out autotune + pitch_scale = pitch_scale*m_autotune + 1.0f*(1-m_autotune); + + //pitchshift::process(pcm,pitch_scale,render_pcm); + + + if (!i->m_finished) { + // mix in + u32 buffer_pos = buffer_start; + u32 block_pos = block_start; + u32 block_end = pcm.get_length(); + + + while (block_posm_position]*(1-m_n_mix)+ + n_pcm[block_pos]*m_n_mix); + + // for mixing with target audio + float target_sample = target_pcm[block_pos]; + + buf[buffer_pos]+=(brain_sample*(1-m_target_mix) + + target_sample*m_target_mix)*0.2*m_volume; + + i->m_position+=pitch_scale; + + ++buffer_pos; + ++block_pos; + } + } + } } void renderer::clean_up() { - // cleanup phase - // delete old ones - std::list::iterator i=m_render_blocks.begin(); - std::list::iterator ni=m_render_blocks.begin(); - while(i!=m_render_blocks.end()) { - ni++; - if (i->m_finished) m_render_blocks.erase(i); - i=ni; - } + // cleanup phase + // delete old ones + std::list::iterator i=m_render_blocks.begin(); + std::list::iterator ni=m_render_blocks.begin(); + while(i!=m_render_blocks.end()) { + ni++; + if (i->m_finished) m_render_blocks.erase(i); + i=ni; + } } - - - - - - - - - - - - -void renderer::old_process(u32 nframes, float *buf) { - if (!m_playing) return; - - // get new blocks from source for the current buffer - u32 tgt_shift = m_target.get_block_size()-m_target.get_overlap(); - u32 tgt_end = (m_target_time+nframes)/(float)tgt_shift; - - if (tgt_shift!=m_last_tgt_shift || - tgt_end>=m_target.get_num_blocks() || m_source.get_num_blocks()==0) { - reset(); - m_last_tgt_shift = tgt_shift; - // next time... - return; - } - - cerr<<"-----------------"<::iterator i=m_render_blocks.begin(); i!=m_render_blocks.end(); ++i) { - const sample &pcm=m_source.get_block(i->m_index).get_pcm(); - const sample &n_pcm=m_source.get_block(i->m_index).get_n_pcm(); - const sample &target_pcm=m_target.get_block(i->m_tgt_index).get_pcm(); - // get the sample offset into the buffer - s32 offset = i->m_time-m_render_time; - - // assume midway through block - u32 block_start = offset; - u32 buffer_start = 0; - if (offset<0) { - block_start=-offset; - if (block_start>=pcm.get_length()) i->m_finished=true; - } else { // block is midway through buffer - block_start=0; - buffer_start=offset; - } - -// cerr<<"-----------------"<m_finished) { - // mix in - u32 buffer_pos = buffer_start; - u32 block_pos = block_start; - u32 block_end = pcm.get_length(); - - - while (block_pos::iterator i=m_render_blocks.begin(); - std::list::iterator ni=m_render_blocks.begin(); - while(i!=m_render_blocks.end()) { - ni++; - if (i->m_finished) m_render_blocks.erase(i); - i=ni; - } - - m_render_time+=nframes; - m_target_time+=nframes; -} - - bool renderer::unit_test() { - brain source; - source.load_sound("test_data/up.wav"); - source.init(10,0,window::RECTANGLE); + brain source; + source.load_sound("test_data/up.wav"); + source.init(10,0,window::RECTANGLE); - brain target; - target.load_sound("test_data/up.wav"); - target.init(10,0,window::RECTANGLE); + brain target; + target.load_sound("test_data/up.wav"); + target.init(10,0,window::RECTANGLE); - renderer rr(source,target); - rr.set_playing(true); - float *buf=new float[400]; - rr.process(10,buf); - rr.process(10,buf); + renderer rr(source,target); + rr.set_playing(true); + float *buf=new float[400]; + rr.process(10,buf); + rr.process(10,buf); - assert(rr.m_render_blocks.size()==2); - rr.process(10,buf); - assert(rr.m_render_blocks.size()==2); - delete[] buf; - buf=new float[20]; - rr.process(20,buf); - assert(rr.m_render_blocks.size()==3); - rr.process(5,buf); - assert(rr.m_render_blocks.size()==1); + assert(rr.m_render_blocks.size()==2); + rr.process(10,buf); + assert(rr.m_render_blocks.size()==2); + delete[] buf; + buf=new float[20]; + rr.process(20,buf); + assert(rr.m_render_blocks.size()==3); + rr.process(5,buf); + assert(rr.m_render_blocks.size()==1); - target.init(10,5,window::RECTANGLE); - rr.process(10,buf); - rr.process(10,buf); - rr.process(10,buf); - rr.process(10,buf); - assert(rr.m_render_blocks.size()==4); - delete[] buf; + target.init(10,5,window::RECTANGLE); + rr.process(10,buf); + rr.process(10,buf); + rr.process(10,buf); + rr.process(10,buf); + assert(rr.m_render_blocks.size()==4); + delete[] buf; - return true; + return true; } diff --git a/samplebrain/src/renderer.h b/samplebrain/src/renderer.h index 42c325a..5911747 100644 --- a/samplebrain/src/renderer.h +++ b/samplebrain/src/renderer.h @@ -23,19 +23,19 @@ namespace spiralcore { -class renderer { -public: -renderer(brain &source, brain &target) : + class renderer { + public: + renderer(brain &source, brain &target) : m_source(source), - m_target(target), - m_search_params(0,0,0,100,0) + m_target(target), + m_search_params(0,0,0,100,0) { init(source,target); } enum search_algo { - BASIC = 0, - REV_BASIC, - SYNAPTIC, - SYNAPTIC_SLIDE + BASIC = 0, + REV_BASIC, + SYNAPTIC, + SYNAPTIC_SLIDE }; void init(brain &source, brain &target); @@ -58,7 +58,7 @@ renderer(brain &source, brain &target) : static bool unit_test(); -private: + private: bool find_render_blocks(u32 nframes); void render(u32 nframes, float *buf); @@ -68,12 +68,17 @@ private: // realtime stuff class render_block { public: - render_block(u32 index, u32 tgt_index, u32 time) : - m_index(index), m_tgt_index(tgt_index), m_time(time), m_finished(false) {} - u32 m_index; - u32 m_tgt_index; // original target block - u32 m_time; // in samples - bool m_finished; + render_block(u32 index, u32 tgt_index, u32 time) : + m_index(index), + m_tgt_index(tgt_index), + m_time(time), + m_finished(false), + m_position(0) {} + u32 m_index; + u32 m_tgt_index; // original target block + u32 m_time; // in samples + bool m_finished; + float m_position; // in samples }; brain &m_source; @@ -89,13 +94,14 @@ private: u32 m_stretch; float m_n_mix; float m_target_mix; + float m_autotune; search_algo m_search_algo; double m_slide_error; u32 m_last_tgt_shift; std::list m_render_blocks; -}; + }; }