deepblue/samplebrain
1
0
Fork 0
mirror of https://gitlab.com/then-try-this/samplebrain.git synced 2025-05-12 18:47:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

realtime mode ish

Browse Source
This commit is contained in:
Dave Griffiths 2015-07-09 21:48:59 +01:00
parent 7f259c21a5
commit d2b56e53e2
10 changed files with 57 additions and 521 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
samplebrain/Makefile.in
View File

@ -1,15 +1,17 @@
TARGET := samplebrain TARGET := samplebrain
TARGET_LIB := libsamplebrain.a TARGET_LIB := libsamplebrain.a
SRCS := src/fft.cpp \ SRCS := src/fft.cpp \
src/brain.cpp \ src/brain.cpp \
src/brain_block.cpp \ src/block.cpp \
src/libmfcc.cpp \ src/libmfcc.cpp \
src/main.cpp \ src/main.cpp \
src/mfcc.cpp \ src/mfcc.cpp \
src/aquila/filter/MelFilterBank.cpp \ src/renderer.cpp \
src/aquila/filter/MelFilter.cpp \ src/aquila/filter/MelFilterBank.cpp \
src/aquila/transform/Dct.cpp src/aquila/filter/MelFilter.cpp \
src/aquila/transform/Dct.cpp \
src/renderer.cpp
TARGET_SRCS := src/main.cpp TARGET_SRCS := src/main.cpp

10
samplebrain/src/brain.cpp
View File

@ -54,7 +54,7 @@ void brain::chop_and_add(const sample &s, u32 block_size, u32 overlap, bool ditc
cerr<<"adding: "<<pos/(float)s.get_length()*100; cerr<<"adding: "<<pos/(float)s.get_length()*100;
sample region; sample region;
s.get_region(region,pos,pos+block_size-1); s.get_region(region,pos,pos+block_size-1);
m_blocks.push_back(brain_block("",region,44100,ditchpcm)); m_blocks.push_back(block("",region,44100,ditchpcm));
pos += (block_size-overlap); pos += (block_size-overlap);
} }
} }
@ -63,16 +63,16 @@ const sample &brain::get_block_pcm(u32 index) const {
return m_blocks[index].get_pcm(); return m_blocks[index].get_pcm();
} }
const brain_block &brain::get_block(u32 index) const { const block &brain::get_block(u32 index) const {
return m_blocks[index]; return m_blocks[index];
} }
// returns index to block // returns index to block
u32 brain::search(const brain_block &target, float ratio) const { u32 brain::search(const block &target, float ratio) const {
double closest = 999999999; double closest = 999999999;
u32 closest_index = 0; u32 closest_index = 0;
u32 index = 0; u32 index = 0;
for (vector<brain_block>::const_iterator i=m_blocks.begin(); i!=m_blocks.end(); ++i) { for (vector<block>::const_iterator i=m_blocks.begin(); i!=m_blocks.end(); ++i) {
double diff = target.compare(*i,ratio); double diff = target.compare(*i,ratio);
if (diff<closest) { if (diff<closest) {
closest=diff; closest=diff;
@ -92,7 +92,7 @@ void brain::resynth(const string &filename, const brain &other, float ratio){
u32 count = 0; u32 count = 0;
cerr<<other.m_blocks.size()<<" brain blocks..."<<endl; cerr<<other.m_blocks.size()<<" brain blocks..."<<endl;
for (vector<brain_block>::iterator i=m_blocks.begin(); i!=m_blocks.end(); ++i) { for (vector<block>::iterator i=m_blocks.begin(); i!=m_blocks.end(); ++i) {
cerr<<'\r'; cerr<<'\r';
cerr<<"searching: "<<count/float(m_blocks.size())*100; cerr<<"searching: "<<count/float(m_blocks.size())*100;
u32 index = other.search(*i,ratio); u32 index = other.search(*i,ratio);

16
samplebrain/src/brain.h
View File

@ -2,7 +2,7 @@
#include <string> #include <string>
#include "jellyfish/core/types.h" #include "jellyfish/core/types.h"
#include "jellyfish/fluxa/sample.h" #include "jellyfish/fluxa/sample.h"
#include "brain_block.h" #include "block.h"
#ifndef BRAIN #ifndef BRAIN
#define BRAIN #define BRAIN
@ -24,11 +24,11 @@ public:
void resynth(const std::string &filename, const brain &other, float ratio); void resynth(const std::string &filename, const brain &other, float ratio);
const sample &get_block_pcm(u32 index) const; const sample &get_block_pcm(u32 index) const;
const brain_block &get_block(u32 index) const; const block &get_block(u32 index) const;
const u32 &get_block_size() const { return m_block_size; } const u32 get_block_size() const { return m_block_size; }
const u32 &get_overlap() const { return m_overlap; } const u32 get_overlap() const { return m_overlap; }
u32 search(const brain_block &target, float ratio) const; u32 search(const block &target, float ratio) const;
static bool unit_test(); static bool unit_test();
@ -36,7 +36,7 @@ private:
void chop_and_add(const sample &s, u32 block_size, u32 overlap, bool ditchpcm=false); void chop_and_add(const sample &s, u32 block_size, u32 overlap, bool ditchpcm=false);
vector<brain_block> m_blocks; vector<block> m_blocks;
vector<sample> m_samples; vector<sample> m_samples;
u32 m_block_size; u32 m_block_size;
@ -44,6 +44,6 @@ private:
}; };
#endif
} }
#endif

138
samplebrain/src/brain_block.cpp
View File

@ -1,138 +0,0 @@
#include <assert.h>
#include <iostream>
#include "libmfcc.h"
#include "brain_block.h"
using namespace spiralcore;
FFT *brain_block::m_fftw;
Aquila::Mfcc *brain_block::m_mfcc_proc;
static const int MFCC_FILTERS=12;
void enveloper(sample &s, u32 start, u32 end) {
for(u32 i=0; i<start; ++i) {
s[i]*=i/(float)start;
}
for(u32 i=0; i<end; ++i) {
s[(s.get_length()-1)-i]*=i/(float)end;
}
}
brain_block::brain_block(const string &filename, const sample &pcm, u32 rate, bool ditchpcm) :
m_pcm(pcm),
m_fft(pcm.get_length()),
m_mfcc(MFCC_FILTERS),
m_block_size(pcm.get_length()),
m_rate(rate),
m_orig_filename(filename)
{
init_fft(m_pcm.get_length());
assert(m_mfcc_proc!=NULL);
assert(m_fftw!=NULL);
enveloper(m_pcm,50,50);
m_fftw->impulse2freq(m_pcm.get_non_const_buffer());
std::vector<std::complex<double>> mfspec;
for (u32 i=0; i<m_block_size; ++i) {
m_fft[i]=m_fftw->m_spectrum[i][0];
mfspec.push_back(std::complex<double>(m_fftw->m_spectrum[i][0],
m_fftw->m_spectrum[i][1]));
}
if (m_block_size>100) m_fft.crop_to(100);
if (ditchpcm) m_pcm.clear();
// calculate mfcc
std::vector<double> m = m_mfcc_proc->calculate(mfspec,MFCC_FILTERS);
for (u32 i=0; i<MFCC_FILTERS; ++i) {
m_mfcc[i] = m[i];
}
}
void brain_block::init_fft(u32 block_size)
{
if (m_fftw == NULL || m_fftw->m_length!=block_size) {
if (m_fftw == NULL) delete m_fftw;
m_fftw = new FFT(block_size);
if (m_mfcc_proc == NULL) delete m_mfcc_proc;
m_mfcc_proc = new Aquila::Mfcc(block_size);
}
}
double brain_block::compare(const brain_block &other, float ratio) const {
double mfcc_acc=0;
double fft_acc=0;
if (ratio==0) {
for (u32 i=0; i<m_fft.get_length(); ++i) {
fft_acc+=(m_fft[i]-other.m_fft[i]) * (m_fft[i]-other.m_fft[i]);
}
return fft_acc/(float)m_fft.get_length();
}
if (ratio==1) {
for (u32 i=0; i<MFCC_FILTERS; ++i) {
mfcc_acc+=(m_mfcc[i]-other.m_mfcc[i]) * (m_mfcc[i]-other.m_mfcc[i]);
}
return mfcc_acc/(float)MFCC_FILTERS;
}
// calculate both
for (u32 i=0; i<m_fft.get_length(); ++i) {
fft_acc+=(m_fft[i]-other.m_fft[i]) * (m_fft[i]-other.m_fft[i]);
}
for (u32 i=0; i<MFCC_FILTERS; ++i) {
mfcc_acc+=(m_mfcc[i]-other.m_mfcc[i]) * (m_mfcc[i]-other.m_mfcc[i]);
}
return (fft_acc/(float)m_fft.get_length())*(1-ratio) +
(mfcc_acc/(float)MFCC_FILTERS)*ratio;
}
bool brain_block::unit_test() {
sample data(200);
for (u32 i=0; i<data.get_length(); i++) {
data[i]=i/(float)data.get_length();
}
brain_block bb("test",data,44100);
assert(bb.m_pcm.get_length()==data.get_length());
//assert(bb.m_fft.get_length()==data.get_length());
assert(bb.m_mfcc.get_length()==MFCC_FILTERS);
assert(bb.m_orig_filename==string("test"));
assert(bb.m_rate==44100);
assert(bb.m_block_size==data.get_length());
brain_block bb2("test",data,44100);
assert(bb.compare(bb2,1)==0);
assert(bb.compare(bb2,0)==0);
assert(bb.compare(bb2,0.5)==0);
sample data2(200);
for (u32 i=0; i<data.get_length(); i++) {
data[i]=i%10;
}
brain_block cpy("test",data,100);
{
brain_block bb3("test",data2,44100);
assert(bb.compare(bb3,1)!=0);
assert(bb.compare(bb3,0)!=0);
assert(bb.compare(bb3,0.5)!=0);
cpy=bb3;
}
assert(cpy.m_pcm.get_length()==200);
return true;
}

40
samplebrain/src/brain_block.h
View File

@ -1,40 +0,0 @@
#include <string>
#include "jellyfish/fluxa/sample.h"
#include "jellyfish/core/types.h"
#include "fft.h"
#include "mfcc.h"
#ifndef BRAIN_BLOCK
#define BRAIN_BLOCK
namespace spiralcore {
class brain_block {
public:
// runs analysis on pcm
brain_block(const std::string &filename, const sample &pcm, u32 rate, bool ditchpcm=false);
// returns distance based on ratio of fft-mfcc values
double compare(const brain_block &other, float ratio) const;
static void init_fft(u32 block_size);
static bool unit_test();
const sample &get_pcm() const { return m_pcm; }
private:
sample m_pcm;
sample m_fft;
sample m_mfcc;
u32 m_block_size;
u32 m_rate;
std::string m_orig_filename;
static FFT *m_fftw;
static Aquila::Mfcc *m_mfcc_proc;
};
}
#endif

165
samplebrain/src/libmfcc.cpp
View File

@ -1,165 +0,0 @@
/*
* libmfcc.c - Code implementation for libMFCC
* Copyright (c) 2010 Jeremy Sawruk
*
* This code is released under the MIT License.
* For conditions of distribution and use, see the license in LICENSE
*/
#include <math.h>
#include "libmfcc.h"
/*
* Computes the specified (mth) MFCC
*
* spectralData - array of mfcc_reals containing the results of FFT computation. This data is already assumed to be purely real
* samplingRate - the rate that the original time-series data was sampled at (i.e 44100)
* NumFilters - the number of filters to use in the computation. Recommended value = 48
* binSize - the size of the spectralData array, usually a power of 2
* m - The mth MFCC coefficient to compute
*
*/
mfcc_real GetCoefficient(mfcc_real* spectralData, unsigned int samplingRate, unsigned int NumFilters, unsigned int binSize, unsigned int m)
{
mfcc_real result = 0.0f;
mfcc_real outerSum = 0.0f;
mfcc_real innerSum = 0.0f;
unsigned int k, l;
// 0 <= m < L
if(m >= NumFilters)
{
// This represents an error condition - the specified coefficient is greater than or equal to the number of filters. The behavior in this case is undefined.
return 0.0f;
}
result = NormalizationFactor(NumFilters, m);
for(l = 1; l <= NumFilters; l++)
{
// Compute inner sum
innerSum = 0.0f;
for(k = 0; k < binSize - 1; k++)
{
innerSum += fabs(spectralData[k] * GetFilterParameter(samplingRate, binSize, k, l));
}
if(innerSum > 0.0f)
{
innerSum = log(innerSum); // The log of 0 is undefined, so don't use it
}
innerSum = innerSum * cos(((m * PI) / NumFilters) * (l - 0.5f));
outerSum += innerSum;
}
result *= outerSum;
return result;
}
/*
* Computes the Normalization Factor (Equation 6)
* Used for internal computation only - not to be called directly
*/
mfcc_real NormalizationFactor(int NumFilters, int m)
{
mfcc_real normalizationFactor = 0.0f;
if(m == 0)
{
normalizationFactor = sqrt(1.0f / NumFilters);
}
else
{
normalizationFactor = sqrt(2.0f / NumFilters);
}
return normalizationFactor;
}
/*
* Compute the filter parameter for the specified frequency and filter bands (Eq. 2)
* Used for internal computation only - not the be called directly
*/
mfcc_real GetFilterParameter(unsigned int samplingRate, unsigned int binSize, unsigned int frequencyBand, unsigned int filterBand)
{
mfcc_real filterParameter = 0.0f;
mfcc_real boundary = (frequencyBand * samplingRate) / binSize; // k * Fs / N
mfcc_real prevCenterFrequency = GetCenterFrequency(filterBand - 1); // fc(l - 1) etc.
mfcc_real thisCenterFrequency = GetCenterFrequency(filterBand);
mfcc_real nextCenterFrequency = GetCenterFrequency(filterBand + 1);
if(boundary >= 0 && boundary < prevCenterFrequency)
{
filterParameter = 0.0f;
}
else if(boundary >= prevCenterFrequency && boundary < thisCenterFrequency)
{
filterParameter = (boundary - prevCenterFrequency) / (thisCenterFrequency - prevCenterFrequency);
filterParameter *= GetMagnitudeFactor(filterBand);
}
else if(boundary >= thisCenterFrequency && boundary < nextCenterFrequency)
{
filterParameter = (boundary - nextCenterFrequency) / (thisCenterFrequency - nextCenterFrequency);
filterParameter *= GetMagnitudeFactor(filterBand);
}
else if(boundary >= nextCenterFrequency && boundary < samplingRate)
{
filterParameter = 0.0f;
}
return filterParameter;
}
/*
* Compute the band-dependent magnitude factor for the given filter band (Eq. 3)
* Used for internal computation only - not the be called directly
*/
mfcc_real GetMagnitudeFactor(unsigned int filterBand)
{
mfcc_real magnitudeFactor = 0.0f;
if(filterBand >= 1 && filterBand <= 14)
{
magnitudeFactor = 0.015;
}
else if(filterBand >= 15 && filterBand <= 48)
{
magnitudeFactor = 2.0f / (GetCenterFrequency(filterBand + 1) - GetCenterFrequency(filterBand -1));
}
return magnitudeFactor;
}
/*
* Compute the center frequency (fc) of the specified filter band (l) (Eq. 4)
* This where the mel-frequency scaling occurs. Filters are specified so that their
* center frequencies are equally spaced on the mel scale
* Used for internal computation only - not the be called directly
*/
mfcc_real GetCenterFrequency(unsigned int filterBand)
{
mfcc_real centerFrequency = 0.0f;
mfcc_real exponent;
if(filterBand == 0)
{
centerFrequency = 0;
}
else if(filterBand >= 1 && filterBand <= 14)
{
centerFrequency = (200.0f * filterBand) / 3.0f;
}
else
{
exponent = filterBand - 14.0f;
centerFrequency = pow(1.0711703, exponent);
centerFrequency *= 1073.4;
}
return centerFrequency;
}

28
samplebrain/src/libmfcc.h
View File

@ -1,28 +0,0 @@
/*
* libmfcc.h - Header for libMFCC
* Copyright (c) 2010 Jeremy Sawruk
*
* This code is released under the MIT License.
* For conditions of distribution and use, see the license in LICENSE
*/
#pragma once
typedef float mfcc_real;
#define PI 3.14159265358979323846264338327
// Returns the specified (mth) MFCC
mfcc_real GetCoefficient(mfcc_real* spectralData, unsigned int samplingRate, unsigned int NumFilters, unsigned int binSize, unsigned int m);
// Compute the normalization factor (For internal computation only - not to be called directly)
mfcc_real NormalizationFactor(int NumFilters, int m);
// Compute the filter parameter for the specified frequency and filter bands (For internal computation only - not the be called directly)
mfcc_real GetFilterParameter(unsigned int samplingRate, unsigned int binSize, unsigned int frequencyBand, unsigned int filterBand);
// Compute the band-dependent magnitude factor for the given filter band (For internal computation only - not the be called directly)
mfcc_real GetMagnitudeFactor(unsigned int filterBand);
// Compute the center frequency (fc) of the specified filter band (l) (For internal computation only - not the be called directly)
mfcc_real GetCenterFrequency(unsigned int filterBand);

48
samplebrain/src/main.cpp
View File

@ -17,50 +17,68 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <iostream> #include <iostream>
#include <unistd.h>
#include "jellyfish/audio.h" #include "jellyfish/audio.h"
#include "brain_block.h" #include "block.h"
#include "brain.h" #include "brain.h"
#include "renderer.h"
using namespace std; using namespace std;
void unit_test() { void unit_test() {
cerr<<"testing brain_block"<<endl; cerr<<"testing block"<<endl;
if (brain_block::unit_test()) cerr<<"passed"<<endl; if (block::unit_test()) cerr<<"passed"<<endl;
else cerr<<"failed"<<endl; else cerr<<"failed"<<endl;
cerr<<"testing brain"<<endl; cerr<<"testing brain"<<endl;
if (brain::unit_test()) cerr<<"passed"<<endl; if (brain::unit_test()) cerr<<"passed"<<endl;
else cerr<<"failed"<<endl; else cerr<<"failed"<<endl;
cerr<<"testing renderer"<<endl;
if (renderer::unit_test()) cerr<<"passed"<<endl;
else cerr<<"failed"<<endl;
}
audio_device *a = NULL;
void run_audio(void* c, unsigned int frames) {
a->left_out.zero();
renderer *rr = (renderer*)c;
rr->process(frames,a->left_out.get_non_const_buffer());
// sleep(1);
} }
int main(int argc, char *argv[]) int main(int argc, char *argv[])
{ {
unit_test(); // unit_test();
cerr<<"starting"<<endl; cerr<<"starting"<<endl;
brain source, target; brain source, target;
// source.load_sound("../sound/source/shostakovich6.wav"); source.load_sound("../sound/source/shostakovich6.wav");
source.load_sound("../sound/source/amen_brother.wav"); // source.load_sound("../sound/source/808.wav");
source.load_sound("../sound/source/808.wav");
source.load_sound("../sound/source/eagle.wav");
source.load_sound("../sound/source/claps.wav");
source.load_sound("../sound/source/dream2.wav");
target.load_sound("../sound/source/camron.wav"); target.load_sound("../sound/source/sb-left.wav");
// target.load_sound("../sound/source/sb-left.wav"); // target.load_sound("../sound/source/sb-left.wav");
cerr<<"loaded sounds"<<endl; cerr<<"loaded sounds"<<endl;
u32 len=3000; u32 len=3000;
source.init(len,len-len/4); source.init(len,len-len);
target.init(len,len-len/4); target.init(len,len-len/2);
cerr<<"ready..."<<endl; cerr<<"ready..."<<endl;
audio_device *a = new audio_device("samplebrain",44100,2048); a = new audio_device("samplebrain",44100,2048);
//target.resynth_listen("shosta-dream-0.5.wav",source,0.5,a); //target.resynth_listen("shosta-dream-0.5.wav",source,0.5,a);
renderer rr(source,target,1);
a->m_client.set_callback(run_audio, &rr);
//target.resynth("shosta-dream-0.5.wav",source,0.5); //target.resynth("shosta-dream-0.5.wav",source,0.5);
while (true) sleep(1);
} }

81
samplebrain/src/realtime_renderer.cpp
View File

@ -1,81 +0,0 @@
#include "realtime_renderer.h"
using namespace spiralcore;
void realtime_renderer::init(const brain &source, const brain &target, float ratio) {
m_source=source;
m_target=target;
m_render_time=0;
m_render_blocks.clear();
m_ratio = ratio;
}
void realtime_renderer::process(u32 nframes, float *buf) {
// get blocks from source for the current buffer
u32 src_shift = m_source.get_block_size()-m_source.get_overlap();
u32 tgt_shift = m_target.get_block_size()-m_target.get_overlap();
u32 tgt_start = m_render_time/m_tgt_shift;
u32 tgt_end = (m_render_time+nframes)/m_tgt_shift;
// get indices for current buffer
for (u32 tgt_index = tgt_start; tgt_index<tgt_end; tgt_index++) {
u32 time=tgt_index*tgt_shift;
u32 src_index = m_source.search(m_target.get_block(tgt_index), m_ratio);
// put them in the index list
m_render_blocks.push_back(render_block(src_index,time));
}
// render all blocks in list
for (list<render_block>::iterator i=m_render_blocks.begin(); i!=m_render_blocks.end(); ++i) {
sample &pcm=m_source.get_block_pcm(i->m_index);
// get the sample offset into the buffer
s32 offset = i->m_time-m_render_time;
// assume midway through block
u32 block_start = offset;
u32 buffer_start = 0;
if (block_start<0) {
block_start=-block_start;
} else { // block is midway through buffer
block_start=0;
buffer_start=offset;
if (buffer_start>nframes) i->m_finished=true;
}
if (!i->m_finished) {
// mix in
u32 buffer_pos = buffer_start;
u32 block_pos = block_start;
u32 block_end = block_start+pcm.get_length();
while (block_pos<block_end && buffer_pos<nframes) {
buf[buffer_pos]+=pcm[block_pos];
++buffer_pos;
++block_pos;
}
}
}
// delete old ones
list<render_block>::iterator i=m_render_blocks.begin();
list<render_block>::iterator ni=m_render_blocks.begin();
while(i!=m_render_blocks.end()) {
ni++;
if (i->m_finished) m_render_blocks.delete(i);
i=ni;
}
}
bool realtime_renderer::unit_test() {
brain source;
source.load_sound("test_data/up.wav");
brain target;
target.load_sound("test_data/up.wav");
realtime_renderer rr();
rr.init(source,target,1);
float *buf=new float[10];
rr.process(10,buf);
}

32
samplebrain/src/realtime_renderer.h
View File

@ -1,32 +0,0 @@
#include <jellyfish/fluxa/sample.h>
#include <brain.h>
namespace spiralcore {
class realtime_renderer {
public:
void init(const brain &source, const brain &target);
void process(u32 nframes, float *buf);
bool unit_test();
private:
// realtime stuff
class render_block {
public:
render_block(u32 index, u32 time) :
m_index(index), m_time(time), m_finished(false) {}
u32 m_index;
u32 m_time; // in samples
bool m_finished;
};
const brain &m_source;
const brain &m_target;
list<render_block> m_render_blocks;
u32 m_render_time;
};
}