ydeng/rhythmbullet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more dynamic audio analyzer with optimizations; music selection page

Browse Source 
progress;
This commit is contained in:
Harrison Deng
2018-09-05 19:28:44 -05:00
parent eaca473dee
commit 49a441132d
14 changed files with 268 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/zero1hd/rhythmbullet/audio/MinimalAudioHeader.java
View File

@@ -19,7 +19,8 @@ public class MinimalAudioHeader {
private SupportedFormats format;
private FileHandle musicFile;
public MinimalAudioHeader(FileHandle musicFile) {
if (musicFile == null) throw new IllegalArgumentException("musicFile for minimal audio headers should not be null.");
this.musicFile = musicFile;
format = SupportedFormats.valueOf(musicFile.extension().toUpperCase());
try {
AudioFile file = AudioFileIO.read(musicFile.file());

10
core/src/zero1hd/rhythmbullet/audio/MusicController.java
View File

@@ -48,7 +48,7 @@ public class MusicController extends Observable implements OnCompletionListener,
if (music != null) {
Gdx.app.debug("MusicController", "Playing from controller.");
music.play();
music.setVolume(prefs.getFloat("music vol", 1f));
music.setVolume(prefs.getFloat("music vol", 100f)/100f);
setChanged();
notifyObservers(states.PLAYING);
} else {
@@ -141,6 +141,10 @@ public class MusicController extends Observable implements OnCompletionListener,
this.shuffle = shuffle;
}
public void setLoop(boolean loop) {
music.setLooping(loop);
}
public boolean isShuffle() {
return shuffle;
}
@@ -149,6 +153,10 @@ public class MusicController extends Observable implements OnCompletionListener,
return autoPlay;
}
public boolean isLoop() {
return music.isLooping();
}
/**
* Loads the current selected song.
*/

288
core/src/zero1hd/rhythmbullet/audio/analyzer/AudioAnalyzer.java
View File

@@ -1,288 +0,0 @@
package zero1hd.rhythmbullet.audio.analyzer;
import com.badlogic.gdx.Gdx;
import com.badlogic.gdx.math.MathUtils;
import com.badlogic.gdx.utils.FloatArray;
import edu.emory.mathcs.jtransforms.fft.FloatFFT_1D;
import zero1hd.rhythmbullet.audio.processor.AudioProcessor;
public class AudioAnalyzer implements Runnable {
private Thread thread;
private String threadName = "Audio-Analyzer";
private volatile boolean work = true;
private int windowSize = 1024;
private FloatArray bassSpectralFlux = new FloatArray();
private FloatArray mSpectralFlux = new FloatArray();
private FloatArray umSpectralFlux = new FloatArray();
private FloatArray bassThreshold = new FloatArray();
private FloatArray mThreshold = new FloatArray();
private FloatArray umThreshold = new FloatArray();
private FloatArray bassPrunned = new FloatArray();
private FloatArray mPrunned = new FloatArray();
private FloatArray umPrunned = new FloatArray();
private FloatArray bassPeaks = new FloatArray();
private FloatArray mPeaks = new FloatArray();
private FloatArray umPeaks = new FloatArray();
private float bassMaxValue, mMaxValue, umMaxValue, secondsPerWindow, mAvg, bassAvg, umAvg;
AudioProcessor processor;
private int PUID;
private int progress;
public boolean start(AudioProcessor processor) {
if (thread == null) {
this.processor = processor;
work = true;
thread = new Thread(this, threadName);
thread.start();
return true;
} else {
return false;
}
}
@Override
public void run() {
spectralFluxAnalysis();
thresholdCalculation();
pruneFluxValues();
peakDetection();
}
private void spectralFluxAnalysis() {
progress = 0;
int tasksDone = 0;
long totalTasks = MathUtils.round((float)processor.getSampleFrames()/windowSize);
float[] audioPCM = new float[windowSize];
float[] spectrum = new float[(windowSize/2)+1];
float[] lastSpectrum = new float[(windowSize/2)+1];
int bassBinBegin = 1;
int bassBinEnd = 11;
int mBinBegin = 50;
int mBinEnd = 250;
int umBinBegin = 350;
int umBinEnd = 513;
Gdx.app.debug("Read freq", String.valueOf(processor.getSampleRate()));
Gdx.app.debug("Using following bin ranges", "\nBass freq begin: " + bassBinBegin + "\nBass freq end: " + bassBinEnd + "\nMain freq begin: " + umBinBegin + "\nMain freq end: " + umBinEnd);
Gdx.app.debug("Total tasks", String.valueOf(totalTasks));
FloatFFT_1D fft = new FloatFFT_1D(windowSize);
int seedDigit = 0;
while (processor.readFrames(audioPCM) > 0 && work) {
fft.realForward(audioPCM);
//Building a PUID (Pseudo unique ID)
if (tasksDone == (seedDigit*totalTasks/9)) {
float avg = 0;
for (int frame = 0; frame < spectrum.length; frame++) {
avg += spectrum[frame];
}
avg /= spectrum.length;
if (avg < 0) {
avg *= -1f;
}
PUID +=(int) Math.pow(10, 9-seedDigit) * ((int)(avg*1000f)-(int)(avg*100f)*10);
seedDigit ++;
}
System.arraycopy(spectrum, 0, lastSpectrum, 0, spectrum.length);
System.arraycopy(audioPCM, 0, spectrum, 0, spectrum.length);
float fluxVal;
//bass detection
fluxVal = 0;
for (int i = bassBinBegin; i < bassBinEnd; i++) {
fluxVal += ((spectrum[i] - lastSpectrum[i])) < 0
? 0 : (spectrum[i] - lastSpectrum[i]);
}
bassSpectralFlux.add(fluxVal);
//m detection
fluxVal = 0;
for (int i = mBinBegin; i < mBinEnd; i++) {
fluxVal += ((spectrum[i] - lastSpectrum[i])) < 0
? 0 : (spectrum[i] - lastSpectrum[i]);
}
mSpectralFlux.add(fluxVal);
//um detection
fluxVal = 0;
for (int i = umBinBegin; i < umBinEnd; i++) {
fluxVal += ((spectrum[i] - lastSpectrum[i])) < 0
? 0 : (spectrum[i] - lastSpectrum[i]);
}
umSpectralFlux.add(fluxVal);
tasksDone++;
progress = (int) (100f*tasksDone/totalTasks);
}
if (work) {
Gdx.app.debug("Audio Analyzer", "Done getting spectral flux.");
Gdx.app.debug("Audio Analyzer", "window count: " + bassSpectralFlux.size);
Gdx.app.debug("Audio Analyzer", "USING SEED: " + PUID);
progress = 100;
}
}
private void thresholdCalculation() {
Gdx.app.debug("Audio Analyzer", "beginning threshold calc.");
float bassThresholdMultiplier = 1.5f;
float mThresholdMultiplier = 1.4f;
float umThresholdMultiplier = 1.4f;
int bassThresholdCalcRange = (int) (0.27/(windowSize/processor.getSampleRate()));
int mThresholdCalcRange = (int) (0.4/(windowSize/processor.getSampleRate()));
int umThresholdCalcRange = (int) (0.4/(windowSize/processor.getSampleRate()));
//threshold calculation
for (int i = 0; i < umSpectralFlux.size && work; i++) {
int start = Math.max(0, i - bassThresholdCalcRange/2);
int end = Math.min(umSpectralFlux.size - 1, i + bassThresholdCalcRange/2);
float average = 0;
for (int j = start; j <= end; j++) {
average += bassSpectralFlux.get(j);
}
average /= (end - start);
bassThreshold.add(average * bassThresholdMultiplier);
start = Math.max(0, i - mThresholdCalcRange/2);
end = Math.min(umSpectralFlux.size - 1, i + mThresholdCalcRange/2);
average = 0;
for (int j = start; j <= end; j++) {
average+= mSpectralFlux.get(j);
}
average /= (end - start);
mThreshold.add(average*mThresholdMultiplier);
start = Math.max(0, i - umThresholdCalcRange/2);
end = Math.min(umSpectralFlux.size - 1, i + umThresholdCalcRange/2);
average = 0;
for (int j = start; j <= end; j++) {
average+= umSpectralFlux.get(j);
}
average /= (end - start);
umThreshold.add(average*umThresholdMultiplier);
}
Gdx.app.debug("Audio Analyzer", "Threshold calculated.");
}
private void pruneFluxValues() {
//pruning data
float prunnedCurrentVal;
for (int i = 0; i < umSpectralFlux.size && work; i++) {
prunnedCurrentVal = bassSpectralFlux.get(i) - bassThreshold.get(i);
if (prunnedCurrentVal >= 0) {
bassPrunned.add(prunnedCurrentVal);
} else {
bassPrunned.add(0);
}
prunnedCurrentVal = mSpectralFlux.get(i) - mThreshold.get(i);
if (prunnedCurrentVal >= 0 ) {
mPrunned.add(prunnedCurrentVal);
} else {
mPrunned.add(0);
}
prunnedCurrentVal = umSpectralFlux.get(i) - umThreshold.get(i);
if (prunnedCurrentVal >= 0 ) {
umPrunned.add(prunnedCurrentVal);
} else {
umPrunned.add(0);
}
}
Gdx.app.debug("Audio Analyzer", "Data prunned.");
}
private void peakDetection() {
int lastBeatID = 0;
float bassBeats = 0;
float mBeats = 0;
float umBeats = 0;
float avgSPB = -1f;
for (int i = 0; i < umPrunned.size-1 && work; i++) {
bassPeaks.add((bassPrunned.get(i) > bassPrunned.get(i+1) ? bassPrunned.get(i) : 0f));
if (bassPeaks.get(i) > bassMaxValue) {
bassMaxValue = bassPeaks.get(i);
}
mPeaks.add((mPrunned.get(i) > mPrunned.get(i+1) ? mPrunned.get(i) : 0f));
if (mPeaks.get(i) > mMaxValue) {
mMaxValue = mPeaks.get(i);
}
umPeaks.add((umPrunned.get(i) > umPrunned.get(i+1) ? umPrunned.get(i) : 0f));
if (umPeaks.get(i) > umMaxValue) {
umMaxValue = umPeaks.get(i);
}
if (avgSPB != -1) {
if (bassPeaks.get(i) == 0) {
avgSPB ++;
} else {
lastBeatID = i;
}
} else if (bassPeaks.get(i) != 0) {
avgSPB = 0;
}
if (bassPeaks.get(i) != 0) {
bassAvg += bassPeaks.get(i);
bassBeats++;
}
if (mPeaks.get(i) != 0) {
mAvg += mPeaks.get(i);
mBeats++;
}
if (umPeaks.get(i) != 0) {
umAvg += umPeaks.get(i);
umBeats++;
}
}
secondsPerWindow = windowSize/processor.getSampleRate();
//then we minus one from the beats so it actually works out
avgSPB -= bassPrunned.size-lastBeatID;
avgSPB *= secondsPerWindow;
avgSPB /= bassBeats;
Gdx.app.debug("Audio Analyzer", "Avg SPB: " + avgSPB);
bassAvg /= bassBeats;
mAvg /= mBeats;
umAvg /= umBeats;
Gdx.app.debug("Audio Analyzer", "Avg bass: " + bassAvg);
Gdx.app.debug("Audio Analyzer", "Avg M: " + mAvg);
Gdx.app.debug("Audio Analyzer", "Avg UM: " + umAvg);
}
public int getProgress() {
return progress;
}
public void stop() {
work = false;
}
}

49
core/src/zero1hd/rhythmbullet/audio/analyzer/AudioAnalyzerSection.java Executable file
View File

@@ -0,0 +1,49 @@
package zero1hd.rhythmbullet.audio.analyzer;
import com.badlogic.gdx.utils.FloatArray;
public class AudioAnalyzerSection {
private int lower, upper, thresholdRange;
private float thresholdFactor;
private FloatArray peaks;
private int pUID;
public AudioAnalyzerSection(int lowerBin, int upperBin, float thresholdFactor, int thresholdRange) {
this.upper = upperBin;
this.lower = lowerBin;
this.thresholdRange = thresholdRange;
this.thresholdFactor = thresholdFactor;
}
public void setPUID(int pUID) {
this.pUID = pUID;
}
public void setPeaks(FloatArray peaks) {
this.peaks = peaks;
}
public int getLower() {
return lower;
}
public int getUpper() {
return upper;
}
public float getThresholdFactor() {
return thresholdFactor;
}
public int getThresholdRange() {
return thresholdRange;
}
public FloatArray getPeaks() {
return peaks;
}
public int getPUID() {
return pUID;
}
}

133
core/src/zero1hd/rhythmbullet/audio/analyzer/DynamicAudioAnalyzer.java Executable file
View File

@@ -0,0 +1,133 @@
package zero1hd.rhythmbullet.audio.analyzer;
import java.util.Observable;
import com.badlogic.gdx.utils.FloatArray;
import edu.emory.mathcs.jtransforms.fft.FloatFFT_1D;
import zero1hd.rhythmbullet.audio.processor.AudioProcessor;
public class DynamicAudioAnalyzer extends Observable implements Runnable {
private volatile boolean work = true;
private Thread thread;
private String threadName = "analyzer";
private final int WINDOWLENGTH = 1024;
private AudioProcessor processor;
private AudioAnalyzerSection[] sections;
private FloatArray[] flux;
private FloatArray[] threshold;
private volatile int pUID = 0;
public DynamicAudioAnalyzer(AudioProcessor processor, AudioAnalyzerSection... sections) {
this.sections = sections;
this.processor = processor;
flux = new FloatArray[sections.length];
threshold = new FloatArray[sections.length];
for (int section = 0; section < sections.length; section++) {
flux[section] = new FloatArray();
threshold[section] = new FloatArray();
}
}
@Override
public void run() {
calculateSpectralFlux();
calculateThreshold();
calculatePeaks();
}
public void start() {
if (thread == null) {
thread = new Thread(this, threadName);
thread.setDaemon(true);
} else {
throw new IllegalStateException("Cannot have two analyzer threads.");
}
}
public void stop() {
work = false;
}
private void calculateSpectralFlux() {
float[] audioPCM = new float[WINDOWLENGTH];
float[] spectrum = new float[(WINDOWLENGTH/2)+1];
float[] lastSpectrum = new float[spectrum.length];
FloatFFT_1D fft = new FloatFFT_1D(WINDOWLENGTH);
int windowsComplete = 0;
int seedCurrentDigit = 0;
int totalWindows = (int) processor.getSampleFrames()/WINDOWLENGTH;
while (processor.readFrames(audioPCM) > 0 && work) {
fft.realForward(audioPCM);
//Building a PUID (Pseudo unique ID)
if (windowsComplete == (seedCurrentDigit*totalWindows/9)) {
float avg = 0;
for (int frame = 0; frame < spectrum.length; frame++) {
avg += spectrum[frame];
}
avg /= spectrum.length;
if (avg < 0) {
avg *= -1f;
}
pUID +=(int) Math.pow(10, 9-seedCurrentDigit) * ((int)(avg*1000f)-(int)(avg*100f)*10);
seedCurrentDigit ++;
}
System.arraycopy(spectrum, 0, lastSpectrum, 0, spectrum.length);
System.arraycopy(audioPCM, 0, spectrum, 0, spectrum.length);
for (int section = 0; section < sections.length; section++) {
float currentFlux = 0;
for (int bin = sections[section].getLower(); bin < sections[section].getUpper(); section++) {
currentFlux += Math.max(0f, spectrum[bin] - lastSpectrum[bin]);
}
flux[section].add(currentFlux);
}
windowsComplete++;
}
for (int section = 0; section < sections.length; section++) {
sections[section].setPUID(pUID);
}
}
private void calculateThreshold() {
for (int section = 0; section < sections.length && work; section++) {
FloatArray fluxArray = flux[section];
for (int bin = 0; bin < fluxArray.size; fluxArray.size++) {
int range = sections[section].getThresholdRange();
int start = Math.max(0, bin - range);
int end = Math.min(fluxArray.size, bin + range);
float average = 0;
for (int pos = start; pos < end; pos++) {
average += fluxArray.get(pos);
}
average /= (end - start);
threshold[section].add(average);
}
}
}
private void calculatePeaks() {
for (int section = 0; section < sections.length && work; section++) {
FloatArray peaks = new FloatArray();
for (int bin = 0; bin < threshold[section].size -1; bin++) {
float prunedFlux = flux[section].get(bin) - threshold[section].get(bin);
float prunedNextFlux = flux[section].get(bin + 1) - threshold[section].get(bin + 1);
peaks.add((prunedFlux > prunedNextFlux) ? prunedFlux : 0);
}
sections[section].setPeaks(peaks);
}
setChanged();
notifyObservers();
}
}

1
core/src/zero1hd/rhythmbullet/audio/metadata/AudioMetadata.java
View File

@@ -13,6 +13,7 @@ import com.badlogic.gdx.utils.Disposable;
public interface AudioMetadata extends Disposable {
/**
* Load the album art data in to memory.
* Will not load if already loaded.
*/
public void loadAlbumCover();

24
core/src/zero1hd/rhythmbullet/audio/metadata/MP3Metadata.java
View File

@@ -33,8 +33,6 @@ public class MP3Metadata implements AudioMetadata {
MP3File mp3file = (MP3File) AudioFileIO.read(fileHandle.file());
ID3v23Tag tag;
tag = (ID3v23Tag) mp3file.getTagAndConvertOrCreateAndSetDefault();
length = mp3file.getAudioHeader().getTrackLength();
SimpleDateFormat f = new SimpleDateFormat("m:ss");
@@ -53,17 +51,19 @@ public class MP3Metadata implements AudioMetadata {
@Override
public void loadAlbumCover() {
MP3File mp3file;
try {
mp3file = (MP3File) AudioFileIO.read(fileHandle.file());
Artwork art = mp3file.getTag().getFirstArtwork();
if (art != null) {
byte[] imageData = art.getBinaryData();
pixmap = new Pixmap(imageData, 0, imageData.length);
if (pixmap == null) {
try {
MP3File mp3file;
mp3file = (MP3File) AudioFileIO.read(fileHandle.file());
Artwork art = mp3file.getTag().getFirstArtwork();
if (art != null) {
byte[] imageData = art.getBinaryData();
pixmap = new Pixmap(imageData, 0, imageData.length);
}
} catch (CannotReadException | IOException | TagException | ReadOnlyFileException | InvalidAudioFrameException e) {
e.printStackTrace();
}
} catch (CannotReadException | IOException | TagException | ReadOnlyFileException | InvalidAudioFrameException e) {
e.printStackTrace();
}
}

21
core/src/zero1hd/rhythmbullet/audio/metadata/WAVMetadata.java
View File

@@ -49,18 +49,19 @@ public class WAVMetadata implements AudioMetadata {
@Override
public void loadAlbumCover() {
try {
AudioFile wav = AudioFileIO.read(fileHandle.file());
Artwork art = wav.getTag().getFirstArtwork();
if (art != null) {
byte[] imageData = art.getBinaryData();
pixmap = new Pixmap(imageData, 0, imageData.length);
if (pixmap == null) {
try {
AudioFile wav = AudioFileIO.read(fileHandle.file());
Artwork art = wav.getTag().getFirstArtwork();
if (art != null) {
byte[] imageData = art.getBinaryData();
pixmap = new Pixmap(imageData, 0, imageData.length);
}
} catch (CannotReadException | IOException | TagException | ReadOnlyFileException | InvalidAudioFrameException e) {
e.printStackTrace();
}
} catch (CannotReadException | IOException | TagException | ReadOnlyFileException | InvalidAudioFrameException e) {
e.printStackTrace();
}
}
@Override
public void unloadAlbumCover() {