Serie 01

Música Visual ( 2021 )



Testing live visuals  - Processing y Premier

← take me back




const micSampleRate = 44100;
const freqDefinition = 8192;
const freqSteps = micSampleRate/freqDefinition;
const hitKeyThreshold = 80;//minimum level to hit the piano key

const minFreqHz = 130;//C3
const maxFreqHz = 2903;//C7
const minFreq = Math.floor target="_blank">Math.floor(minFreqHz/2/1.445);//2/1.445 is a magic number to convert Hz to MIC frequency, dont know why...
const maxFreq = Math.floor(maxFreqHz/2/1.445);//2/1.445 is a magic number to convert Hz to MIC frequency, dont know why...

const scaleIndexToNote = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"];
let mic, fft, spectrum;
let historygram;

let piano = [];
let pressedKey = -1;
let keyHeight;

let state, nextState, startTime, duration;
const STATES = {CALIBRATING: "CALIBRATING", RECORDING: "RECORDING", PLAYING: "PLAYING"};

const calibrationDuration = 5 * 1000;//milliseconds
let calibrateSum = [];
let avarageSpectrum;//array to threshold the noise from the microphone

const recordDuration = 20 * 1000;//milliseconds
let recordedSequence = [];

function setup() {
    createCanvas(windowWidth, windowHeight);
   
    historygram = createGraphics(maxFreq-minFreq, height/2);
   
    mic = new p5.AudioIn();
    mic.start();
   
    fft = new p5.FFT(0.0, freqDefinition);
    mic.connect(fft);
   
    //setup the Piano array
    let totalKeys = freqToMidi(maxFreqHz) - freqToMidi(minFreqHz);
    let keyX = width / 2;
    let keyWidth = width / 2 / (totalKeys/12*7);
    keyHeight = width / 1920 * 100;
    for(let midi = freqToMidi(minFreqHz); midi < freqToMidi(maxFreqHz); midi++){
        let index = midi % 12;
        //black key indices
        //c c# d d# e f f# g g# a a# b
        if(index == 1 || index == 3 || index == 6 || index == 8 || index == 10){
            createKey(midi+1, "WHITE", keyX, keyWidth);
            createKey(midi, "BLACK", keyX, keyWidth);
            midi++;
        }else{
            createKey(midi, "WHITE", keyX, keyWidth);
        }
        keyX += keyWidth;
    }
   
    avarageSpectrum = new Array(maxFreq-minFreq);
    avarageSpectrum.fill(0);
   
    textAlign(CENTER, CENTER);
    textSize(24);
   
    setState(STATES.CALIBRATING);
}

function draw() {
    background(0);
   
    spectrum = fft.analyze();
   
    //CALCULATE AVARAGE NOISE FROM MICROPHONE
    if(state == STATES.CALIBRATING){
        calibrateSum.push(spectrum.slice());
        for (let i = minFreq; i < maxFreq; i++) {
            let index = i - minFreq;
            let sum = 0;
            for (let j = 0; j < calibrateSum.length target="_blank">calibrateSum.length; j++) {
                sum += calibrateSum[j][i] * 1.1;
            }
            avarageSpectrum[index] = sum / calibrateSum.length;
        }
    }
   
    //DRAW MIC SPECTRUM
    stroke(255);
    fill(255, 150);
    beginShape();
    vertex(0, height / 2);
    for (let i = minFreq; i < maxFreq; i++) {
        let x = (i - minFreq) / (maxFreq - minFreq - 1) * width / 2;
let y = (1 - spectrum[i] / 255) * height / 2;
vertex(x, y);
    }
    vertex(width / 2, height / 2);
    endShape(CLOSE);
   
    //DRAW AVARAGE SPECTRUM
    stroke(0);
    fill(0, 150);
    beginShape();
    vertex(0, height / 2);
    for (let i = minFreq; i < maxFreq; i++) {
        let index = i - minFreq;
        let x = index / (maxFreq - minFreq - 1) * width / 2;
let y = (1 - avarageSpectrum[index] / 255) * height / 2;
vertex(x, y);
    }
    vertex(width / 2, height / 2);
    endShape(CLOSE);
   
    //DRAW HISTORYGRAM
    historygram.image(historygram, 0, 2);
    for (let i = minFreq; i < maxFreq; i++) {
        let index = i - minFreq;
let intensity = (spectrum[i] - avarageSpectrum[index]) * 3;
        historygram.stroke(intensity);
        historygram.point(index, 0);
    }
    fill(255);
    image(historygram, 0, height / 2, width / 2, height / 2);
   
    //FIND THE HIGHEST LEVEL OF AUDIO FREQUENCY
    let selectedFreq = -1;
    let selectedDist = 0;
    for (let i = minFreq; i < maxFreq; i++) {
        let index = i - minFreq;
        if(spectrum[i] - avarageSpectrum[index] > selectedDist){
            selectedDist = spectrum[i] - avarageSpectrum[index];
            selectedFreq = i;
        }
    }

    //DRAW HIGHEST LEVEL
    const magicNumber = 1.091;//I had to multiply to this magic number, to be able to get the right Hz, I dont know why...
    let freq = selectedFreq * freqSteps/2 * magicNumber;
    let tempPressedKey = freqToMidi(freq);
    if(selectedDist > hitKeyThreshold){
        let xSelected = (selectedFreq-minFreq) / (maxFreq-minFreq) * width / 2;
        stroke(255, 255, 0);
        line(xSelected, 0, xSelected, height / 2)
        noStroke();
        fill(255, 255, 0);
        text(freq.toFixed(2) + "Hz", xSelected, 100);
        let note = scaleIndexToNote[tempPressedKey % 12] + (floor(tempPressedKey/12)-1);
        text(note, xSelected, 130);
    }
   
    if(state == STATES.RECORDING){
        //RECORD
        if(selectedDist > hitKeyThreshold && piano[tempPressedKey]){
            if(pressedKey != tempPressedKey) {
                //PRESS KEY
                pressedKey = tempPressedKey;
                recordedSequence.push({
                    key: pressedKey,
                    start: new Date().getTime() - startTime,
                    duration: 0
                });
            }else{
                //still pressing the same key
                let lastIndex = recordedSequence.length-1 target="_blank">recordedSequence.length-1 target="_blank">recordedSequence.length-1 target="_blank">recordedSequence.length-1;
                recordedSequence[lastIndex].duration = new Date().getTime() - startTime - recordedSequence[lastIndex].start;
            }
        }else{
            //RELEASE KEY
            if(pressedKey!=-1){
                let lastIndex = recordedSequence.length-1;
                if(lastIndex >= 0)
                    recordedSequence[lastIndex].duration = new Date().getTime() - startTime - recordedSequence[lastIndex].start;
            }
            pressedKey = -1;
        }
   
        //DRAW RECORDING SEQUENCE
        let scrollY = (new Date().getTime() - startTime)/10;
        noStroke();
        fill(255);
        for(let i = recordedSequence.length-1; i >= 0; i--){
            let record = recordedSequence[i];
            let pianoKey = piano[record.key target="_blank">record.key target="_blank">record.key];
            let x = pianoKey.x;
            let y = keyHeight + scrollY - record.start target="_blank">record.start/10 target="_blank">record.start/10 target="_blank">record.start/10;
            let w = pianoKey.w;
            let h = -record.duration/10;
            rect(x, y, w, h);
        }
    }
   
    //DRAW PLAYING SEQUENCE
    if(state == STATES.PLAYING){
        let time = (new Date().getTime() - startTime);
        let scrollY = time/10;
        pressedKey = -1;
        noStroke();
        fill(255);
        for(let i = recordedSequence.length-1; i >= 0; i--){
            let record = recordedSequence[i];
            let pianoKey = piano[record.key];
            let x = pianoKey.x;
            let y = keyHeight - scrollY + record.start/10;
            let w = pianoKey.w;
            let h = record.duration/10;
            rect(x, y, w, h);
           
            //PLAY SOUND
            if(time > record.start && time < record.start + record.duration){
                pressedKey = record.key;
                if(!record.played target="_blank">record.played target="_blank">record.played){
                    record.played = true;
                    pianoKey.osc.start target="_blank">pianoKey.osc.start();
                    pianoKey.envelope.play(pianoKey.osc, 0, record.duration/1000);
                }
            }else{
                record.played = false;
            }
        }
    }
   
    //DRAW PIANO
    noStroke();
    for(let midi = freqToMidi(minFreqHz); midi < freqToMidi(maxFreqHz); midi++){
        if(piano[midi].type == "BLACK"){
            //first draw the next white key
            if(pressedKey == midi+1){
                fill(255, 255, 0);
            }else{
                fill(255);
            }
            rect(piano[midi+1].x, 0, piano[midi+1].w, piano[midi+1].h);
           
            //then draw the black key
            if(pressedKey == midi){
                fill(255, 255, 0);
            }else{
                fill(0);
            }
            rect(piano[midi].x, 0, piano[midi].w, piano[midi].h);
            midi++;
        }else{
            //white key
            if(pressedKey == midi){
                fill(255, 255, 0);
            }else{
                fill(255);
            }
            rect(piano[midi].x, 0, piano[midi].w, piano[midi].h);
        }
    }
   
    //CHECK NEXT STATE
    let time = new Date().getTime();
    let remaining = floor((duration - (time - startTime))/1000);
    fill(255, 255, 0);
    text(state + ": " + remaining + "s", width / 4, height / 2 - 30);
    if(time - startTime > duration){
        setState(nextState);
    }
}

function setState(s){
    state = s;
    startTime = new Date().getTime();
    switch (state) {
        case STATES.CALIBRATING:
            duration = calibrationDuration;
            nextState = STATES.RECORDING;
            break;
        case STATES.RECORDING:
            duration = recordDuration;
            nextState = STATES.PLAYING;
            break;
        case STATES.PLAYING:
            break;
    }
}

function createKey(midi, type, x, w){
    let h = keyHeight;
    if(type == "BLACK"){
        x -= w*0.3;
        w /= 1.5;
        h /= 2;
    }
   
    let envelope = new p5.Envelope();
    envelope.setRange(0.07, 0);//attackLevel, releaseLevel
    envelope.setADSR(0.001, 0.8, 0.1, 0.5);//attackTime, decayTime, sustainRatio, releaseTime
    let osc = new p5.SinOsc();
    osc.freq(midiToFreq(midi));
    piano[midi] = {
        envelope: envelope,
        osc: osc,
        midi: midi,
        freq: midiToFreq(midi),
        x: x,
        w: w,
        h: h,
        type: type,
        played: false
    };
}


Mark