Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use new scratch-audio APIs in music extension #1258

Merged
merged 1 commit into from
Jun 21, 2018
Merged

Use new scratch-audio APIs in music extension #1258

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 87 additions & 64 deletions src/extensions/scratch3_music/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,18 +52,25 @@ class Scratch3MusicBlocks {
this._concurrencyCounter = 0;

/**
* An array of audio buffers, one for each drum sound.
* An array of sound players, one for each drum sound.
* @type {Array}
* @private
*/
this._drumBuffers = [];
this._drumPlayers = [];

/**
* An array of arrays of audio buffers. Each instrument has one or more audio buffers.
* An array of arrays of sound players. Each instrument has one or more audio players.
* @type {Array[]}
* @private
*/
this._instrumentBufferArrays = [];
this._instrumentPlayerArrays = [];

/**
* An array of arrays of sound players. Each instrument mya have an audio player for each playable note.
* @type {Array[]}
* @private
*/
this._instrumentPlayerNoteArrays = [];

/**
* An array of audio bufferSourceNodes. Each time you play an instrument or drum sound,
Expand All @@ -87,14 +94,15 @@ class Scratch3MusicBlocks {
const loadingPromises = [];
this.DRUM_INFO.forEach((drumInfo, index) => {
const filePath = `drums/${drumInfo.fileName}`;
const promise = this._storeSound(filePath, index, this._drumBuffers);
const promise = this._storeSound(filePath, index, this._drumPlayers);
loadingPromises.push(promise);
});
this.INSTRUMENT_INFO.forEach((instrumentInfo, instrumentIndex) => {
this._instrumentBufferArrays[instrumentIndex] = [];
this._instrumentPlayerArrays[instrumentIndex] = [];
this._instrumentPlayerNoteArrays[instrumentIndex] = [];
instrumentInfo.samples.forEach((sample, noteIndex) => {
const filePath = `instruments/${instrumentInfo.dirName}/${sample}`;
const promise = this._storeSound(filePath, noteIndex, this._instrumentBufferArrays[instrumentIndex]);
const promise = this._storeSound(filePath, noteIndex, this._instrumentPlayerArrays[instrumentIndex]);
loadingPromises.push(promise);
});
});
Expand All @@ -104,22 +112,22 @@ class Scratch3MusicBlocks {
}

/**
* Decode a sound and store the buffer in an array.
* Decode a sound and store the player in an array.
* @param {string} filePath - the audio file name.
* @param {number} index - the index at which to store the audio buffer.
* @param {array} bufferArray - the array of buffers in which to store it.
* @param {number} index - the index at which to store the audio player.
* @param {array} playerArray - the array of players in which to store it.
* @return {Promise} - a promise which will resolve once the sound has been stored.
*/
_storeSound (filePath, index, bufferArray) {
_storeSound (filePath, index, playerArray) {
const fullPath = `${filePath}.mp3`;

if (!assetData[fullPath]) return;

// The sound buffer has already been downloaded via the manifest file required above.
// The sound player has already been downloaded via the manifest file required above.
const soundBuffer = assetData[fullPath];

return this._decodeSound(soundBuffer).then(buffer => {
bufferArray[index] = buffer;
return this._decodeSound(soundBuffer).then(player => {
playerArray[index] = player;
});
}

Expand All @@ -129,24 +137,14 @@ class Scratch3MusicBlocks {
* @return {Promise} - a promise which will resolve once the sound has decoded.
*/
_decodeSound (soundBuffer) {
const context = this.runtime.audioEngine && this.runtime.audioEngine.audioContext;
const engine = this.runtime.audioEngine;

if (!context) {
if (!engine) {
return Promise.reject(new Error('No Audio Context Detected'));
}

// Check for newer promise-based API
if (context.decodeAudioData.length === 1) {
return context.decodeAudioData(soundBuffer);
} else { // eslint-disable-line no-else-return
// Fall back to callback API
return new Promise((resolve, reject) =>
context.decodeAudioData(soundBuffer,
buffer => resolve(buffer),
error => reject(error)
)
);
}
return engine.decodeSoundPlayer({data: {buffer: soundBuffer}});
}

/**
Expand Down Expand Up @@ -774,26 +772,34 @@ class Scratch3MusicBlocks {
*/
_playDrumNum (util, drumNum) {
if (util.runtime.audioEngine === null) return;
if (util.target.audioPlayer === null) return;
if (util.target.sprite.soundBank === null) return;
// If we're playing too many sounds, do not play the drum sound.
if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) {
return;
}
const outputNode = util.target.audioPlayer.getInputNode();
const context = util.runtime.audioEngine.audioContext;
const bufferSource = context.createBufferSource();
bufferSource.buffer = this._drumBuffers[drumNum];
bufferSource.connect(outputNode);
bufferSource.start();

const bufferSourceIndex = this._bufferSources.length;
this._bufferSources.push(bufferSource);
const player = this._drumPlayers[drumNum];

if (typeof player === 'undefined') return;

if (player.isPlaying) {
// Take the internal player state and create a new player with it.
// `.play` does this internally but then instructs the sound to
// stop.
player.take();
}

const engine = util.runtime.audioEngine;
const chain = engine.createEffectChain();
chain.setEffectsFromTarget(util.target);
player.connect(chain);

this._concurrencyCounter++;
bufferSource.onended = () => {
player.once('stop', () => {
this._concurrencyCounter--;
delete this._bufferSources[bufferSourceIndex];
};
});

player.play();
}

/**
Expand Down Expand Up @@ -852,7 +858,7 @@ class Scratch3MusicBlocks {
*/
_playNote (util, note, durationSec) {
if (util.runtime.audioEngine === null) return;
if (util.target.audioPlayer === null) return;
if (util.target.sprite.soundBank === null) return;

// If we're playing too many sounds, do not play the note.
if (this._concurrencyCounter > Scratch3MusicBlocks.CONCURRENCY_LIMIT) {
Expand All @@ -867,28 +873,37 @@ class Scratch3MusicBlocks {
const sampleIndex = this._selectSampleIndexForNote(note, sampleArray);

// If the audio sample has not loaded yet, bail out
if (typeof this._instrumentBufferArrays[inst] === 'undefined') return;
if (typeof this._instrumentBufferArrays[inst][sampleIndex] === 'undefined') return;
if (typeof this._instrumentPlayerArrays[inst] === 'undefined') return;
if (typeof this._instrumentPlayerArrays[inst][sampleIndex] === 'undefined') return;

// Create the audio buffer to play the note, and set its pitch
const context = util.runtime.audioEngine.audioContext;
const bufferSource = context.createBufferSource();
// Fetch the sound player to play the note.
const engine = util.runtime.audioEngine;

const bufferSourceIndex = this._bufferSources.length;
this._bufferSources.push(bufferSource);
if (!this._instrumentPlayerNoteArrays[inst][note]) {
this._instrumentPlayerNoteArrays[inst][note] = this._instrumentPlayerArrays[inst][sampleIndex].take();
}

bufferSource.buffer = this._instrumentBufferArrays[inst][sampleIndex];
const sampleNote = sampleArray[sampleIndex];
bufferSource.playbackRate.value = this._ratioForPitchInterval(note - sampleNote);
const player = this._instrumentPlayerNoteArrays[inst][note];

// Create a gain node for this note, and connect it to the sprite's audioPlayer.
const gainNode = context.createGain();
bufferSource.connect(gainNode);
const outputNode = util.target.audioPlayer.getInputNode();
gainNode.connect(outputNode);
if (player.isPlaying) {
// Take the internal player state and create a new player with it.
// `.play` does this internally but then instructs the sound to
// stop.
player.take();
}

// Start playing the note
bufferSource.start();
const chain = engine.createEffectChain();
chain.setEffectsFromTarget(util.target);

// Set its pitch.
const sampleNote = sampleArray[sampleIndex];
const notePitchInterval = this._ratioForPitchInterval(note - sampleNote);

// Create a gain node for this note, and connect it to the sprite's
// simulated effectChain.
const context = engine.audioContext;
const releaseGain = context.createGain();
releaseGain.connect(chain.getInputNode());

// Schedule the release of the note, ramping its gain down to zero,
// and then stopping the sound.
Expand All @@ -898,16 +913,24 @@ class Scratch3MusicBlocks {
}
const releaseStart = context.currentTime + durationSec;
const releaseEnd = releaseStart + releaseDuration;
gainNode.gain.setValueAtTime(1, releaseStart);
gainNode.gain.linearRampToValueAtTime(0.0001, releaseEnd);
bufferSource.stop(releaseEnd);
releaseGain.gain.setValueAtTime(1, releaseStart);
releaseGain.gain.linearRampToValueAtTime(0.0001, releaseEnd);

// Update the concurrency counter
this._concurrencyCounter++;
bufferSource.onended = () => {
player.once('stop', () => {
this._concurrencyCounter--;
delete this._bufferSources[bufferSourceIndex];
};
});

// Start playing the note
player.play();
// Connect the player to the gain node.
player.connect({getInputNode () {
return releaseGain;
}});
// Set playback now after play creates the outputNode.
player.outputNode.playbackRate.value = notePitchInterval;
// Schedule playback to stop.
player.outputNode.stop(releaseEnd);
}

/**
Expand Down