diff --git a/src/background.ts b/src/background.ts index cf03a40f9d..24a172ba15 100644 --- a/src/background.ts +++ b/src/background.ts @@ -861,7 +861,7 @@ async function createWindow() { win.on("close", (event) => { if (!willQuit) { event.preventDefault(); - ipcMainSend(win, "CHECK_EDITED_AND_NOT_SAVE"); + ipcMainSend(win, "PROCESS_BEFORE_QUITTING"); return; } }); @@ -1214,7 +1214,7 @@ app.on("window-all-closed", () => { app.on("before-quit", (event) => { if (!willQuit) { event.preventDefault(); - ipcMainSend(win, "CHECK_EDITED_AND_NOT_SAVE"); + ipcMainSend(win, "PROCESS_BEFORE_QUITTING"); return; } diff --git a/src/components/AcceptTermsDialog.vue b/src/components/AcceptTermsDialog.vue index 19180e5402..3188145ac1 100644 --- a/src/components/AcceptTermsDialog.vue +++ b/src/components/AcceptTermsDialog.vue @@ -87,7 +87,9 @@ export default defineComponent({ store.dispatch("SET_ACCEPT_TERMS", { acceptTerms: acceptTerms ? "Accepted" : "Rejected", }); - !acceptTerms ? store.dispatch("CHECK_EDITED_AND_NOT_SAVE") : undefined; + if (!acceptTerms) { + store.dispatch("PROCESS_BEFORE_QUITTING"); + } modelValueComputed.value = false; }; diff --git a/src/components/MinMaxCloseButtons.vue b/src/components/MinMaxCloseButtons.vue index 5a68e06f47..5975336aa0 100644 --- a/src/components/MinMaxCloseButtons.vue +++ b/src/components/MinMaxCloseButtons.vue @@ -95,8 +95,8 @@ export default defineComponent({ setup() { const store = useStore(); - const closeWindow = async () => { - store.dispatch("CHECK_EDITED_AND_NOT_SAVE"); + const closeWindow = () => { + store.dispatch("PROCESS_BEFORE_QUITTING"); }; const minimizeWindow = () => window.electron.minimizeWindow(); const maximizeWindow = () => window.electron.maximizeWindow(); diff --git a/src/components/Sing/MenuBar.vue b/src/components/Sing/MenuBar.vue index 1719a29c40..0b1f83fcf6 100644 --- a/src/components/Sing/MenuBar.vue +++ b/src/components/Sing/MenuBar.vue @@ -11,6 +11,8 @@ root.type === 'button' ? (subMenuOpenFlags[index] = false) : undefined " /> + + @@ -18,6 +20,7 @@ import { defineComponent, ref, computed, ComputedRef } from "vue"; import { useStore } from "@/store"; import MenuButton from "@/components/MenuButton.vue"; +import MinMaxCloseButtons from "@/components/MinMaxCloseButtons.vue"; import { HotkeyAction, HotkeyReturnType } from "@/type/preload"; import { setHotkeyFunctions } from "@/store/setting"; @@ -57,6 +60,7 @@ export default defineComponent({ components: { MenuButton, + MinMaxCloseButtons, }, setup() { diff --git a/src/helpers/singHelper.ts b/src/helpers/singHelper.ts index f55e41c3e7..7c089fc6e8 100644 --- a/src/helpers/singHelper.ts +++ b/src/helpers/singHelper.ts @@ -66,3 +66,7 @@ export function round(value: number, digits: number) { const powerOf10 = 10 ** digits; return Math.round(value * powerOf10) / powerOf10; } + +export function midiToFrequency(midi: number) { + return 440 * 2 ** ((midi - 69) / 12); +} diff --git a/src/infrastructures/AudioRenderer.ts b/src/infrastructures/AudioRenderer.ts index 3955c50950..9d05969403 100644 --- a/src/infrastructures/AudioRenderer.ts +++ b/src/infrastructures/AudioRenderer.ts @@ -273,6 +273,45 @@ export class OfflineTransport implements BaseTransport { } } +export type AudioEvent = { + readonly time: number; + readonly buffer: AudioBuffer; +}; + +export class AudioSequence implements SoundSequence { + private readonly audioPlayer: AudioPlayer; + private readonly audioEvents: AudioEvent[]; + + constructor(audioPlayer: AudioPlayer, audioEvents: AudioEvent[]) { + this.audioPlayer = audioPlayer; + this.audioEvents = audioEvents; + } + + // スケジュール可能なイベントを生成する + generateEvents(startTime: number) { + return this.audioEvents + .sort((a, b) => a.time - b.time) + .filter((value) => { + const audioEndTime = value.time + value.buffer.duration; + return audioEndTime > startTime; + }) + .map((value): SchedulableEvent => { + const offset = Math.max(startTime - value.time, 0); + return { + time: Math.max(value.time, startTime), + schedule: (contextTime: number) => { + this.audioPlayer.play(contextTime, offset, value.buffer); + }, + }; + }); + } + + // シーケンスの停止をスケジュールする + scheduleStop(contextTime: number) { + this.audioPlayer.allStop(contextTime); + } +} + export interface Instrument { noteOn(contextTime: number, midi: number): void; noteOff(contextTime: number, midi: number): void; @@ -317,12 +356,101 @@ export class NoteSequence implements SoundSequence { .sort((a, b) => a.time - b.time); } - // シーケンス(楽器)の停止をスケジュールする + // シーケンスの停止をスケジュールする scheduleStop(contextTime: number) { this.instrument.allSoundOff(contextTime); } } +class AudioPlayerVoice { + private readonly audioBufferSourceNode: AudioBufferSourceNode; + private readonly buffer: AudioBuffer; + + private _isStopped = false; + private stopContextTime?: number; + + get isStopped() { + return this._isStopped; + } + + constructor(audioContext: BaseAudioContext, buffer: AudioBuffer) { + this.audioBufferSourceNode = audioContext.createBufferSource(); + this.audioBufferSourceNode.buffer = buffer; + this.audioBufferSourceNode.onended = () => { + this._isStopped = true; + }; + this.buffer = buffer; + } + + connect(destination: AudioNode) { + this.audioBufferSourceNode.connect(destination); + } + + play(contextTime: number, offset: number) { + this.audioBufferSourceNode.start(contextTime, offset); + this.stopContextTime = contextTime + this.buffer.duration; + } + + stop(contextTime?: number) { + if (this.stopContextTime === undefined) { + throw new Error("Not started."); + } + if (contextTime === undefined || contextTime < this.stopContextTime) { + this.audioBufferSourceNode.stop(contextTime); + this.stopContextTime = contextTime ?? 0; + } + } +} + +export type AudioPlayerOptions = { + readonly volume: number; +}; + +export class AudioPlayer { + private readonly audioContext: BaseAudioContext; + private readonly gainNode: GainNode; + + private voices: AudioPlayerVoice[] = []; + + constructor(context: Context, options: AudioPlayerOptions = { volume: 1.0 }) { + this.audioContext = context.audioContext; + + this.gainNode = this.audioContext.createGain(); + this.gainNode.gain.value = options.volume; + } + + connect(destination: AudioNode) { + this.gainNode.connect(destination); + } + + disconnect() { + this.gainNode.disconnect(); + } + + play(contextTime: number, offset: number, buffer: AudioBuffer) { + const voice = new AudioPlayerVoice(this.audioContext, buffer); + this.voices = this.voices.filter((value) => { + return !value.isStopped; + }); + this.voices.push(voice); + voice.connect(this.gainNode); + voice.play(contextTime, offset); + } + + allStop(contextTime?: number) { + if (contextTime === undefined) { + this.voices.forEach((value) => { + value.stop(); + }); + this.voices = []; + } else { + this.voices.forEach((value) => { + value.stop(contextTime); + }); + } + } +} + export type Envelope = { readonly attack: number; readonly decay: number; @@ -371,8 +499,8 @@ class SynthVoice { return 440 * 2 ** ((midi - 69) / 12); } - connect(inputNode: AudioNode) { - this.gainNode.connect(inputNode); + connect(destination: AudioNode) { + this.gainNode.connect(destination); } noteOn(contextTime: number) { @@ -532,6 +660,10 @@ export class AudioRenderer { }; } + get audioContext() { + return this.onlineContext.audioContext; + } + get transport() { return this.onlineContext.transport; } @@ -542,7 +674,14 @@ export class AudioRenderer { this.onlineContext = { audioContext, transport }; } - renderToBuffer( + async createAudioBuffer(blob: Blob) { + const audioContext = this.onlineContext.audioContext; + const arrayBuffer = await blob.arrayBuffer(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + return audioBuffer; + } + + async renderToBuffer( sampleRate: number, startTime: number, duration: number, @@ -558,7 +697,8 @@ export class AudioRenderer { callback({ audioContext, transport }); transport.scheduleEvents(startTime, duration); - return audioContext.startRendering(); + const audioBuffer = await audioContext.startRendering(); + return audioBuffer; } dispose() { diff --git a/src/plugins/ipcMessageReceiverPlugin.ts b/src/plugins/ipcMessageReceiverPlugin.ts index 296e51cee2..2c66dea8f3 100644 --- a/src/plugins/ipcMessageReceiverPlugin.ts +++ b/src/plugins/ipcMessageReceiverPlugin.ts @@ -44,8 +44,8 @@ export const ipcMessageReceiver: Plugin = { options.store.dispatch("DETECT_LEAVE_FULLSCREEN") ); - window.electron.onReceivedIPCMsg("CHECK_EDITED_AND_NOT_SAVE", () => { - options.store.dispatch("CHECK_EDITED_AND_NOT_SAVE"); + window.electron.onReceivedIPCMsg("PROCESS_BEFORE_QUITTING", () => { + options.store.dispatch("PROCESS_BEFORE_QUITTING"); }); window.electron.onReceivedIPCMsg( diff --git a/src/store/singing.ts b/src/store/singing.ts index 199f5228c2..a5b66d7b65 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -1,5 +1,8 @@ import { + AudioEvent, + AudioPlayer, AudioRenderer, + AudioSequence, Context, NoteEvent, NoteSequence, @@ -21,7 +24,12 @@ import { createPartialStore } from "./vuex"; import { createUILockAction } from "./ui"; import { WriteFileErrorResult } from "@/type/preload"; import { Midi } from "@tonejs/midi"; -import { getDoremiFromMidi, round } from "@/helpers/singHelper"; +import { + getDoremiFromMidi, + midiToFrequency, + round, +} from "@/helpers/singHelper"; +import { AudioQuery } from "@/openapi"; const ticksToSecondsForConstantBpm = ( resolution: number, @@ -100,16 +108,91 @@ const secondsToTicks = ( ); }; +const generateNoteEvents = ( + resolution: number, + tempos: Tempo[], + notes: Note[] +) => { + return notes.map((value): NoteEvent => { + const noteOnPos = value.position; + const noteOffPos = value.position + value.duration; + return { + midi: value.midi, + noteOnTime: ticksToSeconds(resolution, tempos, noteOnPos), + noteOffTime: ticksToSeconds(resolution, tempos, noteOffPos), + }; + }); +}; + +const copyScore = (score: Score): Score => { + return { + resolution: score.resolution, + tempos: score.tempos.map((value) => ({ ...value })), + timeSignatures: score.timeSignatures.map((value) => ({ ...value })), + notes: score.notes.map((value) => ({ ...value })), + }; +}; + +const _generateHash = async (obj: T) => { + const textEncoder = new TextEncoder(); + const data = textEncoder.encode(JSON.stringify(obj)); + const digest = await crypto.subtle.digest("SHA-256", data); + return Array.from(new Uint8Array(digest)) + .map((v) => v.toString(16).padStart(2, "0")) + .join(""); +}; + +const createPromiseThatResolvesWhen = ( + condition: () => boolean, + interval = 200 +) => { + return new Promise((resolve) => { + const checkCondition = () => { + if (condition()) { + resolve(); + } + window.setTimeout(checkCondition, interval); + }; + checkCondition(); + }); +}; + +type Singer = { + readonly engineId: string; + readonly styleId: number; +}; + +type Phrase = { + readonly singer: Singer | undefined; + readonly score: Score; + // renderingが進むに連れてデータが代入されていく + query?: AudioQuery; + queryHash?: string; // queryの変更を検知するためのハッシュ + buffer?: AudioBuffer; + startTime?: number; +}; + +const generateSingerAndScoreHash = async (obj: { + singer: Singer | undefined; + score: Score; +}) => { + return _generateHash(obj); +}; + +const generateAudioQueryHash = async (obj: AudioQuery) => { + return _generateHash(obj); +}; + type ChannelOptions = { readonly volume: number; }; class SingChannel { private readonly context: Context; - private readonly synth: Synth; private readonly gainNode: GainNode; - private sequence?: SoundSequence; + private phrases: Phrase[] = []; + private sequences: SoundSequence[] = []; get volume() { return this.gainNode.gain.value; @@ -123,28 +206,58 @@ class SingChannel { this.context = context; const audioContext = context.audioContext; - this.synth = new Synth(context); this.gainNode = audioContext.createGain(); - - this.synth.connect(this.gainNode); this.gainNode.connect(audioContext.destination); - this.gainNode.gain.value = options.volume; } - setNoteEvents(noteEvents: NoteEvent[]) { - if (this.sequence) { - this.context.transport.removeSequence(this.sequence); + addPhrase(phrase: Phrase) { + this.phrases.push(phrase); + + let sequence: SoundSequence | undefined; + if (phrase.startTime !== undefined && phrase.buffer) { + const audioEvents: AudioEvent[] = [ + { + time: phrase.startTime, + buffer: phrase.buffer, + }, + ]; + const audioPlayer = new AudioPlayer(this.context); + audioPlayer.connect(this.gainNode); + sequence = new AudioSequence(audioPlayer, audioEvents); + } else { + const noteEvents = generateNoteEvents( + phrase.score.resolution, + phrase.score.tempos, + phrase.score.notes + ); + const synth = new Synth(this.context); + synth.connect(this.gainNode); + sequence = new NoteSequence(synth, noteEvents); } - const sequence = new NoteSequence(this.synth, noteEvents); + this.context.transport.addSequence(sequence); - this.sequence = sequence; + this.sequences.push(sequence); } - dispose() { - if (this.sequence) { - this.context.transport.removeSequence(this.sequence); + removePhrase(phrase: Phrase) { + const index = this.phrases.findIndex((value) => { + return value === phrase; + }); + if (index === -1) { + throw new Error("The specified phrase does not exist."); } + this.phrases.splice(index, 1); + + const sequence = this.sequences[index]; + this.context.transport.removeSequence(sequence); + this.sequences.splice(index, 1); + } + + dispose() { + this.sequences.forEach((value) => { + this.context.transport.removeSequence(value); + }); } } @@ -180,23 +293,6 @@ const isValidNote = (note: Note) => { ); }; -/** - * ノートオンの時間とノートオフの時間を計算し、ノートイベントを生成します。 - */ -const generateNoteEvents = (score: Score, notes: Note[]) => { - const resolution = score.resolution; - const tempos = score.tempos; - return notes.map((value): NoteEvent => { - const noteOnPos = value.position; - const noteOffPos = value.position + value.duration; - return { - midi: value.midi, - noteOnTime: ticksToSeconds(resolution, tempos, noteOnPos), - noteOffTime: ticksToSeconds(resolution, tempos, noteOffPos), - }; - }); -}; - const getFromOptional = (value: T | undefined): T => { if (value === undefined) { throw new Error("The value is undefined."); @@ -221,12 +317,14 @@ if (window.AudioContext) { } let playbackPosition = 0; +const allPhrases = new Map(); + +const audioBufferCache = new Map(); export const singingStoreState: SingingStoreState = { engineId: undefined, styleId: undefined, score: undefined, - renderPhrases: [], // NOTE: UIの状態は試行のためsinging.tsに局所化する+Hydrateが必要 isShowSinger: true, sequencerZoomX: 0.5, @@ -238,6 +336,10 @@ export const singingStoreState: SingingStoreState = { volume: 0, leftLocatorPosition: 0, rightLocatorPosition: 0, + renderingEnabled: false, + startRenderingRequested: false, + stopRenderingRequested: false, + nowRendering: false, }; export const singingStore = createPartialStore({ @@ -260,10 +362,7 @@ export const singingStore = createPartialStore({ state.engineId = engineId; state.styleId = styleId; }, - async action( - { state, getters, dispatch, commit }, - payload: { engineId?: string; styleId?: number } - ) { + async action({ state, getters, dispatch, commit }, payload) { if (state.defaultStyleIds == undefined) throw new Error("state.defaultStyleIds == undefined"); if (getters.USER_ORDERED_CHARACTER_INFOS == undefined) @@ -296,6 +395,8 @@ export const singingStore = createPartialStore({ } } finally { commit("SET_SINGER", { engineId, styleId }); + + dispatch("RENDER"); } }, }, @@ -330,20 +431,17 @@ export const singingStore = createPartialStore({ state.score = score; }, async action({ state, getters, commit, dispatch }, { score }) { - console.log(score); - if (!transport || !singChannel) { - throw new Error("transport or singChannel is undefined."); + if (!transport) { + throw new Error("transport is undefined."); } if (state.nowPlaying) { await dispatch("SING_STOP_AUDIO"); } - commit("SET_SCORE", { score }); - const noteEvents = generateNoteEvents(score, score.notes); - singChannel.setNoteEvents(noteEvents); - transport.time = getters.POSITION_TO_TIME(playbackPosition); + + dispatch("RENDER"); }, }, @@ -355,13 +453,13 @@ export const singingStore = createPartialStore({ score.tempos = tempos; }, // テンポを設定する。既に同じ位置にテンポが存在する場合は置き換える。 - async action({ state, getters, commit }, { tempo }) { + async action({ state, getters, commit, dispatch }, { tempo }) { const score = state.score; if (score === undefined || score.tempos.length === 0) { throw new Error("Score is not initialized."); } - if (!transport || !singChannel) { - throw new Error("transport or singChannel is undefined."); + if (!transport) { + throw new Error("transport is undefined."); } if (!isValidTempo(tempo)) { throw new Error("The tempo is invalid."); @@ -385,10 +483,9 @@ export const singingStore = createPartialStore({ } commit("SET_TEMPO", { index, tempo }); - const noteEvents = generateNoteEvents(score, score.notes); - singChannel.setNoteEvents(noteEvents); - transport.time = getters.POSITION_TO_TIME(playbackPosition); + + dispatch("RENDER"); }, }, @@ -408,8 +505,8 @@ export const singingStore = createPartialStore({ if (score === undefined || score.tempos.length === 0) { throw new Error("Score is not initialized."); } - if (!transport || !singChannel) { - throw new Error("transport or singChannel is undefined."); + if (!transport) { + throw new Error("transport is undefined."); } const index = score.tempos.findIndex((value) => { return value.position === position; @@ -425,10 +522,9 @@ export const singingStore = createPartialStore({ commit("SET_TEMPO", { index, tempo: defaultTempo }); } - const noteEvents = generateNoteEvents(score, score.notes); - singChannel.setNoteEvents(noteEvents); - transport.time = getters.POSITION_TO_TIME(playbackPosition); + + dispatch("RENDER"); }, }, @@ -511,22 +607,16 @@ export const singingStore = createPartialStore({ }, // ノートを追加する // NOTE: 重複削除など別途追加 - async action({ state, commit }, { note }: { note: Note }) { + async action({ state, commit, dispatch }, { note }) { if (state.score === undefined) { throw new Error("Score is not initialized."); } - if (!singChannel) { - throw new Error("singChannel is undefined."); - } if (!isValidNote(note)) { throw new Error("The note is invalid."); } - commit("ADD_NOTE", { note }); - const score = getFromOptional(state.score); - const noteEvents = generateNoteEvents(score, score.notes); - singChannel.setNoteEvents(noteEvents); + dispatch("RENDER"); }, }, @@ -538,25 +628,16 @@ export const singingStore = createPartialStore({ state.score.notes = notes; } }, - async action( - { state, commit }, - { index, note }: { index: number; note: Note } - ) { + async action({ state, commit, dispatch }, { index, note }) { if (state.score === undefined) { throw new Error("Score is not initialized."); } - if (!singChannel) { - throw new Error("singChannel is undefined."); - } if (!isValidNote(note)) { throw new Error("The note is invalid."); } - commit("CHANGE_NOTE", { index, note }); - const score = getFromOptional(state.score); - const noteEvents = generateNoteEvents(score, score.notes); - singChannel.setNoteEvents(noteEvents); + dispatch("RENDER"); }, }, @@ -568,19 +649,13 @@ export const singingStore = createPartialStore({ state.score.notes = notes; } }, - async action({ state, commit }, { index }: { index: number }) { + async action({ state, commit, dispatch }, { index }) { if (state.score === undefined) { throw new Error("Score is not initialized."); } - if (!singChannel) { - throw new Error("singChannel is undefined."); - } - commit("REMOVE_NOTE", { index }); - const score = getFromOptional(state.score); - const noteEvents = generateNoteEvents(score, score.notes); - singChannel.setNoteEvents(noteEvents); + dispatch("RENDER"); }, }, @@ -722,6 +797,382 @@ export const singingStore = createPartialStore({ }, }, + SET_START_RENDERING_REQUESTED: { + mutation(state, { startRenderingRequested }) { + state.startRenderingRequested = startRenderingRequested; + }, + }, + + SET_STOP_RENDERING_REQUESTED: { + mutation(state, { stopRenderingRequested }) { + state.stopRenderingRequested = stopRenderingRequested; + }, + }, + + SET_NOW_RENDERING: { + mutation(state, { nowRendering }) { + state.nowRendering = nowRendering; + }, + }, + + /** + * レンダリングを行う。レンダリング中だった場合は停止して再レンダリングする。 + */ + RENDER: { + async action({ state, getters, commit, dispatch }) { + const preProcessing = (score: Score) => { + const resolution = score.resolution; + const tempos = score.tempos; + const notes = score.notes; + + // 重複するノートを除く + for (let i = 0; i < notes.length; i++) { + const note = notes[i]; + if (i === 0) continue; + const lastNote = notes[i - 1]; + if (note.position < lastNote.position + lastNote.duration) { + notes.splice(i, 1); + i--; + } + } + + // 長いノートを短くする + const maxNoteTime = 0.26; + for (let i = 0; i < notes.length; i++) { + const note = notes[i]; + const noteOnPos = note.position; + const noteOffPos = note.position + note.duration; + const noteOnTime = ticksToSeconds(resolution, tempos, noteOnPos); + const noteOffTime = ticksToSeconds(resolution, tempos, noteOffPos); + + if (noteOffTime - noteOnTime > maxNoteTime) { + let noteOffPos = secondsToTicks( + resolution, + tempos, + noteOnTime + maxNoteTime + ); + noteOffPos = Math.max(note.position + 1, Math.floor(noteOffPos)); + note.duration = noteOffPos - note.position; + } + } + }; + + const searchPhrases = async ( + singer: Singer | undefined, + score: Score + ) => { + const notes = score.notes; + const foundPhrases = new Map(); + let phraseNotes: Note[] = []; + for (let i = 0; i < notes.length; i++) { + const note = notes[i]; + + phraseNotes.push(note); + + if ( + i === notes.length - 1 || + note.position + note.duration !== notes[i + 1].position + ) { + const phraseScore = { + ...score, + notes: phraseNotes, + }; + const hash = await generateSingerAndScoreHash({ + singer, + score: phraseScore, // NOTE: とりあえず拍子も含めてハッシュ生成する + }); + foundPhrases.set(hash, { + singer, + score: phraseScore, + }); + + phraseNotes = []; + } + } + return foundPhrases; + }; + + const generateAndEditQuery = async (singer: Singer, score: Score) => { + if (!getters.IS_ENGINE_READY(singer.engineId)) { + throw new Error("Engine not ready."); + } + + const text = score.notes + .map((value) => value.lyric) + .map((value) => value.replace("は", "ハ")) // TODO: 助詞の扱いはあとで考える + .map((value) => value.replace("へ", "ヘ")) // TODO: 助詞の扱いはあとで考える + .join(""); + + const query = await dispatch("FETCH_AUDIO_QUERY", { + text, + engineId: singer.engineId, + styleId: singer.styleId, + }); + + const moras = query.accentPhrases.map((value) => value.moras).flat(); + + if (moras.length !== score.notes.length) { + throw new Error( + "The number of moras and the number of notes do not match." + ); + } + + // 音素を表示 + const phonemes = moras + .map((value) => { + if (value.consonant === undefined) { + return [value.vowel]; + } else { + return [value.consonant, value.vowel]; + } + }) + .flat() + .join(" "); + window.electron.logInfo(` phonemes: ${phonemes}`); + + // クエリを編集 + let noteIndex = 0; + for (let i = 0; i < query.accentPhrases.length; i++) { + const accentPhrase = query.accentPhrases[i]; + const moras = accentPhrase.moras; + for (let j = 0; j < moras.length; j++) { + const mora = moras[j]; + const note = score.notes[noteIndex]; + + const noteOnTime = ticksToSeconds( + score.resolution, + score.tempos, + note.position + ); + const noteOffTime = ticksToSeconds( + score.resolution, + score.tempos, + note.position + note.duration + ); + + // 長さを編集 + let vowelLength = noteOffTime - noteOnTime; + if (j !== moras.length - 1) { + const nextMora = moras[j + 1]; + if (nextMora.consonantLength !== undefined) { + vowelLength -= nextMora.consonantLength; + } + } else if (i !== query.accentPhrases.length - 1) { + const nextAccentPhrase = query.accentPhrases[i + 1]; + const nextMora = nextAccentPhrase.moras[0]; + if (nextMora.consonantLength !== undefined) { + vowelLength -= nextMora.consonantLength; + } + } + mora.vowelLength = Math.max(0.001, vowelLength); + + // 音高を編集 + const freq = midiToFrequency(note.midi); + mora.pitch = Math.log(freq); + + noteIndex++; + } + } + + return query; + }; + + const calculateStartTime = (score: Score, query: AudioQuery) => { + const firstMora = query.accentPhrases[0].moras[0]; + let startTime = ticksToSeconds( + score.resolution, + score.tempos, + score.notes[0].position + ); + startTime -= query.prePhonemeLength; + startTime -= firstMora.consonantLength ?? 0; + return startTime; + }; + + const synthesize = async (singer: Singer, query: AudioQuery) => { + if (!getters.IS_ENGINE_READY(singer.engineId)) { + throw new Error("Engine not ready."); + } + + const blob = await dispatch("INSTANTIATE_ENGINE_CONNECTOR", { + engineId: singer.engineId, + }).then((instance) => { + return instance.invoke("synthesisSynthesisPost")({ + audioQuery: query, + speaker: singer.styleId, + enableInterrogativeUpspeak: + state.experimentalSetting.enableInterrogativeUpspeak, + }); + }); + return blob; + }; + + const getSinger = (): Singer | undefined => { + if (state.engineId === undefined || state.styleId === undefined) { + return undefined; + } + return { engineId: state.engineId, styleId: state.styleId }; + }; + + // NOTE: 型推論でawaitの前か後かが考慮されないので、関数を介して取得する(型がbooleanになるようにする) + const startRenderingRequested = () => state.startRenderingRequested; + const stopRenderingRequested = () => state.stopRenderingRequested; + + const render = async () => { + if (!state.score || !singChannel || !audioRenderer) { + throw new Error( + "score or singChannel or audioRenderer is undefined." + ); + } + const singChannelRef = singChannel; + const audioRendererRef = audioRenderer; + + // レンダリング中に変更される可能性のあるデータをコピーする + const score = copyScore(state.score); + const singer = getSinger(); + + preProcessing(score); + + // Score -> Phrases + + window.electron.logInfo("Updating phrases..."); + + const foundPhrases = await searchPhrases(singer, score); + for (const [hash, phrase] of foundPhrases) { + if (!allPhrases.has(hash)) { + allPhrases.set(hash, phrase); + singChannelRef.addPhrase(phrase); + } + } + for (const [hash, phrase] of allPhrases) { + if (!foundPhrases.has(hash)) { + allPhrases.delete(hash); + singChannelRef.removePhrase(phrase); + } + } + + window.electron.logInfo("Phrases updated."); + + if (startRenderingRequested() || stopRenderingRequested()) { + return; + } + + for (const phrase of allPhrases.values()) { + if (!phrase.singer) { + continue; + } + + // Phrase -> AudioQuery + + if (!phrase.query) { + window.electron.logInfo(`Generating query...`); + + phrase.query = await generateAndEditQuery( + phrase.singer, + phrase.score + ); + + window.electron.logInfo(`Query generated.`); + } + + if (startRenderingRequested() || stopRenderingRequested()) { + return; + } + + // AudioQuery -> AudioBuffer + // Phrase & AudioQuery -> startTime + + const queryHash = await generateAudioQueryHash(phrase.query); + // クエリが変更されていたら再合成 + if (queryHash !== phrase.queryHash) { + phrase.buffer = audioBufferCache.get(queryHash); + if (phrase.buffer) { + window.electron.logInfo(`Loaded audio buffer from cache.`); + } else { + window.electron.logInfo(`Synthesizing...`); + + const blob = await synthesize(phrase.singer, phrase.query); + phrase.buffer = await audioRendererRef.createAudioBuffer(blob); + audioBufferCache.set(queryHash, phrase.buffer); + + window.electron.logInfo(`Synthesized.`); + } + phrase.queryHash = queryHash; + phrase.startTime = calculateStartTime(phrase.score, phrase.query); + + // NoteSequenceが削除される + singChannelRef.removePhrase(phrase); + // AudioBufferとstartTimeを元にAudioSequenceが作成され、追加される + // TODO: 分かりにくいのでリファクタリングする + singChannelRef.addPhrase(phrase); + } + + if (startRenderingRequested() || stopRenderingRequested()) { + return; + } + } + }; + + commit("SET_START_RENDERING_REQUESTED", { + startRenderingRequested: true, + }); + if (!state.renderingEnabled || state.nowRendering) { + return; + } + + commit("SET_NOW_RENDERING", { nowRendering: true }); + try { + while (startRenderingRequested()) { + commit("SET_START_RENDERING_REQUESTED", { + startRenderingRequested: false, + }); + await render(); + if (stopRenderingRequested()) { + break; + } + } + } catch (e) { + window.electron.logError(e); + throw e; + } finally { + commit("SET_STOP_RENDERING_REQUESTED", { + stopRenderingRequested: false, + }); + commit("SET_NOW_RENDERING", { nowRendering: false }); + } + }, + }, + + /** + * レンダリング停止をリクエストし、停止するまで待機する。 + */ + STOP_RENDERING: { + action: createUILockAction(async ({ state, commit }) => { + if (state.nowRendering) { + window.electron.logInfo("Waiting for rendering to stop..."); + commit("SET_STOP_RENDERING_REQUESTED", { + stopRenderingRequested: true, + }); + await createPromiseThatResolvesWhen(() => !state.nowRendering); + window.electron.logInfo("Rendering stopped."); + } + }), + }, + + SET_RENDERING_ENABLED: { + mutation(state, { renderingEnabled }) { + state.renderingEnabled = renderingEnabled; + }, + async action({ commit, dispatch }, { renderingEnabled }) { + if (renderingEnabled) { + dispatch("RENDER"); + } else { + await dispatch("STOP_RENDERING"); + } + commit("SET_RENDERING_ENABLED", { renderingEnabled }); + }, + }, + IMPORT_MIDI_FILE: { action: createUILockAction( async ({ dispatch }, { filePath }: { filePath?: string }) => { @@ -1200,10 +1651,8 @@ export const singingStore = createPartialStore({ // TODO: ファイル名を設定できるようにする const fileName = "test_export.wav"; - const score = getFromOptional(state.score); const leftLocatorPos = state.leftLocatorPosition; const rightLocatorPos = state.rightLocatorPosition; - const renderStartTime = getters.POSITION_TO_TIME(leftLocatorPos); const renderEndTime = getters.POSITION_TO_TIME(rightLocatorPos); const renderDuration = renderEndTime - renderStartTime; @@ -1216,6 +1665,9 @@ export const singingStore = createPartialStore({ if (state.nowPlaying) { await dispatch("SING_STOP_AUDIO"); } + if (state.nowRendering) { + await dispatch("STOP_RENDERING"); + } if (state.savingSetting.fixedExportEnabled) { filePath = path.join(state.savingSetting.fixedExportDir, fileName); @@ -1249,8 +1701,7 @@ export const singingStore = createPartialStore({ // コンテキストにはノードや発音が登録されているので問題なくレンダリングされます // TODO: 分かりにくいので後でリファクタリングしたい const channel = new SingChannel(context); - const noteEvents = generateNoteEvents(score, score.notes); - channel.setNoteEvents(noteEvents); + allPhrases.forEach((value) => channel.addPhrase(value)); } ); const waveFileData = convertToWavFileData(audioBuffer); diff --git a/src/store/type.ts b/src/store/type.ts index 3193f38031..f6cf130dfe 100644 --- a/src/store/type.ts +++ b/src/store/type.ts @@ -77,11 +77,6 @@ export type Score = { notes: Note[]; }; -export type RenderPhrase = { - renderNotes: Note[]; - query?: AudioQuery; -}; - export type Command = { unixMillisec: number; undoPatches: Patch[]; @@ -697,7 +692,6 @@ export type SingingStoreState = { engineId?: string; styleId?: number; score?: Score; - renderPhrases: RenderPhrase[]; // NOTE: UIの状態などは分割・統合した方がよさそうだが、ボイス側と混在させないためいったん局所化する isShowSinger: boolean; // NOTE: オーディオ再生はボイスと同様もしくは拡張して使う? @@ -710,6 +704,10 @@ export type SingingStoreState = { volume: number; leftLocatorPosition: number; rightLocatorPosition: number; + renderingEnabled: boolean; + startRenderingRequested: boolean; + stopRenderingRequested: boolean; + nowRendering: boolean; }; export type SingingStoreTypes = { @@ -841,6 +839,31 @@ export type SingingStoreTypes = { mutation: { volume: number }; action(payload: { volume: number }): void; }; + + SET_START_RENDERING_REQUESTED: { + mutation: { startRenderingRequested: boolean }; + }; + + SET_STOP_RENDERING_REQUESTED: { + mutation: { stopRenderingRequested: boolean }; + }; + + SET_NOW_RENDERING: { + mutation: { nowRendering: boolean }; + }; + + RENDER: { + action(): void; + }; + + STOP_RENDERING: { + action(): void; + }; + + SET_RENDERING_ENABLED: { + mutation: { renderingEnabled: boolean }; + action(payload: { renderingEnabled: boolean }): void; + }; }; /* @@ -1271,6 +1294,10 @@ export type UiStoreTypes = { CHECK_EDITED_AND_NOT_SAVE: { action(): Promise; }; + + PROCESS_BEFORE_QUITTING: { + action(): Promise; + }; }; /* diff --git a/src/store/ui.ts b/src/store/ui.ts index 432230d9c6..651b2a39a1 100644 --- a/src/store/ui.ts +++ b/src/store/ui.ts @@ -530,6 +530,14 @@ export const uiStore = createPartialStore({ return; } } + }, + }, + + PROCESS_BEFORE_QUITTING: { + async action({ dispatch }) { + await dispatch("SING_STOP_AUDIO"); + await dispatch("CHECK_EDITED_AND_NOT_SAVE"); + await dispatch("STOP_RENDERING"); window.electron.closeWindow(); }, diff --git a/src/type/ipc.ts b/src/type/ipc.ts index b4500cbd42..ae2fa36d39 100644 --- a/src/type/ipc.ts +++ b/src/type/ipc.ts @@ -292,7 +292,7 @@ export type IpcSOData = { return: void; }; - CHECK_EDITED_AND_NOT_SAVE: { + PROCESS_BEFORE_QUITTING: { args: []; return: void; }; diff --git a/src/views/SingerHome.vue b/src/views/SingerHome.vue index da0391b177..bf93886acd 100644 --- a/src/views/SingerHome.vue +++ b/src/views/SingerHome.vue @@ -62,6 +62,7 @@ export default defineComponent({ await store.dispatch("SET_RIGHT_LOCATOR_POSITION", { position: 480 * 4 * 16, }); + await store.dispatch("SET_RENDERING_ENABLED", { renderingEnabled: true }); return {}; }); }, diff --git a/tests/unit/store/Vuex.spec.ts b/tests/unit/store/Vuex.spec.ts index 0a6dc87004..7beac9f8b5 100644 --- a/tests/unit/store/Vuex.spec.ts +++ b/tests/unit/store/Vuex.spec.ts @@ -32,7 +32,6 @@ describe("store/vuex.js test", () => { uiLockCount: 0, dialogLockCount: 0, nowPlayingContinuously: false, - renderPhrases: [], undoCommands: [], redoCommands: [], useGpu: false, @@ -128,6 +127,10 @@ describe("store/vuex.js test", () => { volume: 0, leftLocatorPosition: 0, rightLocatorPosition: 0, + renderingEnabled: false, + startRenderingRequested: false, + stopRenderingRequested: false, + nowRendering: false, }, getters: { ...uiStore.getters, @@ -189,8 +192,6 @@ describe("store/vuex.js test", () => { assert.equal(store.state.audioPlayStartPoint, 0); assert.equal(store.state.uiLockCount, 0); assert.equal(store.state.nowPlayingContinuously, false); - assert.isArray(store.state.renderPhrases); - assert.isEmpty(store.state.renderPhrases); assert.isArray(store.state.undoCommands); assert.isEmpty(store.state.undoCommands); assert.isArray(store.state.redoCommands);