diff --git a/src/background.ts b/src/background.ts
index cf03a40f9d..24a172ba15 100644
--- a/src/background.ts
+++ b/src/background.ts
@@ -861,7 +861,7 @@ async function createWindow() {
win.on("close", (event) => {
if (!willQuit) {
event.preventDefault();
- ipcMainSend(win, "CHECK_EDITED_AND_NOT_SAVE");
+ ipcMainSend(win, "PROCESS_BEFORE_QUITTING");
return;
}
});
@@ -1214,7 +1214,7 @@ app.on("window-all-closed", () => {
app.on("before-quit", (event) => {
if (!willQuit) {
event.preventDefault();
- ipcMainSend(win, "CHECK_EDITED_AND_NOT_SAVE");
+ ipcMainSend(win, "PROCESS_BEFORE_QUITTING");
return;
}
diff --git a/src/components/AcceptTermsDialog.vue b/src/components/AcceptTermsDialog.vue
index 19180e5402..3188145ac1 100644
--- a/src/components/AcceptTermsDialog.vue
+++ b/src/components/AcceptTermsDialog.vue
@@ -87,7 +87,9 @@ export default defineComponent({
store.dispatch("SET_ACCEPT_TERMS", {
acceptTerms: acceptTerms ? "Accepted" : "Rejected",
});
- !acceptTerms ? store.dispatch("CHECK_EDITED_AND_NOT_SAVE") : undefined;
+ if (!acceptTerms) {
+ store.dispatch("PROCESS_BEFORE_QUITTING");
+ }
modelValueComputed.value = false;
};
diff --git a/src/components/MinMaxCloseButtons.vue b/src/components/MinMaxCloseButtons.vue
index 5a68e06f47..5975336aa0 100644
--- a/src/components/MinMaxCloseButtons.vue
+++ b/src/components/MinMaxCloseButtons.vue
@@ -95,8 +95,8 @@ export default defineComponent({
setup() {
const store = useStore();
- const closeWindow = async () => {
- store.dispatch("CHECK_EDITED_AND_NOT_SAVE");
+ const closeWindow = () => {
+ store.dispatch("PROCESS_BEFORE_QUITTING");
};
const minimizeWindow = () => window.electron.minimizeWindow();
const maximizeWindow = () => window.electron.maximizeWindow();
diff --git a/src/components/Sing/MenuBar.vue b/src/components/Sing/MenuBar.vue
index 1719a29c40..0b1f83fcf6 100644
--- a/src/components/Sing/MenuBar.vue
+++ b/src/components/Sing/MenuBar.vue
@@ -11,6 +11,8 @@
root.type === 'button' ? (subMenuOpenFlags[index] = false) : undefined
"
/>
+
+
@@ -18,6 +20,7 @@
import { defineComponent, ref, computed, ComputedRef } from "vue";
import { useStore } from "@/store";
import MenuButton from "@/components/MenuButton.vue";
+import MinMaxCloseButtons from "@/components/MinMaxCloseButtons.vue";
import { HotkeyAction, HotkeyReturnType } from "@/type/preload";
import { setHotkeyFunctions } from "@/store/setting";
@@ -57,6 +60,7 @@ export default defineComponent({
components: {
MenuButton,
+ MinMaxCloseButtons,
},
setup() {
diff --git a/src/helpers/singHelper.ts b/src/helpers/singHelper.ts
index f55e41c3e7..7c089fc6e8 100644
--- a/src/helpers/singHelper.ts
+++ b/src/helpers/singHelper.ts
@@ -66,3 +66,7 @@ export function round(value: number, digits: number) {
const powerOf10 = 10 ** digits;
return Math.round(value * powerOf10) / powerOf10;
}
+
+export function midiToFrequency(midi: number) {
+ return 440 * 2 ** ((midi - 69) / 12);
+}
diff --git a/src/infrastructures/AudioRenderer.ts b/src/infrastructures/AudioRenderer.ts
index 3955c50950..9d05969403 100644
--- a/src/infrastructures/AudioRenderer.ts
+++ b/src/infrastructures/AudioRenderer.ts
@@ -273,6 +273,45 @@ export class OfflineTransport implements BaseTransport {
}
}
+export type AudioEvent = {
+ readonly time: number;
+ readonly buffer: AudioBuffer;
+};
+
+export class AudioSequence implements SoundSequence {
+ private readonly audioPlayer: AudioPlayer;
+ private readonly audioEvents: AudioEvent[];
+
+ constructor(audioPlayer: AudioPlayer, audioEvents: AudioEvent[]) {
+ this.audioPlayer = audioPlayer;
+ this.audioEvents = audioEvents;
+ }
+
+ // スケジュール可能なイベントを生成する
+ generateEvents(startTime: number) {
+ return this.audioEvents
+ .sort((a, b) => a.time - b.time)
+ .filter((value) => {
+ const audioEndTime = value.time + value.buffer.duration;
+ return audioEndTime > startTime;
+ })
+ .map((value): SchedulableEvent => {
+ const offset = Math.max(startTime - value.time, 0);
+ return {
+ time: Math.max(value.time, startTime),
+ schedule: (contextTime: number) => {
+ this.audioPlayer.play(contextTime, offset, value.buffer);
+ },
+ };
+ });
+ }
+
+ // シーケンスの停止をスケジュールする
+ scheduleStop(contextTime: number) {
+ this.audioPlayer.allStop(contextTime);
+ }
+}
+
export interface Instrument {
noteOn(contextTime: number, midi: number): void;
noteOff(contextTime: number, midi: number): void;
@@ -317,12 +356,101 @@ export class NoteSequence implements SoundSequence {
.sort((a, b) => a.time - b.time);
}
- // シーケンス(楽器)の停止をスケジュールする
+ // シーケンスの停止をスケジュールする
scheduleStop(contextTime: number) {
this.instrument.allSoundOff(contextTime);
}
}
+class AudioPlayerVoice {
+ private readonly audioBufferSourceNode: AudioBufferSourceNode;
+ private readonly buffer: AudioBuffer;
+
+ private _isStopped = false;
+ private stopContextTime?: number;
+
+ get isStopped() {
+ return this._isStopped;
+ }
+
+ constructor(audioContext: BaseAudioContext, buffer: AudioBuffer) {
+ this.audioBufferSourceNode = audioContext.createBufferSource();
+ this.audioBufferSourceNode.buffer = buffer;
+ this.audioBufferSourceNode.onended = () => {
+ this._isStopped = true;
+ };
+ this.buffer = buffer;
+ }
+
+ connect(destination: AudioNode) {
+ this.audioBufferSourceNode.connect(destination);
+ }
+
+ play(contextTime: number, offset: number) {
+ this.audioBufferSourceNode.start(contextTime, offset);
+ this.stopContextTime = contextTime + this.buffer.duration;
+ }
+
+ stop(contextTime?: number) {
+ if (this.stopContextTime === undefined) {
+ throw new Error("Not started.");
+ }
+ if (contextTime === undefined || contextTime < this.stopContextTime) {
+ this.audioBufferSourceNode.stop(contextTime);
+ this.stopContextTime = contextTime ?? 0;
+ }
+ }
+}
+
+export type AudioPlayerOptions = {
+ readonly volume: number;
+};
+
+export class AudioPlayer {
+ private readonly audioContext: BaseAudioContext;
+ private readonly gainNode: GainNode;
+
+ private voices: AudioPlayerVoice[] = [];
+
+ constructor(context: Context, options: AudioPlayerOptions = { volume: 1.0 }) {
+ this.audioContext = context.audioContext;
+
+ this.gainNode = this.audioContext.createGain();
+ this.gainNode.gain.value = options.volume;
+ }
+
+ connect(destination: AudioNode) {
+ this.gainNode.connect(destination);
+ }
+
+ disconnect() {
+ this.gainNode.disconnect();
+ }
+
+ play(contextTime: number, offset: number, buffer: AudioBuffer) {
+ const voice = new AudioPlayerVoice(this.audioContext, buffer);
+ this.voices = this.voices.filter((value) => {
+ return !value.isStopped;
+ });
+ this.voices.push(voice);
+ voice.connect(this.gainNode);
+ voice.play(contextTime, offset);
+ }
+
+ allStop(contextTime?: number) {
+ if (contextTime === undefined) {
+ this.voices.forEach((value) => {
+ value.stop();
+ });
+ this.voices = [];
+ } else {
+ this.voices.forEach((value) => {
+ value.stop(contextTime);
+ });
+ }
+ }
+}
+
export type Envelope = {
readonly attack: number;
readonly decay: number;
@@ -371,8 +499,8 @@ class SynthVoice {
return 440 * 2 ** ((midi - 69) / 12);
}
- connect(inputNode: AudioNode) {
- this.gainNode.connect(inputNode);
+ connect(destination: AudioNode) {
+ this.gainNode.connect(destination);
}
noteOn(contextTime: number) {
@@ -532,6 +660,10 @@ export class AudioRenderer {
};
}
+ get audioContext() {
+ return this.onlineContext.audioContext;
+ }
+
get transport() {
return this.onlineContext.transport;
}
@@ -542,7 +674,14 @@ export class AudioRenderer {
this.onlineContext = { audioContext, transport };
}
- renderToBuffer(
+ async createAudioBuffer(blob: Blob) {
+ const audioContext = this.onlineContext.audioContext;
+ const arrayBuffer = await blob.arrayBuffer();
+ const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+ return audioBuffer;
+ }
+
+ async renderToBuffer(
sampleRate: number,
startTime: number,
duration: number,
@@ -558,7 +697,8 @@ export class AudioRenderer {
callback({ audioContext, transport });
transport.scheduleEvents(startTime, duration);
- return audioContext.startRendering();
+ const audioBuffer = await audioContext.startRendering();
+ return audioBuffer;
}
dispose() {
diff --git a/src/plugins/ipcMessageReceiverPlugin.ts b/src/plugins/ipcMessageReceiverPlugin.ts
index 296e51cee2..2c66dea8f3 100644
--- a/src/plugins/ipcMessageReceiverPlugin.ts
+++ b/src/plugins/ipcMessageReceiverPlugin.ts
@@ -44,8 +44,8 @@ export const ipcMessageReceiver: Plugin = {
options.store.dispatch("DETECT_LEAVE_FULLSCREEN")
);
- window.electron.onReceivedIPCMsg("CHECK_EDITED_AND_NOT_SAVE", () => {
- options.store.dispatch("CHECK_EDITED_AND_NOT_SAVE");
+ window.electron.onReceivedIPCMsg("PROCESS_BEFORE_QUITTING", () => {
+ options.store.dispatch("PROCESS_BEFORE_QUITTING");
});
window.electron.onReceivedIPCMsg(
diff --git a/src/store/singing.ts b/src/store/singing.ts
index 199f5228c2..a5b66d7b65 100644
--- a/src/store/singing.ts
+++ b/src/store/singing.ts
@@ -1,5 +1,8 @@
import {
+ AudioEvent,
+ AudioPlayer,
AudioRenderer,
+ AudioSequence,
Context,
NoteEvent,
NoteSequence,
@@ -21,7 +24,12 @@ import { createPartialStore } from "./vuex";
import { createUILockAction } from "./ui";
import { WriteFileErrorResult } from "@/type/preload";
import { Midi } from "@tonejs/midi";
-import { getDoremiFromMidi, round } from "@/helpers/singHelper";
+import {
+ getDoremiFromMidi,
+ midiToFrequency,
+ round,
+} from "@/helpers/singHelper";
+import { AudioQuery } from "@/openapi";
const ticksToSecondsForConstantBpm = (
resolution: number,
@@ -100,16 +108,91 @@ const secondsToTicks = (
);
};
+const generateNoteEvents = (
+ resolution: number,
+ tempos: Tempo[],
+ notes: Note[]
+) => {
+ return notes.map((value): NoteEvent => {
+ const noteOnPos = value.position;
+ const noteOffPos = value.position + value.duration;
+ return {
+ midi: value.midi,
+ noteOnTime: ticksToSeconds(resolution, tempos, noteOnPos),
+ noteOffTime: ticksToSeconds(resolution, tempos, noteOffPos),
+ };
+ });
+};
+
+const copyScore = (score: Score): Score => {
+ return {
+ resolution: score.resolution,
+ tempos: score.tempos.map((value) => ({ ...value })),
+ timeSignatures: score.timeSignatures.map((value) => ({ ...value })),
+ notes: score.notes.map((value) => ({ ...value })),
+ };
+};
+
+const _generateHash = async (obj: T) => {
+ const textEncoder = new TextEncoder();
+ const data = textEncoder.encode(JSON.stringify(obj));
+ const digest = await crypto.subtle.digest("SHA-256", data);
+ return Array.from(new Uint8Array(digest))
+ .map((v) => v.toString(16).padStart(2, "0"))
+ .join("");
+};
+
+const createPromiseThatResolvesWhen = (
+ condition: () => boolean,
+ interval = 200
+) => {
+ return new Promise((resolve) => {
+ const checkCondition = () => {
+ if (condition()) {
+ resolve();
+ }
+ window.setTimeout(checkCondition, interval);
+ };
+ checkCondition();
+ });
+};
+
+type Singer = {
+ readonly engineId: string;
+ readonly styleId: number;
+};
+
+type Phrase = {
+ readonly singer: Singer | undefined;
+ readonly score: Score;
+ // renderingが進むに連れてデータが代入されていく
+ query?: AudioQuery;
+ queryHash?: string; // queryの変更を検知するためのハッシュ
+ buffer?: AudioBuffer;
+ startTime?: number;
+};
+
+const generateSingerAndScoreHash = async (obj: {
+ singer: Singer | undefined;
+ score: Score;
+}) => {
+ return _generateHash(obj);
+};
+
+const generateAudioQueryHash = async (obj: AudioQuery) => {
+ return _generateHash(obj);
+};
+
type ChannelOptions = {
readonly volume: number;
};
class SingChannel {
private readonly context: Context;
- private readonly synth: Synth;
private readonly gainNode: GainNode;
- private sequence?: SoundSequence;
+ private phrases: Phrase[] = [];
+ private sequences: SoundSequence[] = [];
get volume() {
return this.gainNode.gain.value;
@@ -123,28 +206,58 @@ class SingChannel {
this.context = context;
const audioContext = context.audioContext;
- this.synth = new Synth(context);
this.gainNode = audioContext.createGain();
-
- this.synth.connect(this.gainNode);
this.gainNode.connect(audioContext.destination);
-
this.gainNode.gain.value = options.volume;
}
- setNoteEvents(noteEvents: NoteEvent[]) {
- if (this.sequence) {
- this.context.transport.removeSequence(this.sequence);
+ addPhrase(phrase: Phrase) {
+ this.phrases.push(phrase);
+
+ let sequence: SoundSequence | undefined;
+ if (phrase.startTime !== undefined && phrase.buffer) {
+ const audioEvents: AudioEvent[] = [
+ {
+ time: phrase.startTime,
+ buffer: phrase.buffer,
+ },
+ ];
+ const audioPlayer = new AudioPlayer(this.context);
+ audioPlayer.connect(this.gainNode);
+ sequence = new AudioSequence(audioPlayer, audioEvents);
+ } else {
+ const noteEvents = generateNoteEvents(
+ phrase.score.resolution,
+ phrase.score.tempos,
+ phrase.score.notes
+ );
+ const synth = new Synth(this.context);
+ synth.connect(this.gainNode);
+ sequence = new NoteSequence(synth, noteEvents);
}
- const sequence = new NoteSequence(this.synth, noteEvents);
+
this.context.transport.addSequence(sequence);
- this.sequence = sequence;
+ this.sequences.push(sequence);
}
- dispose() {
- if (this.sequence) {
- this.context.transport.removeSequence(this.sequence);
+ removePhrase(phrase: Phrase) {
+ const index = this.phrases.findIndex((value) => {
+ return value === phrase;
+ });
+ if (index === -1) {
+ throw new Error("The specified phrase does not exist.");
}
+ this.phrases.splice(index, 1);
+
+ const sequence = this.sequences[index];
+ this.context.transport.removeSequence(sequence);
+ this.sequences.splice(index, 1);
+ }
+
+ dispose() {
+ this.sequences.forEach((value) => {
+ this.context.transport.removeSequence(value);
+ });
}
}
@@ -180,23 +293,6 @@ const isValidNote = (note: Note) => {
);
};
-/**
- * ノートオンの時間とノートオフの時間を計算し、ノートイベントを生成します。
- */
-const generateNoteEvents = (score: Score, notes: Note[]) => {
- const resolution = score.resolution;
- const tempos = score.tempos;
- return notes.map((value): NoteEvent => {
- const noteOnPos = value.position;
- const noteOffPos = value.position + value.duration;
- return {
- midi: value.midi,
- noteOnTime: ticksToSeconds(resolution, tempos, noteOnPos),
- noteOffTime: ticksToSeconds(resolution, tempos, noteOffPos),
- };
- });
-};
-
const getFromOptional = (value: T | undefined): T => {
if (value === undefined) {
throw new Error("The value is undefined.");
@@ -221,12 +317,14 @@ if (window.AudioContext) {
}
let playbackPosition = 0;
+const allPhrases = new Map();
+
+const audioBufferCache = new Map();
export const singingStoreState: SingingStoreState = {
engineId: undefined,
styleId: undefined,
score: undefined,
- renderPhrases: [],
// NOTE: UIの状態は試行のためsinging.tsに局所化する+Hydrateが必要
isShowSinger: true,
sequencerZoomX: 0.5,
@@ -238,6 +336,10 @@ export const singingStoreState: SingingStoreState = {
volume: 0,
leftLocatorPosition: 0,
rightLocatorPosition: 0,
+ renderingEnabled: false,
+ startRenderingRequested: false,
+ stopRenderingRequested: false,
+ nowRendering: false,
};
export const singingStore = createPartialStore({
@@ -260,10 +362,7 @@ export const singingStore = createPartialStore({
state.engineId = engineId;
state.styleId = styleId;
},
- async action(
- { state, getters, dispatch, commit },
- payload: { engineId?: string; styleId?: number }
- ) {
+ async action({ state, getters, dispatch, commit }, payload) {
if (state.defaultStyleIds == undefined)
throw new Error("state.defaultStyleIds == undefined");
if (getters.USER_ORDERED_CHARACTER_INFOS == undefined)
@@ -296,6 +395,8 @@ export const singingStore = createPartialStore({
}
} finally {
commit("SET_SINGER", { engineId, styleId });
+
+ dispatch("RENDER");
}
},
},
@@ -330,20 +431,17 @@ export const singingStore = createPartialStore({
state.score = score;
},
async action({ state, getters, commit, dispatch }, { score }) {
- console.log(score);
- if (!transport || !singChannel) {
- throw new Error("transport or singChannel is undefined.");
+ if (!transport) {
+ throw new Error("transport is undefined.");
}
if (state.nowPlaying) {
await dispatch("SING_STOP_AUDIO");
}
-
commit("SET_SCORE", { score });
- const noteEvents = generateNoteEvents(score, score.notes);
- singChannel.setNoteEvents(noteEvents);
-
transport.time = getters.POSITION_TO_TIME(playbackPosition);
+
+ dispatch("RENDER");
},
},
@@ -355,13 +453,13 @@ export const singingStore = createPartialStore({
score.tempos = tempos;
},
// テンポを設定する。既に同じ位置にテンポが存在する場合は置き換える。
- async action({ state, getters, commit }, { tempo }) {
+ async action({ state, getters, commit, dispatch }, { tempo }) {
const score = state.score;
if (score === undefined || score.tempos.length === 0) {
throw new Error("Score is not initialized.");
}
- if (!transport || !singChannel) {
- throw new Error("transport or singChannel is undefined.");
+ if (!transport) {
+ throw new Error("transport is undefined.");
}
if (!isValidTempo(tempo)) {
throw new Error("The tempo is invalid.");
@@ -385,10 +483,9 @@ export const singingStore = createPartialStore({
}
commit("SET_TEMPO", { index, tempo });
- const noteEvents = generateNoteEvents(score, score.notes);
- singChannel.setNoteEvents(noteEvents);
-
transport.time = getters.POSITION_TO_TIME(playbackPosition);
+
+ dispatch("RENDER");
},
},
@@ -408,8 +505,8 @@ export const singingStore = createPartialStore({
if (score === undefined || score.tempos.length === 0) {
throw new Error("Score is not initialized.");
}
- if (!transport || !singChannel) {
- throw new Error("transport or singChannel is undefined.");
+ if (!transport) {
+ throw new Error("transport is undefined.");
}
const index = score.tempos.findIndex((value) => {
return value.position === position;
@@ -425,10 +522,9 @@ export const singingStore = createPartialStore({
commit("SET_TEMPO", { index, tempo: defaultTempo });
}
- const noteEvents = generateNoteEvents(score, score.notes);
- singChannel.setNoteEvents(noteEvents);
-
transport.time = getters.POSITION_TO_TIME(playbackPosition);
+
+ dispatch("RENDER");
},
},
@@ -511,22 +607,16 @@ export const singingStore = createPartialStore({
},
// ノートを追加する
// NOTE: 重複削除など別途追加
- async action({ state, commit }, { note }: { note: Note }) {
+ async action({ state, commit, dispatch }, { note }) {
if (state.score === undefined) {
throw new Error("Score is not initialized.");
}
- if (!singChannel) {
- throw new Error("singChannel is undefined.");
- }
if (!isValidNote(note)) {
throw new Error("The note is invalid.");
}
-
commit("ADD_NOTE", { note });
- const score = getFromOptional(state.score);
- const noteEvents = generateNoteEvents(score, score.notes);
- singChannel.setNoteEvents(noteEvents);
+ dispatch("RENDER");
},
},
@@ -538,25 +628,16 @@ export const singingStore = createPartialStore({
state.score.notes = notes;
}
},
- async action(
- { state, commit },
- { index, note }: { index: number; note: Note }
- ) {
+ async action({ state, commit, dispatch }, { index, note }) {
if (state.score === undefined) {
throw new Error("Score is not initialized.");
}
- if (!singChannel) {
- throw new Error("singChannel is undefined.");
- }
if (!isValidNote(note)) {
throw new Error("The note is invalid.");
}
-
commit("CHANGE_NOTE", { index, note });
- const score = getFromOptional(state.score);
- const noteEvents = generateNoteEvents(score, score.notes);
- singChannel.setNoteEvents(noteEvents);
+ dispatch("RENDER");
},
},
@@ -568,19 +649,13 @@ export const singingStore = createPartialStore({
state.score.notes = notes;
}
},
- async action({ state, commit }, { index }: { index: number }) {
+ async action({ state, commit, dispatch }, { index }) {
if (state.score === undefined) {
throw new Error("Score is not initialized.");
}
- if (!singChannel) {
- throw new Error("singChannel is undefined.");
- }
-
commit("REMOVE_NOTE", { index });
- const score = getFromOptional(state.score);
- const noteEvents = generateNoteEvents(score, score.notes);
- singChannel.setNoteEvents(noteEvents);
+ dispatch("RENDER");
},
},
@@ -722,6 +797,382 @@ export const singingStore = createPartialStore({
},
},
+ SET_START_RENDERING_REQUESTED: {
+ mutation(state, { startRenderingRequested }) {
+ state.startRenderingRequested = startRenderingRequested;
+ },
+ },
+
+ SET_STOP_RENDERING_REQUESTED: {
+ mutation(state, { stopRenderingRequested }) {
+ state.stopRenderingRequested = stopRenderingRequested;
+ },
+ },
+
+ SET_NOW_RENDERING: {
+ mutation(state, { nowRendering }) {
+ state.nowRendering = nowRendering;
+ },
+ },
+
+ /**
+ * レンダリングを行う。レンダリング中だった場合は停止して再レンダリングする。
+ */
+ RENDER: {
+ async action({ state, getters, commit, dispatch }) {
+ const preProcessing = (score: Score) => {
+ const resolution = score.resolution;
+ const tempos = score.tempos;
+ const notes = score.notes;
+
+ // 重複するノートを除く
+ for (let i = 0; i < notes.length; i++) {
+ const note = notes[i];
+ if (i === 0) continue;
+ const lastNote = notes[i - 1];
+ if (note.position < lastNote.position + lastNote.duration) {
+ notes.splice(i, 1);
+ i--;
+ }
+ }
+
+ // 長いノートを短くする
+ const maxNoteTime = 0.26;
+ for (let i = 0; i < notes.length; i++) {
+ const note = notes[i];
+ const noteOnPos = note.position;
+ const noteOffPos = note.position + note.duration;
+ const noteOnTime = ticksToSeconds(resolution, tempos, noteOnPos);
+ const noteOffTime = ticksToSeconds(resolution, tempos, noteOffPos);
+
+ if (noteOffTime - noteOnTime > maxNoteTime) {
+ let noteOffPos = secondsToTicks(
+ resolution,
+ tempos,
+ noteOnTime + maxNoteTime
+ );
+ noteOffPos = Math.max(note.position + 1, Math.floor(noteOffPos));
+ note.duration = noteOffPos - note.position;
+ }
+ }
+ };
+
+ const searchPhrases = async (
+ singer: Singer | undefined,
+ score: Score
+ ) => {
+ const notes = score.notes;
+ const foundPhrases = new Map();
+ let phraseNotes: Note[] = [];
+ for (let i = 0; i < notes.length; i++) {
+ const note = notes[i];
+
+ phraseNotes.push(note);
+
+ if (
+ i === notes.length - 1 ||
+ note.position + note.duration !== notes[i + 1].position
+ ) {
+ const phraseScore = {
+ ...score,
+ notes: phraseNotes,
+ };
+ const hash = await generateSingerAndScoreHash({
+ singer,
+ score: phraseScore, // NOTE: とりあえず拍子も含めてハッシュ生成する
+ });
+ foundPhrases.set(hash, {
+ singer,
+ score: phraseScore,
+ });
+
+ phraseNotes = [];
+ }
+ }
+ return foundPhrases;
+ };
+
+ const generateAndEditQuery = async (singer: Singer, score: Score) => {
+ if (!getters.IS_ENGINE_READY(singer.engineId)) {
+ throw new Error("Engine not ready.");
+ }
+
+ const text = score.notes
+ .map((value) => value.lyric)
+ .map((value) => value.replace("は", "ハ")) // TODO: 助詞の扱いはあとで考える
+ .map((value) => value.replace("へ", "ヘ")) // TODO: 助詞の扱いはあとで考える
+ .join("");
+
+ const query = await dispatch("FETCH_AUDIO_QUERY", {
+ text,
+ engineId: singer.engineId,
+ styleId: singer.styleId,
+ });
+
+ const moras = query.accentPhrases.map((value) => value.moras).flat();
+
+ if (moras.length !== score.notes.length) {
+ throw new Error(
+ "The number of moras and the number of notes do not match."
+ );
+ }
+
+ // 音素を表示
+ const phonemes = moras
+ .map((value) => {
+ if (value.consonant === undefined) {
+ return [value.vowel];
+ } else {
+ return [value.consonant, value.vowel];
+ }
+ })
+ .flat()
+ .join(" ");
+ window.electron.logInfo(` phonemes: ${phonemes}`);
+
+ // クエリを編集
+ let noteIndex = 0;
+ for (let i = 0; i < query.accentPhrases.length; i++) {
+ const accentPhrase = query.accentPhrases[i];
+ const moras = accentPhrase.moras;
+ for (let j = 0; j < moras.length; j++) {
+ const mora = moras[j];
+ const note = score.notes[noteIndex];
+
+ const noteOnTime = ticksToSeconds(
+ score.resolution,
+ score.tempos,
+ note.position
+ );
+ const noteOffTime = ticksToSeconds(
+ score.resolution,
+ score.tempos,
+ note.position + note.duration
+ );
+
+ // 長さを編集
+ let vowelLength = noteOffTime - noteOnTime;
+ if (j !== moras.length - 1) {
+ const nextMora = moras[j + 1];
+ if (nextMora.consonantLength !== undefined) {
+ vowelLength -= nextMora.consonantLength;
+ }
+ } else if (i !== query.accentPhrases.length - 1) {
+ const nextAccentPhrase = query.accentPhrases[i + 1];
+ const nextMora = nextAccentPhrase.moras[0];
+ if (nextMora.consonantLength !== undefined) {
+ vowelLength -= nextMora.consonantLength;
+ }
+ }
+ mora.vowelLength = Math.max(0.001, vowelLength);
+
+ // 音高を編集
+ const freq = midiToFrequency(note.midi);
+ mora.pitch = Math.log(freq);
+
+ noteIndex++;
+ }
+ }
+
+ return query;
+ };
+
+ const calculateStartTime = (score: Score, query: AudioQuery) => {
+ const firstMora = query.accentPhrases[0].moras[0];
+ let startTime = ticksToSeconds(
+ score.resolution,
+ score.tempos,
+ score.notes[0].position
+ );
+ startTime -= query.prePhonemeLength;
+ startTime -= firstMora.consonantLength ?? 0;
+ return startTime;
+ };
+
+ const synthesize = async (singer: Singer, query: AudioQuery) => {
+ if (!getters.IS_ENGINE_READY(singer.engineId)) {
+ throw new Error("Engine not ready.");
+ }
+
+ const blob = await dispatch("INSTANTIATE_ENGINE_CONNECTOR", {
+ engineId: singer.engineId,
+ }).then((instance) => {
+ return instance.invoke("synthesisSynthesisPost")({
+ audioQuery: query,
+ speaker: singer.styleId,
+ enableInterrogativeUpspeak:
+ state.experimentalSetting.enableInterrogativeUpspeak,
+ });
+ });
+ return blob;
+ };
+
+ const getSinger = (): Singer | undefined => {
+ if (state.engineId === undefined || state.styleId === undefined) {
+ return undefined;
+ }
+ return { engineId: state.engineId, styleId: state.styleId };
+ };
+
+ // NOTE: 型推論でawaitの前か後かが考慮されないので、関数を介して取得する(型がbooleanになるようにする)
+ const startRenderingRequested = () => state.startRenderingRequested;
+ const stopRenderingRequested = () => state.stopRenderingRequested;
+
+ const render = async () => {
+ if (!state.score || !singChannel || !audioRenderer) {
+ throw new Error(
+ "score or singChannel or audioRenderer is undefined."
+ );
+ }
+ const singChannelRef = singChannel;
+ const audioRendererRef = audioRenderer;
+
+ // レンダリング中に変更される可能性のあるデータをコピーする
+ const score = copyScore(state.score);
+ const singer = getSinger();
+
+ preProcessing(score);
+
+ // Score -> Phrases
+
+ window.electron.logInfo("Updating phrases...");
+
+ const foundPhrases = await searchPhrases(singer, score);
+ for (const [hash, phrase] of foundPhrases) {
+ if (!allPhrases.has(hash)) {
+ allPhrases.set(hash, phrase);
+ singChannelRef.addPhrase(phrase);
+ }
+ }
+ for (const [hash, phrase] of allPhrases) {
+ if (!foundPhrases.has(hash)) {
+ allPhrases.delete(hash);
+ singChannelRef.removePhrase(phrase);
+ }
+ }
+
+ window.electron.logInfo("Phrases updated.");
+
+ if (startRenderingRequested() || stopRenderingRequested()) {
+ return;
+ }
+
+ for (const phrase of allPhrases.values()) {
+ if (!phrase.singer) {
+ continue;
+ }
+
+ // Phrase -> AudioQuery
+
+ if (!phrase.query) {
+ window.electron.logInfo(`Generating query...`);
+
+ phrase.query = await generateAndEditQuery(
+ phrase.singer,
+ phrase.score
+ );
+
+ window.electron.logInfo(`Query generated.`);
+ }
+
+ if (startRenderingRequested() || stopRenderingRequested()) {
+ return;
+ }
+
+ // AudioQuery -> AudioBuffer
+ // Phrase & AudioQuery -> startTime
+
+ const queryHash = await generateAudioQueryHash(phrase.query);
+ // クエリが変更されていたら再合成
+ if (queryHash !== phrase.queryHash) {
+ phrase.buffer = audioBufferCache.get(queryHash);
+ if (phrase.buffer) {
+ window.electron.logInfo(`Loaded audio buffer from cache.`);
+ } else {
+ window.electron.logInfo(`Synthesizing...`);
+
+ const blob = await synthesize(phrase.singer, phrase.query);
+ phrase.buffer = await audioRendererRef.createAudioBuffer(blob);
+ audioBufferCache.set(queryHash, phrase.buffer);
+
+ window.electron.logInfo(`Synthesized.`);
+ }
+ phrase.queryHash = queryHash;
+ phrase.startTime = calculateStartTime(phrase.score, phrase.query);
+
+ // NoteSequenceが削除される
+ singChannelRef.removePhrase(phrase);
+ // AudioBufferとstartTimeを元にAudioSequenceが作成され、追加される
+ // TODO: 分かりにくいのでリファクタリングする
+ singChannelRef.addPhrase(phrase);
+ }
+
+ if (startRenderingRequested() || stopRenderingRequested()) {
+ return;
+ }
+ }
+ };
+
+ commit("SET_START_RENDERING_REQUESTED", {
+ startRenderingRequested: true,
+ });
+ if (!state.renderingEnabled || state.nowRendering) {
+ return;
+ }
+
+ commit("SET_NOW_RENDERING", { nowRendering: true });
+ try {
+ while (startRenderingRequested()) {
+ commit("SET_START_RENDERING_REQUESTED", {
+ startRenderingRequested: false,
+ });
+ await render();
+ if (stopRenderingRequested()) {
+ break;
+ }
+ }
+ } catch (e) {
+ window.electron.logError(e);
+ throw e;
+ } finally {
+ commit("SET_STOP_RENDERING_REQUESTED", {
+ stopRenderingRequested: false,
+ });
+ commit("SET_NOW_RENDERING", { nowRendering: false });
+ }
+ },
+ },
+
+ /**
+ * レンダリング停止をリクエストし、停止するまで待機する。
+ */
+ STOP_RENDERING: {
+ action: createUILockAction(async ({ state, commit }) => {
+ if (state.nowRendering) {
+ window.electron.logInfo("Waiting for rendering to stop...");
+ commit("SET_STOP_RENDERING_REQUESTED", {
+ stopRenderingRequested: true,
+ });
+ await createPromiseThatResolvesWhen(() => !state.nowRendering);
+ window.electron.logInfo("Rendering stopped.");
+ }
+ }),
+ },
+
+ SET_RENDERING_ENABLED: {
+ mutation(state, { renderingEnabled }) {
+ state.renderingEnabled = renderingEnabled;
+ },
+ async action({ commit, dispatch }, { renderingEnabled }) {
+ if (renderingEnabled) {
+ dispatch("RENDER");
+ } else {
+ await dispatch("STOP_RENDERING");
+ }
+ commit("SET_RENDERING_ENABLED", { renderingEnabled });
+ },
+ },
+
IMPORT_MIDI_FILE: {
action: createUILockAction(
async ({ dispatch }, { filePath }: { filePath?: string }) => {
@@ -1200,10 +1651,8 @@ export const singingStore = createPartialStore({
// TODO: ファイル名を設定できるようにする
const fileName = "test_export.wav";
- const score = getFromOptional(state.score);
const leftLocatorPos = state.leftLocatorPosition;
const rightLocatorPos = state.rightLocatorPosition;
-
const renderStartTime = getters.POSITION_TO_TIME(leftLocatorPos);
const renderEndTime = getters.POSITION_TO_TIME(rightLocatorPos);
const renderDuration = renderEndTime - renderStartTime;
@@ -1216,6 +1665,9 @@ export const singingStore = createPartialStore({
if (state.nowPlaying) {
await dispatch("SING_STOP_AUDIO");
}
+ if (state.nowRendering) {
+ await dispatch("STOP_RENDERING");
+ }
if (state.savingSetting.fixedExportEnabled) {
filePath = path.join(state.savingSetting.fixedExportDir, fileName);
@@ -1249,8 +1701,7 @@ export const singingStore = createPartialStore({
// コンテキストにはノードや発音が登録されているので問題なくレンダリングされます
// TODO: 分かりにくいので後でリファクタリングしたい
const channel = new SingChannel(context);
- const noteEvents = generateNoteEvents(score, score.notes);
- channel.setNoteEvents(noteEvents);
+ allPhrases.forEach((value) => channel.addPhrase(value));
}
);
const waveFileData = convertToWavFileData(audioBuffer);
diff --git a/src/store/type.ts b/src/store/type.ts
index 3193f38031..f6cf130dfe 100644
--- a/src/store/type.ts
+++ b/src/store/type.ts
@@ -77,11 +77,6 @@ export type Score = {
notes: Note[];
};
-export type RenderPhrase = {
- renderNotes: Note[];
- query?: AudioQuery;
-};
-
export type Command = {
unixMillisec: number;
undoPatches: Patch[];
@@ -697,7 +692,6 @@ export type SingingStoreState = {
engineId?: string;
styleId?: number;
score?: Score;
- renderPhrases: RenderPhrase[];
// NOTE: UIの状態などは分割・統合した方がよさそうだが、ボイス側と混在させないためいったん局所化する
isShowSinger: boolean;
// NOTE: オーディオ再生はボイスと同様もしくは拡張して使う?
@@ -710,6 +704,10 @@ export type SingingStoreState = {
volume: number;
leftLocatorPosition: number;
rightLocatorPosition: number;
+ renderingEnabled: boolean;
+ startRenderingRequested: boolean;
+ stopRenderingRequested: boolean;
+ nowRendering: boolean;
};
export type SingingStoreTypes = {
@@ -841,6 +839,31 @@ export type SingingStoreTypes = {
mutation: { volume: number };
action(payload: { volume: number }): void;
};
+
+ SET_START_RENDERING_REQUESTED: {
+ mutation: { startRenderingRequested: boolean };
+ };
+
+ SET_STOP_RENDERING_REQUESTED: {
+ mutation: { stopRenderingRequested: boolean };
+ };
+
+ SET_NOW_RENDERING: {
+ mutation: { nowRendering: boolean };
+ };
+
+ RENDER: {
+ action(): void;
+ };
+
+ STOP_RENDERING: {
+ action(): void;
+ };
+
+ SET_RENDERING_ENABLED: {
+ mutation: { renderingEnabled: boolean };
+ action(payload: { renderingEnabled: boolean }): void;
+ };
};
/*
@@ -1271,6 +1294,10 @@ export type UiStoreTypes = {
CHECK_EDITED_AND_NOT_SAVE: {
action(): Promise;
};
+
+ PROCESS_BEFORE_QUITTING: {
+ action(): Promise;
+ };
};
/*
diff --git a/src/store/ui.ts b/src/store/ui.ts
index 432230d9c6..651b2a39a1 100644
--- a/src/store/ui.ts
+++ b/src/store/ui.ts
@@ -530,6 +530,14 @@ export const uiStore = createPartialStore({
return;
}
}
+ },
+ },
+
+ PROCESS_BEFORE_QUITTING: {
+ async action({ dispatch }) {
+ await dispatch("SING_STOP_AUDIO");
+ await dispatch("CHECK_EDITED_AND_NOT_SAVE");
+ await dispatch("STOP_RENDERING");
window.electron.closeWindow();
},
diff --git a/src/type/ipc.ts b/src/type/ipc.ts
index b4500cbd42..ae2fa36d39 100644
--- a/src/type/ipc.ts
+++ b/src/type/ipc.ts
@@ -292,7 +292,7 @@ export type IpcSOData = {
return: void;
};
- CHECK_EDITED_AND_NOT_SAVE: {
+ PROCESS_BEFORE_QUITTING: {
args: [];
return: void;
};
diff --git a/src/views/SingerHome.vue b/src/views/SingerHome.vue
index da0391b177..bf93886acd 100644
--- a/src/views/SingerHome.vue
+++ b/src/views/SingerHome.vue
@@ -62,6 +62,7 @@ export default defineComponent({
await store.dispatch("SET_RIGHT_LOCATOR_POSITION", {
position: 480 * 4 * 16,
});
+ await store.dispatch("SET_RENDERING_ENABLED", { renderingEnabled: true });
return {};
});
},
diff --git a/tests/unit/store/Vuex.spec.ts b/tests/unit/store/Vuex.spec.ts
index 0a6dc87004..7beac9f8b5 100644
--- a/tests/unit/store/Vuex.spec.ts
+++ b/tests/unit/store/Vuex.spec.ts
@@ -32,7 +32,6 @@ describe("store/vuex.js test", () => {
uiLockCount: 0,
dialogLockCount: 0,
nowPlayingContinuously: false,
- renderPhrases: [],
undoCommands: [],
redoCommands: [],
useGpu: false,
@@ -128,6 +127,10 @@ describe("store/vuex.js test", () => {
volume: 0,
leftLocatorPosition: 0,
rightLocatorPosition: 0,
+ renderingEnabled: false,
+ startRenderingRequested: false,
+ stopRenderingRequested: false,
+ nowRendering: false,
},
getters: {
...uiStore.getters,
@@ -189,8 +192,6 @@ describe("store/vuex.js test", () => {
assert.equal(store.state.audioPlayStartPoint, 0);
assert.equal(store.state.uiLockCount, 0);
assert.equal(store.state.nowPlayingContinuously, false);
- assert.isArray(store.state.renderPhrases);
- assert.isEmpty(store.state.renderPhrases);
assert.isArray(store.state.undoCommands);
assert.isEmpty(store.state.undoCommands);
assert.isArray(store.state.redoCommands);