Updated sound engine, should be much more stable now

Author: williameclee

assets/custom-questions.json

@@ -0,0 +1,6 @@
+[
+	[
+		"I don't know you.",
+		"What's your name?"
+	]
+]

assets/word-buffers.json

@@ -1,16 +1,23 @@
 {
   "nounCtx": {
-	"canvas": "nounCanvas",
+    "canvas": "nounCanvas",
     "order": 1,
-	"scrollSpeed": 1,
+    "scrollSpeed": 1,
     "cursor": ["cursorX", "cursorY"],
-	"cursorOffset": [0, -20]
+    "cursorOffset": [0, -20]
   },
   "adjCtx": {
-	"canvas": "adjCanvas",
+    "canvas": "adjCanvas",
     "order": 0,
-	"scrollSpeed": 0.8,
+    "scrollSpeed": 0.8,
     "cursor": ["adjCursorX", "adjCursorY"],
-	"cursorOffset": [0, 0]
+    "cursorOffset": [0, 0]
+  },
+  "questionCtx": {
+    "canvas": "questionCanvas",
+    "order": 2,
+    "scrollSpeed": 0.2,
+    "cursor": ["cursorX", "cursorY"],
+    "cursorOffset": [0, 0]
   }
 }

assets/word-styles.json

@@ -58,5 +58,10 @@
     "buffer": "nounCtx",
     "style": ["", "", [50, 250], "Times New Roman", "", "black", "alphabetic"],
     "offsetRatio": [1.0, 0.0]
+  },
+  "custom-sentence": {
+    "buffer": "questionCtx",
+    "style": ["", "", [40, 40], "monospace", "", "black", "alphabetic"],
+    "offsetRatio": [1.0, 0.0]
   }
 }

src-words2music/audio.js

@@ -1,11 +1,31 @@
+export const audioCtx = new (window.AudioContext || window.webkitAudioContext)({
+	latencyHint: "interactive", // Use "interactive" for low latency
+}); // AudioContext for the browser
+console.log("AudioContext initialised:", audioCtx);
+
 // Some simple synthetic instruments
 export const metroSynth = new Tone.MembraneSynth().toDestination();
+export const glitchSynth = new Tone.MembraneSynth().toDestination();
+export const teleportSynth = new Tone.MonoSynth({
+	oscillator: { type: "sine" },
+	envelope: {
+		attack: 0.5,
+		decay: 0.2,
+		sustain: 0.1,
+		release: 2
+	},
+	filter: {
+		Q: 3,
+		type: "lowpass",
+		rolloff: -12
+	}
+})
 
 // Load tonejs-instruments sample library
 export let instruments = {};
 export async function loadIntsruments() {
 	instruments = await SampleLibrary.load({
-		instruments: ['piano', 'bass-electric', 'bassoon', 'cello', 'clarinet', 'contrabass', 'french-horn', 'guitar-acoustic', 'guitar-electric', 'guitar-nylon', 'harp', 'organ', 'saxophone', 'trombone', 'trumpet', 'tuba', 'violin'],
+		instruments: ['piano', 'bass-electric', 'bassoon', 'cello', 'clarinet', 'contrabass', 'french-horn', 'guitar-acoustic', 'guitar-electric', 'guitar-nylon', 'harp', 'organ', 'saxophone', 'trombone', 'trumpet', 'tuba', 'violin', 'xylophone'],
 		baseUrl: "src/tonejs-instruments/samples/",
 	})
 	console.log("SampleLibrary loaded: ", Object.keys(instruments));
@@ -15,11 +35,7 @@ export async function loadIntsruments() {
 let lastSoundTime = 0;
 
 export async function playDotSound(y, instrument, scale = null, length = "8n", volume = 0, snapTime = null) {
-	if (Tone.now() - lastSoundTime < 0.02) {
-		console.log("Too soon to play sound");
-		return;
-	}
-	lastSoundTime = Tone.now();
+	await audioCtx.resume();
 
 	// Handle frequency
 	var frequency;
@@ -68,6 +84,15 @@ export async function playDotSound(y, instrument, scale = null, length = "8n", v
 		return;
 	}
 	const time = snap2subdivision(snapTime);
+	try {
+		const lastTime = y.lastQuickAudioTime;
+		if (Math.abs(time - lastTime) < 0.01) {
+			console.log("Skipping sound: too close to last sound");
+			return;
+		}
+		y.lastQuickAudioTime = time;
+	} catch (e) {
+	}
 	instrument.triggerAttackRelease(frequency, length, time);
 }
 

src-words2music/dot.js

@@ -2,11 +2,13 @@ export class Dot {
 	constructor(x, radius) {
 		this.x = x;
 		this.y = 0;
-		this.u = 0.3;
+		this.u = 1;
+		this.u0 = 1;
 		this.v = 0;
 		this.r = radius;
 		this.g = 12;
-		this.viscosity = 4;
+		this.vViscosity = 4;
+		this.hViscosity = 2;
 		this.xPrev = this.x;
 		this.yPrev = this.y;
 		this.uPrev = this.u;
@@ -32,11 +34,17 @@ export class Dot {
 		}
 
 		// x-motion
+		if (this.inSlowMo) {
+			this.u -= this.hViscosity * this.u * deltaTime;
+			this.u = Math.max(this.u, this.u0 * 0.5);
+		} else {
+			this.u += this.hViscosity * (this.u0 - this.u) * deltaTime;
+		}
 		this.xTarget = this.x + this.u * deltaTime;
 
 		// y-motion
 		if (this.inSlowMo) {
-			this.v -= this.viscosity * this.v * deltaTime;
+			this.v -= this.vViscosity * this.v * deltaTime;
 			this.v += (0.2 * this.g) * deltaTime;
 		} else if (this.inFloatingMode) {
 			this.v += (-0.5 * this.g) * deltaTime;
@@ -78,6 +86,13 @@ export class Dot {
 		if (this.inFloatingMode || this.inSlowMo) {
 			this.x = this.xTarget;
 			this.y = this.yTarget;
+			if (collisionResult.hasCollided == 1) {
+				this.inFloatingMode = true;
+				this.inSlowMo = false;
+			} else if (collisionResult.hasCollided == 2) {
+				this.inFloatingMode = false;
+				this.inSlowMo = true;
+			}
 			return;
 		}
 
@@ -117,6 +132,8 @@ export class Dot {
 		const randomFactor = Math.random();
 		if (randomFactor < 0.7) {
 			this.v *= 1 / 2;
+		} else if (randomFactor < 0.8) {
+			this.v *= 2;
 		}
 		// console.log(`Collision detected at: x = ${this.x.toFixed(2)}, y = ${this.y.toFixed(2)}, u = ${this.u.toFixed(2)}, v = ${this.v.toFixed(2)}`);
 	}

src-words2music/scene.js

@@ -1,7 +1,7 @@
 import { Dot } from "./dot.js"
-import { loadFontStyles, addWordToCanvas, loadWordBuffers, wordBuffers } from "./words.js"
+import { loadFontStyles, addWord2canvas, loadWordBuffers, wordBuffers, loadCustomQuestions, customQuestions, printSentence2canvas } from "./words.js"
 import { playDotSound } from "./audio.js";
-import { loadIntsruments, instruments, metroSynth } from "./audio.js";
+import { loadIntsruments, instruments, metroSynth, glitchSynth, teleportSynth } from "./audio.js";
 
 let scales = {};  // where all scales will be stored
 let currentScale = [];  // active scale during playback
@@ -28,14 +28,20 @@ export class ScrollingScene {
 		this.dotXTarget = this.dot.x; // where the dot should be
 		this.clearSentenceBuffer = false;
 		this.updateScene = false;
+		this.hasPrintedSentence = false;
+		this.lastQuickAudioTime = 0;
 	}
 
 	simulate(deltaTime) {
 		if (this.wordNeedsUpdate) {
-			addWordToCanvas(this);
+			addWord2canvas(this);
 			this.wordBuffer = "";
 			this.wordNeedsUpdate = false;
 		}
+		if (!this.hasPrintedSentence) {
+			printSentence2canvas(customQuestions[0], this)
+			this.hasPrintedSentence = true;
+		}
 		this.wordsCtx.clearRect(0, 0,
 			this.wordsCtx.canvas.width, this.wordsCtx.canvas.height);
 
@@ -101,12 +107,12 @@ export class ScrollingScene {
 				console.warning("Error playing sound:", e);
 			}
 		} else if (this.dot.inSlowMo) {
-			// Slow falling mode
-			playDotSound(this.dot, new Tone.MembraneSynth().toDestination(),
-				currentScale, "4n", -20, "8n");
+			// playDotSound(this.dot, "harp",
+			// 	currentScale, "4n", -20, "8n");
+			playDotSound(this.dot, teleportSynth,
+				currentScale, "4n", -10, "8n");
 		} else if (this.dot.inFloatingMode) {
-			// Floating mode
-			playDotSound(this.dot, new Tone.MembraneSynth().toDestination(),
+			playDotSound(this.dot, glitchSynth,
 				"all", "16n", -20, "16n");
 		} else if (this.dot.hasCollided) {
 			// Collision
@@ -212,7 +218,7 @@ export async function setupScene(scene, bpm = 120) {
 	Tone.Transport.loop = false;
 
 	const loaded = await SampleLibrary.load({
-		instruments: ['piano', 'bass-electric', 'bassoon', 'cello', 'clarinet', 'contrabass', 'french-horn', 'guitar-acoustic', 'guitar-electric', 'guitar-nylon', 'harp', 'organ', 'saxophone', 'trombone', 'trumpet', 'tuba', 'violin'],
+		instruments: ['piano', 'bass-electric', 'bassoon', 'cello', 'clarinet', 'contrabass', 'french-horn', 'guitar-acoustic', 'guitar-electric', 'guitar-nylon', 'harp', 'organ', 'saxophone', 'trombone', 'trumpet', 'tuba', 'violin', 'xylophone'],
 		baseUrl: "src/tonejs-instruments/samples/",
 	})
 	Object.assign(instruments, loaded);
@@ -226,21 +232,28 @@ export async function setupScene(scene, bpm = 120) {
 	}, "4n");
 	Tone.Transport.scheduleRepeat(() => {
 		const pos = Tone.Transport.position.split('.')[0];
-		document.getElementById("metronome-container").textContent = `Metronome: ${pos}`;
+		// show on screen
+		const metronomeContainer = document.getElementById("metronome-container");
+		if (metronomeContainer) {
+			metronomeContainer.innerText = `Metronome: ${pos}`;
+		}
 	}, "8n");
 
 	await loadScales("assets/scales.json");
 	Tone.Transport.scheduleRepeat(() => {
 		const scaleName = pickRandomScale();
 		const scaleContainer = document.getElementById("scale-container");
-		scaleContainer.innerText = `Scale: ${scaleName}`;
-		console.log("New scale:", currentScale);
+		if (scaleContainer) {
+			scaleContainer.innerText = `Scale: ${scaleName}`;
+		}
 	}, "2m");
 
 	// Words
 	await loadFontStyles("assets/word-styles.json");
 	await loadWordBuffers("assets/word-buffers.json", scene);
 	scene.wordBufferNames = Object.keys(wordBuffers).sort((a, b) => wordBuffers[a].order - wordBuffers[b].order);
+	await loadCustomQuestions("assets/custom-questions.json");
+	console.log("Custom questions loaded:", customQuestions);
 	scene.sentenceBuffer = "";
 	scene.wordBuffer = "";
 	scene.wordNeedsUpdate = false;

src-words2music/words.js

@@ -40,7 +40,13 @@ export async function loadWordBuffers(src = "assets/word-buffers.json", scene) {
 
 }
 
-export function addWordToCanvas(scene, randomiseFontSize = true) {
+export let customQuestions = [];
+export async function loadCustomQuestions(src = "assets/cusotm-questions.json") {
+	const response = await fetch(src);
+	customQuestions = await response.json();
+}
+
+export function addWord2canvas(scene, randomiseFontSize = true) {
 	if (scene.wordBuffer.trim() === "") return;
 
 	const wordType = getWordType(scene.wordBuffer, scene.sentenceBuffer);
@@ -98,6 +104,49 @@ export function addWordToCanvas(scene, randomiseFontSize = true) {
 	}
 }
 
+export function printSentence2canvas(sentence, scene, wordType = "custom-sentence") {
+	const fontInfo = fontStyles[wordType];
+	const fontStyle = fontInfo["style"] || fontStyles["default"]["style"];
+	console.log("Font info:", fontInfo, fontStyle);
+	if (fontStyle[4] === "uppercase") {
+		scene.wordBuffer = scene.wordBuffer.toUpperCase();
+	} else if (fontStyle[4] === "lowercase") {
+		scene.wordBuffer = scene.wordBuffer.toLowerCase();
+	}
+
+	const fontSize = fontStyle[2][0];
+
+	const ctx = scene[fontInfo["buffer"]] || scene.wordsCtx;
+	console.log(`${fontStyle[0]} ${fontStyle[1]} ${fontSize}px ${fontStyle[3]}`)
+	ctx.font = `${fontStyle[0]} ${fontStyle[1]} ${fontSize}px ${fontStyle[3]}`;
+	ctx.fillStyle = fontStyle[5] || "black";
+	ctx.textBaseline = fontStyle[6] || "alphebatic";
+
+	// const wordWidth = ctx.measureText(scene.wordBuffer).width;
+
+	const metrics = ctx.measureText(sentence[0]);
+	const lineHeight = metrics.actualBoundingBoxAscent + metrics.actualBoundingBoxDescent;
+
+	console.log("lineHeight:", lineHeight);
+
+	var cursorY = scene.height * 0.3;
+	var maxWidth = 0;
+	for (let i = 0; i < sentence.length; i++) {
+		const phrase = sentence[i];
+		const phraseWidth = ctx.measureText(phrase).width;
+		if (phraseWidth > maxWidth) {
+			maxWidth = phraseWidth;
+		}
+	}
+	const cursorX = Math.round((scene.width - maxWidth) / 2);
+	
+	for (let i = 0; i < sentence.length; i++) {
+		const phrase = sentence[i];
+		ctx.fillText(phrase, cursorX, cursorY);
+		cursorY -= lineHeight;
+	}
+}
+
 // Find which word type a word is
 export function getWordType(word, sentence = "") {
 	if (sentence.length === 0) {

src-words2music/words2music.js

@@ -1,4 +1,5 @@
 import { ScrollingScene, setupScene } from "./scene.js"
+import { audioCtx } from "./audio.js"
 
 const inputBox = document.getElementById("input");
 const canvas = document.getElementById("canvas");
@@ -29,7 +30,6 @@ function update(timestamp) {
 		lastTimestamp = timestamp;
 	}
 	if (scene.dot.hasCollidedWithGround && !scene.metronomeStarted) {
-		Tone.start();
 		Tone.Transport.start();
 		scene.metronomeStarted = true;
 	}
@@ -62,9 +62,10 @@ function update(timestamp) {
 // Starting and stopping everything 
 const startScene = document.getElementById("startScene");
 if (startScene) {
-	startScene.addEventListener("click", () => {
+	startScene.addEventListener("click", async () => {
 		console.log("Starting scene");
-		Tone.start();
+		await Tone.start();
+		await audioCtx.resume();
 		scene.updateScene = !scene.updateScene;
 		if (scene.updateScene) {
 			startScene.value = "Pause";