Add mic mode (beta): play without MIDI via microphone pitch detection

ROADMAP.md

@@ -77,12 +77,15 @@ dans le `CHANGELOG`.
   pour le confirmer ; sans lecture de confirmation à temps (ou si elle se passe
   mal), le morceau redescend en perfectionnement.
 
-- **Mode micro (non MIDI)** — détecter les notes jouées via le microphone pour
-  les pianos acoustiques et claviers sans MIDI. Ouvrirait l'application à un
-  public beaucoup plus large. Grande inconnue : la qualité atteignable de la
-  détection de pitch, surtout en polyphonie (accords, deux mains) et en
-  conditions réelles (micro de laptop, acoustique de la pièce). À prototyper
-  pour évaluer la faisabilité avant d'en faire un vrai chantier.
+- **Mode micro : polyphonie** — le prototype monophonique est livré (bouton 🎤
+  sur la page partition : détection de pitch dans le navigateur via
+  [pitchy](https://github.com/ianprime0509/pitchy) (McLeod Pitch Method), une
+  note à la fois — mélodies et travail mains séparées). La grande inconnue
+  reste entière pour la polyphonie (accords, deux mains), qui demande une
+  autre approche que le DSP classique — l'état de l'art navigateur est un
+  modèle de transcription type Onsets and Frames (Magenta.js) ou Basic Pitch
+  (Spotify), pensés pour l'offline plus que le temps réel — et à évaluer en
+  conditions réelles (micro de laptop, acoustique de la pièce).
 
 - **App iOS (wrapper natif)** — même objectif d'élargir le public, mais plus
   simple et balisé que le mode micro : Safari/iOS ne supporte pas le Web MIDI,

public/js/app.js

@@ -1,4 +1,5 @@
 import { initMidi } from './midi.js'
+import { initMicInput } from './micInput.js'
 import { initMusicXML } from './musicxml.js'
 import { initFingeringEditor } from './fingeringEditor.js'
 import { initCassettes } from './cassettes.js'
@@ -20,6 +21,7 @@ const CHART_DATE_AXIS = new Intl.DateTimeFormat(locale(), { day: 'numeric', mont
 
 export function midiApp() {
   const midi = initMidi()
+  const micInput = initMicInput()
   const musicxml = initMusicXML()
   const fingeringEditor = initFingeringEditor({
     getOsmdInstance: musicxml.getOsmdInstance,
@@ -38,6 +40,7 @@ export function midiApp() {
     ...headerMenu(),
     bluetoothConnected: false,
     midiDeviceName: null,
+    micActive: false,
     osmdInstance: null,
     isRecording: false,
     isReplaying: false,
@@ -239,6 +242,14 @@ export function midiApp() {
       }
     },
 
+    // Mic mode (non-MIDI): detected notes are turned into synthetic MIDI
+    // messages and fed through parseMidiMessage, like cassette replay.
+    async toggleMic() {
+      if (micInput.state.micActive) micInput.stop()
+      else await micInput.start((data) => midi.parseMidiMessage(data))
+      this.micActive = micInput.state.micActive
+    },
+
     detectedOS() {
       const ua = navigator.userAgent
       if (/Mac/.test(ua)) return 'mac'

public/js/changelog.js

@@ -15,6 +15,17 @@
 // See CLAUDE.md ("Changelog in-app") for the update rule.
 
 export const CHANGELOG = [
+  {
+    date: '2026-07-06',
+    items: {
+      fr: [
+        "Mode micro (bêta) : pas de clavier MIDI ? Sur la page partition, le bouton 🎤 fait écouter votre piano par le microphone et valide les notes jouées. Une note à la fois pour l'instant — idéal pour les mélodies et le travail mains séparées ; les accords ne sont pas encore reconnus.",
+      ],
+      en: [
+        "Mic mode (beta): no MIDI keyboard? On the score page, the 🎤 button lets the app listen to your piano through the microphone and validate the notes you play. One note at a time for now — great for melodies and hands-separate practice; chords aren't recognised yet.",
+      ],
+    },
+  },
   {
     date: '2026-06-24',
     items: {

public/js/locales/en.js

@@ -125,6 +125,9 @@ export default {
     midiConnected: 'MIDI keyboard connected',
     midiKeyboard: 'MIDI keyboard',
     connectMidi: '🎹 Connect MIDI keyboard',
+    micButton: '🎤 Mic mode',
+    micActive: '🎤 Listening…',
+    micTitle: 'Mic mode (beta): note detection through the microphone, one note at a time — chords are not recognised',
     history: '📜 History',
     historyAria: 'History',
     modeGroupAria: 'Practice mode',
@@ -209,6 +212,9 @@ export default {
     windowsStep3: 'Set up BLE MIDI Bridge to route to the loopMIDI port',
     windowsReturn: 'Once the bridge is set up, come back to this page and click “Connect MIDI keyboard”.',
     linuxBody: 'Connect your MIDI keyboard over USB or set up the Bluetooth MIDI connection via BlueZ. Then come back to this page and click “Connect MIDI keyboard”.',
+    micHelpAlt:
+      'Acoustic piano or no MIDI? Try mic mode (beta): the app listens to your piano through the microphone and validates the notes you play, one note at a time (chords are not recognised yet).',
+    midiHelpTryMic: '🎤 Try mic mode (beta)',
     midiHelpRetry: 'Retry the connection',
     resultTitleStrict: '⏱ Strict playthrough finished',
     resultTitleTraining: '🎉 Congratulations — Training finished',
@@ -227,6 +233,8 @@ export default {
     invalidMusicXml: 'This file does not appear to be a valid MusicXML file',
     musicXmlLoad: 'Error loading the MusicXML file',
     cassetteSave: 'Error saving the cassette',
+    micUnsupported: 'Microphone access is not supported by this browser',
+    micDenied: 'Could not access the microphone. Check your browser permissions.',
   },
   landing: {
     pageTitle: 'Piano Trainer · Learn the piano with real-time feedback',

public/js/locales/fr.js

@@ -125,6 +125,10 @@ export default {
     midiConnected: 'Clavier MIDI connecté',
     midiKeyboard: 'Clavier MIDI',
     connectMidi: '🎹 Connecter clavier MIDI',
+    micButton: '🎤 Mode micro',
+    micActive: '🎤 Écoute en cours…',
+    micTitle:
+      'Mode micro (bêta) : détection des notes via le microphone, une note à la fois — les accords ne sont pas reconnus',
     history: '📜 Historique',
     historyAria: 'Historique',
     modeGroupAria: 'Mode de pratique',
@@ -209,6 +213,9 @@ export default {
     windowsStep3: 'Configurez BLE MIDI Bridge pour rediriger vers le port loopMIDI',
     windowsReturn: 'Une fois le pont configuré, revenez sur cette page et cliquez sur « Connecter clavier MIDI ».',
     linuxBody: 'Connectez votre clavier MIDI via USB ou configurez la connexion Bluetooth MIDI via BlueZ. Revenez ensuite sur cette page et cliquez sur « Connecter clavier MIDI ».',
+    micHelpAlt:
+      "Piano acoustique ou pas de MIDI ? Essayez le mode micro (bêta) : l'application écoute votre piano via le microphone et valide les notes jouées, une note à la fois (les accords ne sont pas encore reconnus).",
+    midiHelpTryMic: '🎤 Essayer le mode micro (bêta)',
     midiHelpRetry: 'Réessayer la connexion',
     resultTitleStrict: '⏱ Playthrough strict terminé',
     resultTitleTraining: '🎉 Félicitations — Entraînement terminé',
@@ -227,6 +234,8 @@ export default {
     invalidMusicXml: 'Ce fichier ne semble pas être un fichier MusicXML valide',
     musicXmlLoad: 'Erreur lors du chargement du fichier MusicXML',
     cassetteSave: 'Erreur lors de la sauvegarde de la cassette',
+    micUnsupported: "L'accès au microphone n'est pas supporté par ce navigateur",
+    micDenied: "Impossible d'accéder au microphone. Vérifiez les permissions du navigateur.",
   },
   landing: {
     pageTitle: 'Piano Trainer · Apprenez le piano avec un retour en temps réel',

public/js/micInput.js

@@ -0,0 +1,90 @@
+// Mic mode (non-MIDI input) — prototype.
+//
+// Listens to the microphone, detects the played pitch (pitchDetection.js)
+// and emits synthetic MIDI Note On/Off messages through the provided
+// callback — the same path cassette replay uses — so validation, training,
+// strict mode and recording all work unchanged.
+
+import { detectPitch, freqToMidi, computeRms, createNoteTracker } from './pitchDetection.js'
+import { NOTE_ON, NOTE_OFF } from './midi.js'
+import { t } from './i18n.js'
+
+// 4096 samples ≈ 93 ms at 44.1 kHz — long enough to resolve the lowest
+// piano strings, short enough to keep note-on latency playable.
+const FFT_SIZE = 4096
+const FRAME_MS = 40
+
+const state = {
+  micActive: false,
+}
+
+let audioContext = null
+let mediaStream = null
+let frameTimer = null
+let tracker = null
+
+export function initMicInput() {
+  return { start, stop, state }
+}
+
+async function start(emitMidiMessage) {
+  if (state.micActive) return true
+
+  if (!navigator.mediaDevices?.getUserMedia) {
+    alert(t('errors.micUnsupported'))
+    return false
+  }
+
+  try {
+    // Voice-call processing (echo cancellation & co) eats piano partials —
+    // ask for the raw signal.
+    mediaStream = await navigator.mediaDevices.getUserMedia({
+      audio: { echoCancellation: false, noiseSuppression: false, autoGainControl: false },
+    })
+  } catch (e) {
+    console.error('Accès micro refusé:', e)
+    alert(t('errors.micDenied'))
+    return false
+  }
+
+  audioContext = new AudioContext()
+  const analyser = audioContext.createAnalyser()
+  analyser.fftSize = FFT_SIZE
+  audioContext.createMediaStreamSource(mediaStream).connect(analyser)
+
+  tracker = createNoteTracker({
+    onNoteOn: (midiNote, rms) => emitMidiMessage([NOTE_ON, midiNote, velocityFromRms(rms)]),
+    onNoteOff: (midiNote) => emitMidiMessage([NOTE_OFF, midiNote, 0]),
+  })
+
+  const samples = new Float32Array(FFT_SIZE)
+  frameTimer = setInterval(() => {
+    analyser.getFloatTimeDomainData(samples)
+    const frequency = detectPitch(samples, audioContext.sampleRate)
+    tracker.push({ midi: frequency === null ? null : freqToMidi(frequency), rms: computeRms(samples) })
+  }, FRAME_MS)
+
+  state.micActive = true
+  console.log('Micro connecté (mode micro)')
+  return true
+}
+
+function stop() {
+  if (!state.micActive) return
+  clearInterval(frameTimer)
+  frameTimer = null
+  tracker.flush() // release any held note so validation doesn't hang
+  tracker = null
+  mediaStream.getTracks().forEach((track) => track.stop())
+  mediaStream = null
+  audioContext.close()
+  audioContext = null
+  state.micActive = false
+  console.log('Micro déconnecté')
+}
+
+// The mic gives loudness, not key velocity — a rough monotonic mapping is
+// enough (velocity only feeds recordings, validation ignores it).
+function velocityFromRms(rms) {
+  return Math.max(20, Math.min(110, Math.round(Math.sqrt(rms) * 300)))
+}

public/js/pitchDetection.js

@@ -0,0 +1,107 @@
+// Pitch detection for the mic mode (non-MIDI input) — prototype.
+//
+// No Web Audio dependency, so everything here is unit-testable in node.
+// micInput.js owns the browser glue (getUserMedia, AnalyserNode) and feeds
+// audio frames into these functions.
+//
+// Monophonic only: MPM finds a single fundamental per frame. Polyphony
+// (chords, two hands) needs a different approach entirely (e.g. an Onsets
+// and Frames-style transcription model) — see ROADMAP.
+
+import { PitchDetector } from '../vendor/pitchy.4.1.0.bundle.min.js'
+
+// One detector per buffer size (it preallocates FFT scratch space).
+const detectors = new Map()
+
+// Wraps pitchy's McLeod Pitch Method detector. Returns the detected
+// fundamental in Hz, or null when the detection isn't confident enough or
+// falls outside the accepted range — for note validation, a miss is much
+// cheaper than a false positive.
+//
+// maxFreq stops at ~C7: mic detection of the top piano octave is unreliable,
+// and staying below C8 keeps a stray high detection from ever hitting the
+// app's C8 navigate-back key.
+export function detectPitch(buffer, sampleRate, { minFreq = 27.5, maxFreq = 2200, minClarity = 0.9 } = {}) {
+  let detector = detectors.get(buffer.length)
+  if (!detector) {
+    detector = PitchDetector.forFloat32Array(buffer.length)
+    detectors.set(buffer.length, detector)
+  }
+  const [frequency, clarity] = detector.findPitch(buffer, sampleRate)
+  if (clarity < minClarity || frequency < minFreq || frequency > maxFreq) return null
+  return frequency
+}
+
+export function freqToMidi(frequency) {
+  return Math.round(69 + 12 * Math.log2(frequency / 440))
+}
+
+export function computeRms(buffer) {
+  let sum = 0
+  for (let i = 0; i < buffer.length; i++) sum += buffer[i] * buffer[i]
+  return Math.sqrt(sum / buffer.length)
+}
+
+// Turns a per-frame stream of { midi, rms } observations into debounced
+// Note On / Note Off events. A note fires after minOnFrames consecutive
+// frames of the same pitch (kills single-frame glitches) and releases after
+// offFrames frames where that pitch is no longer heard (decay, damper, or
+// another note taking over). Monophonic: confirming a new note releases the
+// previous one.
+export function createNoteTracker({ onNoteOn, onNoteOff, minOnFrames = 2, offFrames = 3, minRms = 0.01 } = {}) {
+  let activeNote = null
+  let candidate = null
+  let candidateCount = 0
+  let missCount = 0
+
+  function push({ midi, rms }) {
+    const heard = rms >= minRms && midi != null ? midi : null
+
+    if (heard !== null && heard === activeNote) {
+      missCount = 0
+      candidate = null
+      candidateCount = 0
+      return
+    }
+
+    if (heard === null) {
+      candidate = null
+      candidateCount = 0
+    } else if (heard === candidate) {
+      candidateCount++
+    } else {
+      candidate = heard
+      candidateCount = 1
+    }
+
+    if (candidate !== null && candidateCount >= minOnFrames) {
+      if (activeNote !== null) onNoteOff(activeNote)
+      activeNote = candidate
+      candidate = null
+      candidateCount = 0
+      missCount = 0
+      onNoteOn(activeNote, rms)
+      return
+    }
+
+    if (activeNote !== null) {
+      missCount++
+      if (missCount >= offFrames) {
+        onNoteOff(activeNote)
+        activeNote = null
+        missCount = 0
+      }
+    }
+  }
+
+  // Release whatever is held — called when the mic is switched off.
+  function flush() {
+    if (activeNote !== null) onNoteOff(activeNote)
+    activeNote = null
+    candidate = null
+    candidateCount = 0
+    missCount = 0
+  }
+
+  return { push, flush }
+}

public/score.html

@@ -79,6 +79,16 @@
           x-text="$t('score.connectMidi')"
         >🎹 Connecter clavier MIDI</button>
 
+        <!-- Mic mode (non MIDI) — hidden once a MIDI keyboard is connected -->
+        <button
+          class="pt-midi-status"
+          :class="{ 'is-disconnected': !micActive }"
+          x-show="micActive || !bluetoothConnected"
+          @click="toggleMic"
+          :title="$t('score.micTitle')"
+          x-text="micActive ? $t('score.micActive') : $t('score.micButton')"
+        >🎤 Mode micro</button>
+
         <!-- History (only for library scores) -->
         <button
           class="pt-topbar__pill"
@@ -467,7 +477,10 @@ <h4>Linux</h4>
           </div>
         </template>
 
+        <p x-text="$t('score.micHelpAlt')">Piano acoustique ou pas de MIDI ? Essayez le mode micro (bêta) : l'application écoute votre piano via le microphone et valide les notes jouées, une note à la fois.</p>
+
         <footer>
+          <button class="pt-action-button" @click="showMidiHelpModal = false; toggleMic()" x-text="$t('score.midiHelpTryMic')">🎤 Essayer le mode micro (bêta)</button>
           <button class="pt-action-button is-primary" @click="showMidiHelpModal = false; connectMIDI()" x-text="$t('score.midiHelpRetry')">Réessayer la connexion</button>
         </footer>
       </article>