From dfb4c93e4e400e48482e4d8ede80941ddbf775be Mon Sep 17 00:00:00 2001 From: Nicole Tietz-Sokolskaya Date: Fri, 31 Jan 2025 14:48:53 -0500 Subject: [PATCH] implement basic typing via midi --- Cargo.toml | 4 +- doc/typing.md | 61 +++++++++++++++++ src/bin/main.rs | 57 +++++++++++----- src/lib.rs | 1 + src/midi.rs | 13 ++++ src/parser.rs | 4 +- src/typing.rs | 169 ++++++++++++++++++++++++++++++++++++++++++++++++ src/ui.rs | 2 +- 8 files changed, 291 insertions(+), 20 deletions(-) create mode 100644 doc/typing.md create mode 100644 src/typing.rs diff --git a/Cargo.toml b/Cargo.toml index b6add1d..41522b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,8 @@ edition = "2021" [dependencies] anyhow = "1.0.86" crossbeam = "0.8.4" -eframe = "0.29.1" -egui = "0.29.1" +eframe = "0" +egui = "0" enigo = { version = "0.2.1", features = ["serde"] } hex = "0.4.3" midir = "0.10.0" diff --git a/doc/typing.md b/doc/typing.md new file mode 100644 index 0000000..2ad48b7 --- /dev/null +++ b/doc/typing.md @@ -0,0 +1,61 @@ + +# Typing system design + + +This document talks about how data flows through the system, and how it's +displayed, so that users can enter text using a MIDI device. + +There are a few distinct pieces: +- **data flow**: how do we get from MIDI to the display and to the typing portion? +- **encoding**: how do we represent text so that we can enter it? +- **parsing**: how do we read a MIDI stream to convert it into our text encoding? +- **display**: how do we present the state to a user so that they can see what they're doing? + + +## Data flow and architecture + +We already have the ability for data to flow in, and for devices to be added +and removed. What we need here is another piece that will keep track of state +for the typing daemon. + +The typing daemon will be a state machine, with state transitions for entering +new bits and for emitting text (which will then move back to a clear state). + +It will be subscribed to all incoming MIDI messages, and it will store a +separate state for each connection ID, so things you enter on the drums will +not alter what you're entering on the wind synth, for example. + +It will emit text directly, via `enigo` (or, in debug mode, to the console). +And it will emit its current state on another queue, which can be used to +display the characters which can be typed. + + +## Encoding + +Each MIDI device will be in a given *base*. Wind synth will be in base 11 by +default, so we can read the semitone and ignore which octave it's in. We could +expand this to use a wider range, or narrow it to belong to a particular key, +but this is the starting point. Drums will be in base 2 by default, with the +snare and kick drum providing the bits and everything else being ignored. + +We will support a given range of characters, and some modifiers as well. Other +keys can be added, but this is where we'll start: just text! the characters +supported initially will be 'a'..'z', 'A'..'Z', '0'..'9', '.', and ','. + +Each of these will be assigned a number, which will then be converted to base 2 +for entry. + + + +## Display + +Given a list of + + + + + + + + + diff --git a/src/bin/main.rs b/src/bin/main.rs index de8176d..1a4f07c 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -4,20 +4,17 @@ use std::{ }; use anyhow::{anyhow, Result}; +use enigo::{Enigo, Keyboard, Settings}; use midi_keys::{ log::load_raw_log, midi::daemon::Category, parser::parse_message, + typing::{base_values, encode, midi_to_scale_degree, TypingState}, ui::{display_state_daemon, DisplayQueues, DisplayState}, }; -//use enigo::{Direction, Enigo, Key, Keyboard, Settings}; use midir::{MidiInput, MidiInputPort}; fn main() { - //let mut enigo = Enigo::new(&Settings::default()).unwrap(); - //enigo.text("echo \"hello world\"").unwrap(); - //enigo.key(Key::Return, Direction::Press).unwrap(); - let (dbg_send, dbg_recv) = crossbeam::channel::unbounded(); let (ws_send, ws_recv) = crossbeam::channel::unbounded(); let (drum_send, drum_recv) = crossbeam::channel::unbounded(); @@ -37,16 +34,46 @@ fn main() { let _handle = display_state_daemon(queues, state.clone()); println!("started daemon"); - std::thread::spawn(move || loop { - match dbg_recv.recv() { - Ok(m) => println!("debug: {m:?}"), - Err(err) => println!("err: {err:?}"), - } - }); - std::thread::spawn(move || loop { - match ws_recv.recv() { - Ok(m) => println!("windsynth: {m:?}"), - Err(err) => println!("err(ws): {err:?}"), + for c in "hello, world".chars() { + let v = encode(c).unwrap(); + println!("{c}: {:?} - {:?}", v, base_values(v, 7, 3)); + } + for c in "Hello, WORLD!".chars() { + let v = encode(c).unwrap(); + println!("{c}: {:?} - {:?}", v, base_values(v, 7, 3)); + } + + //std::thread::spawn(move || loop { + // match dbg_recv.recv() { + // Ok(m) => println!("debug: {m:?}"), + // Err(err) => println!("err: {err:?}"), + // } + //}); + std::thread::spawn(move || { + let mut steps = 0; + let mut state = TypingState::new(7); + let mut enigo = Enigo::new(&Settings::default()).unwrap(); + loop { + match ws_recv.recv() { + Ok((_cid, _ts, m)) => { + if let Some((note, _velocity)) = m.note_on_values() { + if let Some(degree) = midi_to_scale_degree(60, note) { + steps += 1; + println!("got degree: {degree}, steps: {steps}, state: {state:?}"); + state.enter(degree); + + if steps % 3 == 0 { + let (b, c) = state.emit(); + println!("got {b}, {c:?}"); + if let Some(c) = c { + enigo.text(&format!("{c}")); + } + } + } + } + } + Err(err) => println!("err(ws): {err:?}"), + } } }); std::thread::spawn(move || loop { diff --git a/src/lib.rs b/src/lib.rs index 7920f79..6f018a0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ pub mod log; pub mod midi; pub mod parser; +pub mod typing; pub mod ui; diff --git a/src/midi.rs b/src/midi.rs index 2ed2725..a67891c 100644 --- a/src/midi.rs +++ b/src/midi.rs @@ -10,6 +10,19 @@ impl Message { pub fn new(raw: Vec, parsed: ParsedMessage) -> Self { Message { raw, parsed } } + + /// Extracts the note value and velocity for any NoteOn messages. + /// This is a convenience function to let us avoid a match in all the places + /// we need to extract this. + pub fn note_on_values(&self) -> Option<(u8, u8)> { + match &self.parsed { + ParsedMessage::Voice(VoiceMessage { + category: VoiceCategory::NoteOn { note, velocity }, + .. + }) => Some((*note, *velocity)), + _ => None, + } + } } #[derive(PartialEq, Eq, Debug, Clone)] diff --git a/src/parser.rs b/src/parser.rs index 3e63a22..76784dd 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -149,11 +149,11 @@ mod tests { let note = 0x24; let velocity = 0x51; - let expected = Message::Voice(VoiceMessage::new( + let expected = ParsedMessage::Voice(VoiceMessage::new( VoiceCategory::NoteOn { note, velocity }, 0, )); - assert_eq!(parsed, expected,); + assert_eq!(parsed.parsed, expected,); assert_eq!(remainder.len(), 0); } diff --git a/src/typing.rs b/src/typing.rs new file mode 100644 index 0000000..8e83046 --- /dev/null +++ b/src/typing.rs @@ -0,0 +1,169 @@ +#[derive(Debug, Copy, Clone)] +pub struct TypingState { + base: u8, + current: u8, +} + +impl TypingState { + pub fn new(base: u8) -> TypingState { + TypingState { base, current: 0 } + } + + /// Add the value into the current accumulator. + pub fn enter(&mut self, value: u8) { + self.current = self.current.saturating_mul(self.base).saturating_add(value); + } + + /// Decode the current accumulator as a character, or None if it's invalid. + /// This resets the accumulator to 0! + pub fn emit(&mut self) -> (u8, Option) { + let x = self.current; + let c = decode(x); + + self.current = 0; + + (x, c) + } +} + +/// Chunks a u8 into a sequence of values in a given base. +pub fn base_values(mut x: u8, base: u8, len: usize) -> Vec { + let mut vals = vec![]; + for _ in 0..len { + vals.push(x % base); + x = x / base; + } + vals.reverse(); + + vals +} + +/// Determines the scale degree of a given MIDI note relative to the provided +/// scale root. +pub fn midi_to_scale_degree(root: u8, note: u8) -> Option { + let semitones = (note as i32 - root as i32).rem_euclid(12); + Some(match semitones { + 0 => 0, + 2 => 1, + 4 => 2, + 5 => 3, + 7 => 4, + 9 => 5, + 11 => 6, + _ => return None, + }) +} + +/// Converts a character to its assigned number for use in text entry. This is +/// stores them as u8 to keep things simple and fixed length. If we want to +/// support the full Unicode space, we can have expand this to u32. +pub fn encode(c: char) -> Option { + Some(match c { + 'a'..='z' => c as u8 - 'a' as u8, + 'A'..='Z' => c as u8 - 'A' as u8 + 26, + '0'..='9' => c as u8 - '0' as u8 + 26 * 2, + '.' => 62, + ',' => 63, + '!' => 64, + ' ' => 65, + _ => return None, + }) +} + +/// Converts a number to its corresponding character for use in text entry. +/// If a character isn't implemented, it will return None. +pub fn decode(x: u8) -> Option { + let c = match x { + 0..=25 => 'a' as u8 + x, + 26..=51 => 'A' as u8 + x - 26, + 52..=61 => '0' as u8 + x - 26 * 2, + 62 => '.' as u8, + 63 => ',' as u8, + 64 => '!' as u8, + 65 => ' ' as u8, + _ => return None, + }; + Some(char::from(c)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encodes_lowercase() { + assert_eq!(encode('a'), Some(0)); + assert_eq!(encode('b'), Some(1)); + assert_eq!(encode('z'), Some(25)); + } + + #[test] + fn encodes_uppercase() { + assert_eq!(encode('A'), Some(26)); + assert_eq!(encode('B'), Some(27)); + assert_eq!(encode('Z'), Some(51)); + } + + #[test] + fn encodes_numbers() { + assert_eq!(encode('0'), Some(52)); + assert_eq!(encode('1'), Some(53)); + assert_eq!(encode('9'), Some(61)); + } + + #[test] + fn encodes_punctuation() { + assert_eq!(encode('.'), Some(62)); + assert_eq!(encode(','), Some(63)); + assert_eq!(encode('!'), Some(64)); + } + + #[test] + fn decodes_lowercase() { + assert_eq!(decode(0), Some('a')); + assert_eq!(decode(1), Some('b')); + assert_eq!(decode(25), Some('z')); + } + + #[test] + fn decodes_uppercase() { + assert_eq!(decode(26), Some('A')); + assert_eq!(decode(27), Some('B')); + assert_eq!(decode(51), Some('Z')); + } + + #[test] + fn decodes_numbers() { + assert_eq!(decode(52), Some('0')); + assert_eq!(decode(53), Some('1')); + assert_eq!(decode(61), Some('9')); + } + + #[test] + fn decodes_punctuation() { + assert_eq!(decode(62), Some('.')); + assert_eq!(decode(63), Some(',')); + assert_eq!(decode(64), Some('!')); + } + + #[test] + fn converts_to_scale_degree() { + assert_eq!(midi_to_scale_degree(60, 58), None); + assert_eq!(midi_to_scale_degree(60, 59), Some(6)); + assert_eq!(midi_to_scale_degree(60, 60), Some(0)); + assert_eq!(midi_to_scale_degree(60, 61), None); + assert_eq!(midi_to_scale_degree(60, 62), Some(1)); + assert_eq!(midi_to_scale_degree(60, 63), None); + assert_eq!(midi_to_scale_degree(60, 64), Some(2)); + assert_eq!(midi_to_scale_degree(60, 65), Some(3)); + assert_eq!(midi_to_scale_degree(60, 66), None); + assert_eq!(midi_to_scale_degree(60, 67), Some(4)); + assert_eq!(midi_to_scale_degree(60, 68), None); + assert_eq!(midi_to_scale_degree(60, 69), Some(5)); + assert_eq!(midi_to_scale_degree(60, 70), None); + assert_eq!(midi_to_scale_degree(60, 71), Some(6)); + assert_eq!(midi_to_scale_degree(60, 72), Some(0)); + assert_eq!(midi_to_scale_degree(60, 73), None); + assert_eq!(midi_to_scale_degree(60, 74), Some(1)); + } +} diff --git a/src/ui.rs b/src/ui.rs index b5bc571..072ae96 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -232,7 +232,7 @@ impl eframe::App for MidiKeysApp { frame.content_ui.label(RichText::new(port_name).strong()); - display_midi_message(idx, msg, &mut frame.content_ui, show_raw); + //display_midi_message(idx, msg, &mut frame.content_ui, show_raw); frame.end(ui); } });