diff --git a/Cargo.lock b/Cargo.lock index de1ac492..92e6d8b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -807,8 +807,8 @@ dependencies = [ "crossbeam-queue", "derive_more", "fixhandle", + "fixparser", "fixpostprocessor", - "fixprocedure", "fixshell", "futures", "hex", @@ -833,20 +833,21 @@ dependencies = [ ] [[package]] -name = "fixpostprocessor" +name = "fixparser" version = "0.1.0" dependencies = [ - "anyhow", - "wasm-encoder", - "wasmparser", + "dlmalloc", + "fixutils", + "hex", ] [[package]] -name = "fixprocedure" +name = "fixpostprocessor" version = "0.1.0" dependencies = [ - "dlmalloc", - "fixutils", + "anyhow", + "wasm-encoder", + "wasmparser", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 604fc715..85a44fb3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [ "common", "vmm", "kernel", "macros" , "user", "arca" , "arcane", "fix", "fix/handle", "fix/shell" ] +members = [ "common", "vmm", "kernel", "macros" , "user", "arca" , "arcane", "fix", "fix/handle", "fix/shell", "fix/parser" ] default-members = [ "common", "vmm", "macros", "arca", "fix/handle" ] resolver = "2" diff --git a/addblob.fix b/addblob.fix index 0ce79428..f99e553b 100644 --- a/addblob.fix +++ b/addblob.fix @@ -1,2 +1,2 @@ -(let ((add @"./target/x86_64-unknown-none/addblob")) - !*(add !*(add 2 3) 1)) \ No newline at end of file +(let ((add @"./target/x86_64-unknown-none/debug/addblob")) + *#(add *#(add 2u64 3u64) 1u64)) \ No newline at end of file diff --git a/addblob_extended.fix b/addblob_extended.fix index f221d6d6..12b89109 100644 --- a/addblob_extended.fix +++ b/addblob_extended.fix @@ -1,27 +1,26 @@ -(let ((add @"./target/x86_64-unknown-none/addblob") - (a 1) (b 2) (c 3) (d 4) (e 5) (f 6) (g 7) - (h 8) (i 9) (j 10) (k 11) (l 12) (m 13) - (n 14) (o 15) (p 16)) - !*(add - !*(add - !*(add - !*(add a b) - !*(add c d) +(let ((add @"./target/x86_64-unknown-none/debug/addblob") + (a 1u64) (b 2u64) (c 3u64) (d 4u64) (e 5u64) (f 6u64) (g 7u64) (h 8u64) + (i 9u64) (j 10u64) (k 11u64) (l 12u64) (m 13u64) (n 14u64) (o 15u64) (p 16u64)) + *#(add + *#(add + *#(add + *#(add a b) + *#(add c d) ) - !*(add - !*(add e f) - !*(add g h) + *#(add + *#(add e f) + *#(add g h) ) ) - !*(add - !*(add - !*(add i j) - !*(add k l) + *#(add + *#(add + *#(add i j) + *#(add k l) ) - !*(add - !*(add m n) - !*(add o p) + *#(add + *#(add m n) + *#(add o p) ) ) ) -) +) \ No newline at end of file diff --git a/demo.fix b/demo.fix index 2aa9a39c..f7c04eb9 100644 --- a/demo.fix +++ b/demo.fix @@ -2,33 +2,40 @@ -- The whole file is interpreted as a single expression. Expressions can be comprised of: -- Integer Literals - 42 + 42u8 -- String Literals "hello" -- Trees - (1 2 3) + (1u8 2u8 3u8) -- References &"hello" -- Identification Thunks - ^&4 + '&4u8 -- Application Thunks - *(1 2) + #(1u8 2u8) + + -- Selection Thunks + ["hello" 2u8] -- Zero-indexed + ["hello" 1u8 3u8] -- Inclusive start and exclusive end -- Strict Encodes - !^&2 + *'&2u8 + + -- Shallow Encodes + +'&2u8 -- Let bindings that tie expressions to variables - (let ((x 42)) x) + (let ((x 42u8)) x) -- @ path executables @"./target/x86_64-unknown-none/addblob" - -- Primitives (To be implemented) - $interpret + -- Primitives + *#($identity 2u8) -} \ No newline at end of file diff --git a/fix/Cargo.toml b/fix/Cargo.toml index 2d5bc7b9..54cb074e 100644 --- a/fix/Cargo.toml +++ b/fix/Cargo.toml @@ -49,7 +49,7 @@ crossbeam-queue = { [build-dependencies] fixshell = { path = "shell", artifact="staticlib", target = "x86_64-unknown-none" } fixpostprocessor = { path = "postprocessor" } -fixprocedure = { path = "procedure", artifact = "cdylib", target = "wasm32-unknown-unknown" } +fixparser = { path = "parser", artifact = "cdylib", target = "wasm32-unknown-unknown" } anyhow = "1.0.98" bindgen = "0.72.1" cc = "1.2.30" diff --git a/fix/build.rs b/fix/build.rs index 8c230119..0a01fa6a 100644 --- a/fix/build.rs +++ b/fix/build.rs @@ -213,6 +213,10 @@ fn main() -> Result<()> { symlink(dst, link)?; } } + let parser = PathBuf::from(env::var_os("CARGO_CDYLIB_FILE_FIXPARSER").unwrap()); + let wasm = fixpostprocessor::process(&std::fs::read(&parser)?)?; + let (c, h) = wasm2c(&wasm)?; + std::fs::write(Path::new(&out_dir).join("fixparser"), c2elf(&c, &h)?)?; let cwd = std::env::var("CARGO_MANIFEST_DIR").unwrap(); diff --git a/fix/etc/memmap.ld b/fix/etc/memmap.ld index 215a1f24..25a41548 100644 --- a/fix/etc/memmap.ld +++ b/fix/etc/memmap.ld @@ -2,7 +2,7 @@ ENTRY(_start) MEMORY { cls (rwx) : ORIGIN = 0x0000000000000000, LENGTH = 2M - upper (rw) : ORIGIN = 0xFFFF800100100000, LENGTH = 8M + upper (rw) : ORIGIN = 0xFFFF800100100000, LENGTH = 32M } SECTIONS { diff --git a/fix/parser/Cargo.toml b/fix/parser/Cargo.toml new file mode 100644 index 00000000..05c11dab --- /dev/null +++ b/fix/parser/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "fixparser" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +fixutils = { path = "../utils" } +dlmalloc = { version = "0.2.14", features = ["global"] } +hex = { version = "0.4.3", default-features = false, features = ["alloc"] } \ No newline at end of file diff --git a/fix/parser/src/grammar.md b/fix/parser/src/grammar.md new file mode 100644 index 00000000..af1c2e90 --- /dev/null +++ b/fix/parser/src/grammar.md @@ -0,0 +1,47 @@ +### Text Format + +``` +A ::= O | K | E Any + | name Identifier + | $name Stdlib primitive + | @"/path/" Executable path + +binding ::= name = A Binds identifier to a value + +int ::= num_u8 Integer + | num_u16 + | num_u32 + | num_u64 + | num_u128 + +B ::= int Blob + | string + +T ::= (A*) Tree + +R ::= &B | &T Ref + +O ::= B | T | R Data + +K ::= 'O Thunk (Identify) + | #T Thunk (Apply) + | [A*] Thunk (Select) + | [O int] Thunk (Select 1) + | [O int int] Thunk (Select N) + +E ::= *K Encode (Strict) + | +K Encode (Shallow) + +comment ::= -- Single line + | {- -} Multi-line +``` + +### Example + +``` +compiler = @"/path/" + +*#(*#(compiler $def_limits + "int f(int x, int y) {return x + y;}") + $def_limits 19u32 4u32) +``` diff --git a/fix/src/parser/lexer.rs b/fix/parser/src/lexer.rs similarity index 64% rename from fix/src/parser/lexer.rs rename to fix/parser/src/lexer.rs index 188479ae..8b4f2523 100644 --- a/fix/src/parser/lexer.rs +++ b/fix/parser/src/lexer.rs @@ -1,6 +1,10 @@ -use super::token::Token; +use crate::token::Token; +use alloc::{ + format, + string::{String, ToString}, + vec::Vec, +}; use core::{iter::Peekable, str::Chars}; -use kernel::prelude::*; pub struct Lexer<'a> { characters: Peekable>, @@ -36,10 +40,14 @@ impl<'a> Lexer<'a> { let token = match character { '(' => Token::LParen, ')' => Token::RParen, + '[' => Token::LBracket, + ']' => Token::RBracket, '&' => Token::Ampersand, '*' => Token::Asterisk, - '^' => Token::Caret, - '!' => Token::Bang, + '+' => Token::Plus, + '\'' => Token::Apostrophe, + '#' => Token::Pound, + '$' => Token::Primitive(self.take(String::new(), Self::is_identifier)), '"' => { let text = self.take(String::new(), |ch| ch != '"'); if self.characters.next() != Some('"') { @@ -52,14 +60,21 @@ impl<'a> Lexer<'a> { let digits = self.take(String::new(), |ch| ch.is_ascii_hexdigit()); Token::Bytes(hex::decode(&digits).map_err(|error| error.to_string())?) } - // Negative numbers - '-' if self.peek(|character| character.is_ascii_digit()) => { - let number = self.take(String::new(), |ch| ch.is_ascii_digit()); - Token::Number(-number.parse::().map_err(|error| error.to_string())?) - } character if character.is_ascii_digit() => { - let number = self.take(String::from(character), |ch| ch.is_ascii_digit()); - Token::Number(number.parse::().map_err(|error| error.to_string())?) + let digits = self.take(String::from(character), |ch| { + ch.is_ascii_digit() || ch == '_' + }); + let suffix = self.take(String::new(), |ch| ch.is_ascii_alphanumeric()); + let bytes = match suffix.as_str() { + "u8" => digits.parse::().map(|n| n.to_le_bytes().to_vec()), + "u16" => digits.parse::().map(|n| n.to_le_bytes().to_vec()), + "u32" => digits.parse::().map(|n| n.to_le_bytes().to_vec()), + "u64" => digits.parse::().map(|n| n.to_le_bytes().to_vec()), + "u128" => digits.parse::().map(|n| n.to_le_bytes().to_vec()), + "" => return Err(format!("integer literal {digits} missing suffix")), + other => return Err(format!("unknown integer suffix: {other}")), + }; + Token::Bytes(bytes.map_err(|error| error.to_string())?) } character if Self::is_identifier(character) => { Token::Identifier(self.take(String::from(character), Self::is_identifier)) diff --git a/fix/parser/src/lib.rs b/fix/parser/src/lib.rs new file mode 100644 index 00000000..f0993939 --- /dev/null +++ b/fix/parser/src/lib.rs @@ -0,0 +1,30 @@ +#![cfg_attr(target_arch = "wasm32", no_std, feature(asm_experimental_arch))] +extern crate alloc; + +use dlmalloc::GlobalDlmalloc; +#[global_allocator] +static ALLOCATOR: GlobalDlmalloc = GlobalDlmalloc; + +mod lexer; +mod parser; +mod token; + +use fixutils::*; +use lexer::Lexer; +use parser::Parser; + +num_memories!(48); +num_tables!(24); + +#[fix_entrypoint] +pub fn _fixpoint_apply(combination: RustHandle<'static>) -> Result, FixError> { + let arguments = combination.to_entries()?; + + let source_handle = arguments.get(1).expect("expected source"); + let source = source_handle.to_bytes()?; + let source = core::str::from_utf8(&source).expect("source should be valid UTF-8"); + + let tokens = Lexer::new(source).tokenize().expect("failed to tokenize"); + let mut parser = Parser::new(tokens, arguments.get(2).expect("expected environment"))?; + parser.parse_program() +} diff --git a/fix/parser/src/parser.rs b/fix/parser/src/parser.rs new file mode 100644 index 00000000..22df22ed --- /dev/null +++ b/fix/parser/src/parser.rs @@ -0,0 +1,119 @@ +use alloc::{collections::BTreeMap, string::String, vec::Vec}; + +use crate::token::Token; +use fixutils::*; + +pub struct Parser { + tokens: Vec, + position: usize, + environment: BTreeMap>, + context: BTreeMap>, +} + +impl Parser { + pub fn new( + tokens: Vec, + environment_handle: &RustHandle<'static>, + ) -> Result { + let mut environment = BTreeMap::new(); + for entry in environment_handle.to_entries()? { + let entry = entry.to_entries()?; + let name = *entry.first().expect("expect name"); + let object = entry.get(1).expect("expect object"); + environment.insert( + String::from_utf8(name.to_bytes()?).expect("valid name"), + *object, + ); + } + + Ok(Self { + tokens, + position: 0, + environment, + context: BTreeMap::new(), + }) + } + + pub fn parse_program(&mut self) -> Result, FixError> { + let handle = self.parse_expr()?; + self.expect(&Token::Eof, "expected end of program"); + Ok(handle) + } + + fn parse_expr(&mut self) -> Result, FixError> { + Ok(match self.advance() { + Token::String(string) => RustHandle::from_bytes(string.as_bytes())?, + Token::Bytes(bytes) => RustHandle::from_bytes(&bytes)?, + Token::Identifier(name) => *self.context.get(&name).expect("undefined identifier"), + Token::Primitive(name) => *self.environment.get(&name).expect("undefined primitive"), + Token::Ampersand => create_ref(self.parse_expr()?), + Token::Apostrophe => create_identification_thunk(self.parse_expr()?), + Token::Pound => create_application_thunk(self.parse_expr()?), + Token::Asterisk => create_strict_encode(self.parse_expr()?), + Token::Plus => create_shallow_encode(self.parse_expr()?), + Token::LParen => { + if let Some(Token::Identifier(token)) = self.peek(self.position) + && token == "let" + { + self.advance(); + self.parse_let()? + } else { + RustHandle::from_entries(&self.parse_handles(&Token::RParen)?)? + } + } + Token::LBracket => create_selection_thunk(RustHandle::from_entries( + &self.parse_handles(&Token::RBracket)?, + )?), + token => panic!("unexpected token: {token:?}"), + }) + } + + fn parse_handles(&mut self, close: &Token) -> Result>, FixError> { + let mut handles = Vec::new(); + while !self.matches(close) { + handles.push(self.parse_expr()?); + } + Ok(handles) + } + + fn parse_let(&mut self) -> Result, FixError> { + self.expect(&Token::LParen, "expected '(' for let bindings"); + let outer_context = self.context.clone(); + while self.matches(&Token::LParen) { + let Token::Identifier(name) = self.advance() else { + panic!("expected name in let binding") + }; + let handle = self.parse_expr()?; + self.expect(&Token::RParen, "expected ')' for let binding"); + self.context.insert(name, handle); + } + self.expect(&Token::RParen, "expected ')' for let bindings"); + let body = self.parse_expr()?; + self.expect(&Token::RParen, "expected ')' for let"); + self.context = outer_context; + Ok(body) + } + + fn expect(&mut self, token: &Token, message: &str) { + assert!(self.matches(token), "{message}"); + } + + fn matches(&mut self, token: &Token) -> bool { + if self.peek(self.position) == Some(token) { + self.position += 1; + true + } else { + false + } + } + + fn peek(&self, position: usize) -> Option<&Token> { + self.tokens.get(position) + } + + fn advance(&mut self) -> Token { + let token = self.peek(self.position).cloned().unwrap_or(Token::Eof); + self.position += 1; + token + } +} diff --git a/fix/parser/src/token.rs b/fix/parser/src/token.rs new file mode 100644 index 00000000..06f600b7 --- /dev/null +++ b/fix/parser/src/token.rs @@ -0,0 +1,19 @@ +use alloc::{string::String, vec::Vec}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Token { + Identifier(String), + String(String), + Bytes(Vec), + Primitive(String), + LParen, + RParen, + LBracket, + RBracket, + Ampersand, + Apostrophe, + Asterisk, + Plus, + Pound, + Eof, +} diff --git a/fix/procedure/Cargo.toml b/fix/procedure/Cargo.toml deleted file mode 100644 index e0691708..00000000 --- a/fix/procedure/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "fixprocedure" -version = "0.1.0" -edition = "2024" - -[lib] -crate-type = ["cdylib"] - -[dependencies] -fixutils = { path = "../utils" } -dlmalloc = { version = "0.2.14", features = ["global"] } \ No newline at end of file diff --git a/fix/procedure/src/lib.rs b/fix/procedure/src/lib.rs deleted file mode 100644 index a2896847..00000000 --- a/fix/procedure/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -#![cfg_attr(target_arch = "wasm32", no_std, feature(asm_experimental_arch))] -use dlmalloc::GlobalDlmalloc; -#[global_allocator] -static ALLOCATOR: GlobalDlmalloc = GlobalDlmalloc; -use fixutils::*; - -num_memories!(1); -num_tables!(2); - -#[fix_entrypoint] -pub fn _fixpoint_apply(combination: RustHandle<'static>) -> Result, FixError> { - let blob_handle = RustHandle::from_bytes(b"hello")?; - let mut entries = combination.to_entries()?; - entries.push(blob_handle); - Ok(create_strict_encode(create_identification_thunk( - RustHandle::from_entries(&entries)?, - ))) -} diff --git a/fix/shell/src/lib.rs b/fix/shell/src/lib.rs index 9d589a33..491ab2db 100644 --- a/fix/shell/src/lib.rs +++ b/fix/shell/src/lib.rs @@ -77,7 +77,8 @@ pub unsafe extern "C" fn _rsstart() -> ! { main(); } -static mut MODULE_BUF: [u8; 1024] = [0; 1024]; +const MODULE_BUF_SIZE: usize = 8192; +static mut MODULE_BUF: [u8; MODULE_BUF_SIZE] = [0; MODULE_BUF_SIZE]; pub fn main() -> ! { let combination = os::argument(); @@ -89,7 +90,7 @@ pub fn main() -> ! { wasm_rt_init(); let module_size = wasm_rt_module_size(); let module = unsafe { - assert!(module_size <= 1024); + assert!(module_size <= MODULE_BUF_SIZE); &raw mut MODULE_BUF[0] as *mut c_void }; wasm2c_module_instantiate(module, core::ptr::null()); diff --git a/fix/src/evaluator.rs b/fix/src/evaluator.rs index 3eb48d67..4bdc7378 100644 --- a/fix/src/evaluator.rs +++ b/fix/src/evaluator.rs @@ -36,6 +36,62 @@ impl Evaluator { self.runtime.execute(combination) } + pub fn select(&self, selection: Tree) -> Handle { + let handles = self.storage().get_tree(selection).unwrap(); + match *handles { + [target, index] => self.select_index(target, self.read_index(index)), + [target, start, end] => { + self.select_range(target, self.read_index(start), self.read_index(end)) + } + _ => panic!("selection thunk got {} handles", handles.len()), + } + } + + pub fn select_index(&self, target: Handle, index: usize) -> Handle { + if index >= target.len() { + panic!("Invalid index {index} for selection thunk"); + } + match target { + Handle::Object(Object::Tree(tree)) | Handle::Ref(Ref::Tree(tree)) => { + self.storage().get_tree(tree).unwrap()[index] + } + Handle::Object(Object::Blob(blob)) | Handle::Ref(Ref::Blob(blob)) => { + let data = self.storage().get_blob(blob).unwrap(); + Ref::Blob(self.storage().add_blob(&[data[index]])).into() + } + _ => panic!("expected blob or tree handle for selection thunk"), + } + } + + pub fn select_range(&self, target: Handle, begin: usize, end: usize) -> Handle { + if begin >= end { + panic!("Invalid range [{begin}, {end}) for seleciton thunk"); + } + match target { + Handle::Object(Object::Tree(tree)) | Handle::Ref(Ref::Tree(tree)) => { + let data = self.storage().get_tree(tree).unwrap(); + Ref::Tree(self.storage().add_tree(&data[begin..end])).into() + } + Handle::Object(Object::Blob(blob)) | Handle::Ref(Ref::Blob(blob)) => { + let data = self.storage().get_blob(blob).unwrap(); + Ref::Blob(self.storage().add_blob(&data[begin..end])).into() + } + _ => panic!("expected blob or tree handle for selection thunk"), + } + } + + pub fn read_index(&self, handle: Handle) -> usize { + let Handle::Object(Object::Blob(blob)) = handle else { + panic!("expected blob handle for selection index") + }; + let bytes = self.storage().get_blob(blob).unwrap(); + // Make buffer fit all supported integer widths + let mut buffer = [0; 16]; + assert!(bytes.len() <= buffer.len()); + buffer[..bytes.len()].copy_from_slice(&bytes); + usize::try_from(u128::from_le_bytes(buffer)).expect("selection index should be in range") + } + pub fn lift(&self, handle: Handle) -> Handle { match handle { Handle::Ref(r) => match r { @@ -59,7 +115,10 @@ impl Evaluator { fn think(&self, thunk: Thunk) -> Handle { match thunk { Thunk::Identification(reference) => self.lift(Handle::Ref(reference)), - Thunk::Selection(_) => todo!(), + Thunk::Selection(tree) => { + let evaled = self.eval_tree(tree); + self.select(evaled) + } Thunk::Application(tree) => { let evaled = self.eval_tree(tree); self.apply(evaled) diff --git a/fix/src/interpreter.rs b/fix/src/interpreter.rs deleted file mode 100644 index 9d4c13bb..00000000 --- a/fix/src/interpreter.rs +++ /dev/null @@ -1,111 +0,0 @@ -extern crate alloc; -use alloc::collections::BTreeMap; - -use crate::{Storage, parser::Expr}; -mod fixshell; -use fixhandle::*; -use fixshell::FixShell; -use kernel::prelude::*; - -pub struct Interpreter<'a> { - storage: &'a dyn Storage, - context: BTreeMap, -} - -impl<'a> Interpreter<'a> { - pub fn new(storage: &'a dyn Storage) -> Self { - Self { - storage, - context: BTreeMap::new(), - } - } - - pub fn interpret(&mut self, expression: &Expr) -> Handle { - match expression { - Expr::String(str) => self.create_blob(str.as_bytes()), - Expr::Number(num) => self.create_blob(&i64::to_le_bytes(*num)), - Expr::Bytes(bytes) => self.create_blob(bytes), - Expr::Identifier(name) => *self.context.get(name).expect("undefined identifier"), - Expr::Ref(object) => Self::create_ref(self.interpret(object)), - Expr::Tree(handles) => { - let handles: Vec = handles.iter().map(|x| self.interpret(x)).collect(); - self.create_tree(&handles) - } - Expr::Application(tree) => Self::create_application_thunk(self.interpret(tree)), - Expr::Identification(tree) => Self::create_identification_thunk(self.interpret(tree)), - Expr::StrictEncode(thunk) => Self::create_strict_encode(self.interpret(thunk)), - Expr::Let { bindings, body } => { - let outer_context = self.context.clone(); - for (name, expr) in bindings { - let handle = self.interpret(expr); - self.context.insert(name.clone(), handle); - } - let handle = self.interpret(body); - self.context = outer_context; - handle - } - } - } -} - -impl FixShell for Interpreter<'_> { - type Handle = Handle; - - fn create_blob(&self, data: &[u8]) -> Self::Handle { - self.storage.add_blob(data).into() - } - - fn create_tree(&self, data: &[Self::Handle]) -> Self::Handle { - self.storage.add_tree(data).into() - } - - fn create_ref(handle: Self::Handle) -> Self::Handle { - match handle { - Handle::Object(Object::Blob(blob)) => Handle::Ref(Ref::Blob(blob)), - Handle::Object(Object::Tree(tree)) => Handle::Ref(Ref::Tree(tree)), - _ => panic!("expected blob or tree handle"), - } - } - - fn get_blob_data(&self, handle: Self::Handle) -> Box<[u8]> { - let Handle::Object(Object::Blob(blob)) = handle else { - panic!("expected blob handle") - }; - self.storage - .get_blob(blob) - .expect("blob data exists for handle") - } - - fn get_tree_data(&self, handle: Self::Handle) -> Box<[Self::Handle]> { - let Handle::Object(Object::Tree(tree)) = handle else { - panic!("expected tree handle") - }; - self.storage - .get_tree(tree) - .expect("tree data exists for handle") - } - - fn create_application_thunk(handle: Self::Handle) -> Self::Handle { - let Handle::Object(Object::Tree(tree)) = handle else { - panic!("expected tree handle for applicaiton") - }; - Thunk::Application(tree).into() - } - - fn create_identification_thunk(handle: Self::Handle) -> Self::Handle { - Thunk::Identification(match handle { - Handle::Object(Object::Blob(blob)) => Ref::Blob(blob), - Handle::Object(Object::Tree(tree)) => Ref::Tree(tree), - Handle::Ref(reference) => reference, - _ => panic!("expected blob or tree handle"), - }) - .into() - } - - fn create_strict_encode(handle: Self::Handle) -> Self::Handle { - let Handle::Thunk(thunk) = handle else { - panic!("expected thunk for strict encode") - }; - Encode::Strict(thunk).into() - } -} diff --git a/fix/src/interpreter/fixshell.rs b/fix/src/interpreter/fixshell.rs deleted file mode 100644 index 154466a3..00000000 --- a/fix/src/interpreter/fixshell.rs +++ /dev/null @@ -1,20 +0,0 @@ -use kernel::prelude::Box; - -pub trait FixShell: Sized { - type Handle; - - fn create_blob(&self, data: &[u8]) -> Self::Handle; - fn create_tree(&self, data: &[Self::Handle]) -> Self::Handle; - fn create_ref(handle: Self::Handle) -> Self::Handle; - - #[allow(dead_code)] // these functions are unused so far - // (and ultimately the interpreter should use the "real" Fix shell) - fn get_blob_data(&self, handle: Self::Handle) -> Box<[u8]>; - #[allow(dead_code)] - fn get_tree_data(&self, handle: Self::Handle) -> Box<[Self::Handle]>; - - fn create_application_thunk(handle: Self::Handle) -> Self::Handle; - fn create_identification_thunk(handle: Self::Handle) -> Self::Handle; - - fn create_strict_encode(handle: Self::Handle) -> Self::Handle; -} diff --git a/fix/src/interpreter/grammar.md b/fix/src/interpreter/grammar.md deleted file mode 100644 index 5c9bc33e..00000000 --- a/fix/src/interpreter/grammar.md +++ /dev/null @@ -1,27 +0,0 @@ -``` -A ::= O | K | E | L Any - | $atom Special/Primitive - | @path Executable path - -B ::= int literal Blob - | string literal Blob - -T ::= (A*) Tree - -R ::= &B | &T Ref - -O ::= B | T | R Data - -K ::= ^O Thunk (Identify) - | *T Thunk (Apply) - | #T Thunk (Digest) - | ~T Thunk (Select) - | O[B] Thunk (Select 1) - | O[B:B] Thunk (Select N) - -E ::= !K Encode (Strict) - | ?K Encode (Shallow) - -L ::= (let ((name A)*) A) Let -``` - diff --git a/fix/src/lib.rs b/fix/src/lib.rs index b74973d4..78f1caca 100644 --- a/fix/src/lib.rs +++ b/fix/src/lib.rs @@ -2,9 +2,9 @@ #![no_std] pub mod evaluator; -pub mod interpreter; -pub mod parser; +pub mod preprocessor; pub mod runtime; +pub mod stdlib; pub mod storage; pub mod handle { @@ -13,6 +13,6 @@ pub mod handle { pub use evaluator::*; pub use handle::*; -pub use interpreter::*; +pub use preprocessor::*; pub use runtime::*; pub use storage::*; diff --git a/fix/src/main.rs b/fix/src/main.rs index 9903ff34..2aec9e55 100644 --- a/fix/src/main.rs +++ b/fix/src/main.rs @@ -12,9 +12,8 @@ use kernel::prelude::*; use fix::arca::FixOnArca; use fix::*; -use lexer::*; -use parser::*; -use preprocessor::*; + +pub const PARSER: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/fixparser")); #[cfg(test)] mod testing; @@ -88,11 +87,14 @@ fn eval_parallel_program( evaluator: &Arc>, ) -> Handle { let processed = Preprocessor::new(source).preprocess().unwrap(); - let tokens = Lexer::new(&processed).tokenize().unwrap(); - let program = Parser::new(&tokens).parse_program().unwrap(); - - let mut interpreter = Interpreter::new(evaluator.storage()); - evaluator.eval(interpreter.interpret(&program)) + let parser: Handle = evaluator.storage().add_blob(PARSER).into(); + let source = evaluator.storage().add_blob(processed.as_bytes()); + let environment = stdlib::build_environment(evaluator.storage()); + let combination = evaluator + .storage() + .add_tree(&[parser, source.into(), environment]); + + evaluator.eval(Encode::Strict(Thunk::Application(combination)).into()) } // `fix eval `: read command file and print result. @@ -116,11 +118,14 @@ fn eval_file(path: &str) { // parse, interpret, and evaluate source text. fn eval_program(source: &str, evaluator: &Evaluator) -> Handle { let processed = Preprocessor::new(source).preprocess().unwrap(); - let tokens = Lexer::new(&processed).tokenize().unwrap(); - let program = Parser::new(&tokens).parse_program().unwrap(); - - let mut interpreter = Interpreter::new(evaluator.storage()); - evaluator.eval(interpreter.interpret(&program)) + let parser: Handle = evaluator.storage().add_blob(PARSER).into(); + let source = evaluator.storage().add_blob(processed.as_bytes()); + let environment = stdlib::build_environment(evaluator.storage()); + let combination = evaluator + .storage() + .add_tree(&[parser, source.into(), environment]); + + evaluator.eval(Encode::Strict(Thunk::Application(combination)).into()) } #[cfg(test)] @@ -144,34 +149,49 @@ mod tests { let evaluator = Evaluator::new(FixOnArca::default()); { - assert_eq!(eval_value("42", &evaluator), 42i64.to_le_bytes()); - assert_eq!(eval_value("-1", &evaluator), (-1i64).to_le_bytes()); + assert_eq!(eval_value("42u8", &evaluator), 42u8.to_le_bytes()); + //assert_eq!(eval_value("-1", &evaluator), (-1i64).to_le_bytes()); assert_eq!(eval_value("\"hello\"", &evaluator), b"hello"); - assert_eq!(eval_value("(1 2 3)", &evaluator), 3i64.to_le_bytes()); - assert_eq!(eval_value("()", &evaluator), 0i64.to_le_bytes()); + assert_eq!(eval_value("(1u8 2u8 3u8)", &evaluator), 3u64.to_le_bytes()); + assert_eq!(eval_value("()", &evaluator), 0u64.to_le_bytes()); assert_eq!(eval_value("&\"hello\"", &evaluator), b"hello"); - assert_eq!(eval_value("&(1 2 3)", &evaluator), 3i64.to_le_bytes()); + assert_eq!(eval_value("&(1u8 2u8 3u8)", &evaluator), 3u64.to_le_bytes()); - assert_eq!(eval_value("!^&2", &evaluator), 2i64.to_le_bytes()); + assert_eq!(eval_value("*'&2u8", &evaluator), 2u8.to_le_bytes()); + assert_eq!( + eval_value("*[(1u8 2u8 3u8 4u8) 2u8]", &evaluator), + 3u8.to_le_bytes() + ); assert_eq!( - eval_value("(let ((x 42)) x)", &evaluator), - 42i64.to_le_bytes() + eval_value("(let ((x 42u64)) x)", &evaluator), + 42u64.to_le_bytes() ); assert_eq!( - eval_value("(let ((x 1) (y 2)) (x y))", &evaluator), - 2i64.to_le_bytes() + eval_value("(let ((x 1u8) (y 2u8)) (x y))", &evaluator), + 2u64.to_le_bytes() ); assert_eq!( - eval_value("(let ((x 1)) (let ((x 2)) x))", &evaluator), - 2i64.to_le_bytes() + eval_value("(let ((x 1u8)) (let ((x 2u64)) x))", &evaluator), + 2u64.to_le_bytes() ); assert_eq!( - eval_value("(let ((x 1)) (let ((y (let ((x 2)) x))) x))", &evaluator), - 1i64.to_le_bytes() + eval_value( + "(let ((x 1u64)) (let ((y (let ((x 2u8)) x))) x))", + &evaluator + ), + 1u64.to_le_bytes() + ); + } + + // primitive test + { + assert_eq!( + eval_value("*#($identity 2u8)", &evaluator), + 2u64.to_le_bytes() ); } } diff --git a/fix/src/parallel_evaluator.rs b/fix/src/parallel_evaluator.rs index e1886b65..28272966 100644 --- a/fix/src/parallel_evaluator.rs +++ b/fix/src/parallel_evaluator.rs @@ -83,6 +83,62 @@ impl Evaluator { self.runtime.execute(combination) } + pub fn select(&self, selection: Tree) -> Handle { + let handles = self.storage().get_tree(selection).unwrap(); + match *handles { + [target, index] => self.select_index(target, self.read_index(index)), + [target, start, end] => { + self.select_range(target, self.read_index(start), self.read_index(end)) + } + _ => panic!("selection thunk got {} handles", handles.len()), + } + } + + pub fn select_index(&self, target: Handle, index: usize) -> Handle { + if index >= target.len() { + panic!("Invalid index {index} for selection thunk"); + } + match target { + Handle::Object(Object::Tree(tree)) | Handle::Ref(Ref::Tree(tree)) => { + self.storage().get_tree(tree).unwrap()[index] + } + Handle::Object(Object::Blob(blob)) | Handle::Ref(Ref::Blob(blob)) => { + let data = self.storage().get_blob(blob).unwrap(); + Ref::Blob(self.storage().add_blob(&[data[index]])).into() + } + _ => panic!("expected blob or tree handle for selection thunk"), + } + } + + pub fn select_range(&self, target: Handle, begin: usize, end: usize) -> Handle { + if begin >= end { + panic!("Invalid range [{begin}, {end}) for seleciton thunk"); + } + match target { + Handle::Object(Object::Tree(tree)) | Handle::Ref(Ref::Tree(tree)) => { + let data = self.storage().get_tree(tree).unwrap(); + Ref::Tree(self.storage().add_tree(&data[begin..end])).into() + } + Handle::Object(Object::Blob(blob)) | Handle::Ref(Ref::Blob(blob)) => { + let data = self.storage().get_blob(blob).unwrap(); + Ref::Blob(self.storage().add_blob(&data[begin..end])).into() + } + _ => panic!("expected blob or tree handle for selection thunk"), + } + } + + pub fn read_index(&self, handle: Handle) -> usize { + let Handle::Object(Object::Blob(blob)) = handle else { + panic!("expected blob handle for selection index") + }; + let bytes = self.storage().get_blob(blob).unwrap(); + // Make buffer fit all supported integer widths + let mut buffer = [0; 16]; + assert!(bytes.len() <= buffer.len()); + buffer[..bytes.len()].copy_from_slice(&bytes); + usize::try_from(u128::from_le_bytes(buffer)).expect("selection index should be in range") + } + pub fn lift(&self, handle: Handle) -> Handle { match handle { Handle::Ref(r) => match r { @@ -106,7 +162,10 @@ impl Evaluator { fn think(&self, thunk: Thunk, eval_mode: EvalType) -> Handle { match thunk { Thunk::Identification(reference) => self.lift(Handle::Ref(reference)), - Thunk::Selection(_) => todo!(), + Thunk::Selection(tree) => { + let evaled = self.eval_tree(tree, eval_mode); + self.select(evaled) + } Thunk::Application(tree) => { let evaled = self.eval_tree(tree, eval_mode); self.apply(evaled) @@ -119,7 +178,8 @@ impl Evaluator { match thought { Handle::Object(_) => thought, Handle::Ref(_) => self.lift(thought), - Handle::Thunk(_) | Handle::Encode(_) => todo!(), + Handle::Thunk(thunk) => self.force(thunk, eval_mode), + Handle::Encode(encode) => self.lift(self.encode(encode, eval_mode)), } } @@ -170,7 +230,7 @@ impl Evaluator { //println!("evaluating {handle}"); match handle { Handle::Ref(reference) => self.eval(self.lift(Handle::Ref(reference))), - Handle::Thunk(_) => todo!(), + Handle::Thunk(_) => handle, Handle::Object(obj) => match obj { Object::Blob(blob) => blob.into(), Object::Tree(tree) => self.eval_tree(tree, eval_mode).into(), diff --git a/fix/src/parser.rs b/fix/src/parser.rs deleted file mode 100644 index 50c73894..00000000 --- a/fix/src/parser.rs +++ /dev/null @@ -1,102 +0,0 @@ -use kernel::prelude::*; -pub use token::{Expr, Token}; - -pub mod lexer; -pub mod preprocessor; -pub mod token; - -pub struct Parser<'a> { - tokens: &'a [Token], - position: usize, -} - -impl<'a> Parser<'a> { - pub fn new(tokens: &'a [Token]) -> Self { - Self { - tokens, - position: 0, - } - } - - pub fn parse_program(&mut self) -> Result { - let expr = self.parse_expr()?; - self.expect(&Token::Eof, "expected end of program")?; - Ok(expr) - } - - fn parse_expr(&mut self) -> Result { - match self.advance() { - Token::Number(number) => Ok(Expr::Number(number)), - Token::String(string) => Ok(Expr::String(string)), - Token::Bytes(bytes) => Ok(Expr::Bytes(bytes)), - Token::Identifier(value) => Ok(Expr::Identifier(value)), - Token::Ampersand => Ok(Expr::Ref(Box::new(self.parse_expr()?))), - Token::Caret => Ok(Expr::Identification(Box::new(self.parse_expr()?))), - Token::Asterisk => Ok(Expr::Application(Box::new(self.parse_expr()?))), - Token::Bang => Ok(Expr::StrictEncode(Box::new(self.parse_expr()?))), - Token::LParen => { - if let Some(Token::Identifier(token)) = self.peek(self.position) - && token == "let" - { - self.advance(); - self.parse_let() - } else { - Ok(Expr::Tree(self.parse_handles()?)) - } - } - token => Err(format!("unexpected token: {token:?}")), - } - } - - fn parse_handles(&mut self) -> Result, String> { - let mut handles = Vec::new(); - while !self.matches(&Token::RParen) { - handles.push(self.parse_expr()?); - } - Ok(handles) - } - - fn parse_let(&mut self) -> Result { - self.expect(&Token::LParen, "expected '(' for let bindings")?; - let mut bindings = Vec::new(); - while self.matches(&Token::LParen) { - let Token::Identifier(name) = self.advance() else { - return Err(String::from("expected name in let binding")); - }; - let value = self.parse_expr()?; - self.expect(&Token::RParen, "expected ')' for let binding")?; - bindings.push((name, value)); - } - self.expect(&Token::RParen, "expected ')' for let bindings")?; - let body = Box::new(self.parse_expr()?); - self.expect(&Token::RParen, "expected ')' for let")?; - Ok(Expr::Let { bindings, body }) - } - - fn expect(&mut self, token: &Token, message: &str) -> Result<(), String> { - if self.matches(token) { - Ok(()) - } else { - Err(String::from(message)) - } - } - - fn matches(&mut self, token: &Token) -> bool { - if self.peek(self.position) == Some(token) { - self.position += 1; - true - } else { - false - } - } - - fn peek(&self, position: usize) -> Option<&Token> { - self.tokens.get(position) - } - - fn advance(&mut self) -> Token { - let token = self.peek(self.position).cloned().unwrap_or(Token::Eof); - self.position += 1; - token - } -} diff --git a/fix/src/parser/token.rs b/fix/src/parser/token.rs deleted file mode 100644 index 891f14aa..00000000 --- a/fix/src/parser/token.rs +++ /dev/null @@ -1,34 +0,0 @@ -use core::clone::Clone; -use kernel::prelude::*; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Token { - Identifier(String), - Number(i64), - String(String), - Bytes(Vec), - LParen, - RParen, - Ampersand, - Caret, - Asterisk, - Bang, - Eof, -} - -#[derive(Debug, Clone)] -pub enum Expr { - Number(i64), - String(String), - Bytes(Vec), - Identifier(String), - Ref(Box), - Tree(Vec), - Application(Box), - Identification(Box), - StrictEncode(Box), - Let { - bindings: Vec<(String, Expr)>, - body: Box, - }, -} diff --git a/fix/src/parser/preprocessor.rs b/fix/src/preprocessor.rs similarity index 82% rename from fix/src/parser/preprocessor.rs rename to fix/src/preprocessor.rs index a5ca8ae6..2a600662 100644 --- a/fix/src/parser/preprocessor.rs +++ b/fix/src/preprocessor.rs @@ -1,11 +1,7 @@ -use super::lexer::*; use core::{iter::Peekable, str::Chars}; use kernel::host::fs::{File, Whence}; use kernel::prelude::*; -// Temporary placeholder until the standard library format is finalized -pub const STDLIB: &str = "./fix/stdlib"; - pub struct Preprocessor<'a> { characters: Peekable>, } @@ -45,16 +41,6 @@ impl<'a> Preprocessor<'a> { } } } - '$' => { - let mut name = String::new(); - while let Some(character) = - self.characters.next_if(|&ch| Lexer::is_identifier(ch)) - { - name.push(character); - } - let program = read_file(&format!("{STDLIB}/{name}"))?; - output.push_str(&format!("0x{}", hex::encode(program))); - } '@' => { if self.characters.next() != Some('"') { return Err(String::from("expected path after '@'")); diff --git a/fix/src/stdlib.rs b/fix/src/stdlib.rs new file mode 100644 index 00000000..2a34f453 --- /dev/null +++ b/fix/src/stdlib.rs @@ -0,0 +1,18 @@ +use crate::handle::*; +use crate::storage::Storage; +use kernel::prelude::Vec; + +pub const PRIMITIVES: &[(&str, &[u8])] = &[( + "identity", + include_bytes!(concat!(env!("OUT_DIR"), "/identity")), +)]; + +pub fn build_environment(storage: &dyn Storage) -> Handle { + let mut environment: Vec = Vec::new(); + for (name, blob) in PRIMITIVES { + let name = storage.add_blob(name.as_bytes()); + let primitive = storage.add_blob(blob); + environment.push(storage.add_tree(&[name.into(), primitive.into()]).into()); + } + storage.add_tree(&environment).into() +} diff --git a/fix/utils/src/lib.rs b/fix/utils/src/lib.rs index 99281331..a26ec2c9 100644 --- a/fix/utils/src/lib.rs +++ b/fix/utils/src/lib.rs @@ -89,7 +89,7 @@ impl<'a> RustHandle<'a> { Memory::from_blob(*self)?.to_bytes(self.len()) } - pub fn to_entries(&self) -> Result>, FixError> { + pub fn to_entries(&self) -> Result>, FixError> { Table::from_tree(*self)?.to_entries(self.len()) } } diff --git a/fix/wasm/addblob.wat b/fix/wasm/addblob.wat index d4f1683e..a0c7408b 100644 --- a/fix/wasm/addblob.wat +++ b/fix/wasm/addblob.wat @@ -9,7 +9,7 @@ (func (export "_fixpoint_apply") (param $encode externref) (result externref) ;; attach combination tree (call $attach_tree - (i32.const 0) + (i32.const 1) (local.get $encode)) ;; grow rw-memory (memory.grow diff --git a/fix/wasm/fixprocedure.wasm b/fix/wasm/fixprocedure.wasm deleted file mode 100644 index ee0425e9..00000000 Binary files a/fix/wasm/fixprocedure.wasm and /dev/null differ diff --git a/fix/wasm/identity.wat b/fix/wasm/identity.wat new file mode 100644 index 00000000..84ea9c12 --- /dev/null +++ b/fix/wasm/identity.wat @@ -0,0 +1,5 @@ +(module + (func (export "_fixpoint_apply") (param $encode externref) (result externref) + local.get 0 + ) +) \ No newline at end of file diff --git a/fix/wasm/slowaddblob.wat b/fix/wasm/slowaddblob.wat index d992ad6a..5c5b0919 100644 --- a/fix/wasm/slowaddblob.wat +++ b/fix/wasm/slowaddblob.wat @@ -22,7 +22,7 @@ ;; Attach the combination tree. (call $attach_tree - (i32.const 0) + (i32.const 1) (local.get $encode)) ;; Grow rw-memory by zero pages, preserving the original behavior. diff --git a/selection.fix b/selection.fix new file mode 100644 index 00000000..97a2a7c2 --- /dev/null +++ b/selection.fix @@ -0,0 +1,2 @@ +(let ((nums &(1u8 2u8 3u8 4u8)) (word &"hello")) + *[word 0u8] ) \ No newline at end of file