metrify/src/parse.rs

825 lines
21 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use crate::units::{NonMetric, NonMetricQuantity};
enum Expect {
Number,
Unit,
}
#[derive(Debug, PartialEq)]
pub enum ParseError {
NotValidNumber(String),
UnexpectedUnit(String),
UnknownUnit(String),
ExpectedUnit,
AmbiguousUnit(String, &'static str, &'static str),
}
pub fn parse(input: &str) -> Result<Vec<NonMetricQuantity>, ParseError> {
let mut quantities = Vec::new();
let mut state = Expect::Number;
let mut amount = None;
for token in tokenize(input) {
match (&state, token) {
(Expect::Number, Token::Number(number)) => {
let number = parse_number(number)?;
amount = Some(number);
state = Expect::Unit;
}
(Expect::Number, Token::Unit(unit)) => {
return Err(ParseError::UnexpectedUnit(unit));
}
(Expect::Unit, Token::Number(_)) => {
unreachable!("token stream can't contain two numbers in a row");
}
(Expect::Unit, Token::Unit(unit)) => {
let unit = parse_unit(unit)?;
let quantity = NonMetricQuantity {
amount: amount.take().expect("must have read a number to be in this state"),
unit: unit,
};
quantities.push(quantity);
state = Expect::Number;
}
}
}
match state {
Expect::Number => {}
Expect::Unit => {
return Err(ParseError::ExpectedUnit);
}
}
Ok(quantities)
}
fn parse_number(input: String) -> Result<f64, ParseError> {
let no_whitespace: String = input.chars().filter(|c| !c.is_whitespace()).collect();
no_whitespace.parse().or_else(|_| Err(ParseError::NotValidNumber(input)))
}
fn parse_unit(input: String) -> Result<NonMetric, ParseError> {
match input.as_str() {
// Length
"inch" => Ok(NonMetric::Inch),
"inches" => Ok(NonMetric::Inch),
"in" => Ok(NonMetric::Inch),
"\"" => Ok(NonMetric::Inch),
"" => Ok(NonMetric::Inch),
"foot" => Ok(NonMetric::Foot),
"feet" => Ok(NonMetric::Foot),
"ft" => Ok(NonMetric::Foot),
"'" => Ok(NonMetric::Foot),
"" => Ok(NonMetric::Foot),
"yard" => Ok(NonMetric::Yard),
"yards" => Ok(NonMetric::Yard),
"yd" => Ok(NonMetric::Yard),
"mile" => Ok(NonMetric::Mile),
"miles" => Ok(NonMetric::Mile),
"mi" => Ok(NonMetric::Mile),
"m" => Ok(NonMetric::Mile),
// Weight
"ounce" => Ok(NonMetric::Ounce),
"ounces" => Ok(NonMetric::Ounce),
"oz" => Ok(NonMetric::Ounce),
"pound" => Ok(NonMetric::Pound),
"pounds" => Ok(NonMetric::Pound),
"lb" => Ok(NonMetric::Pound),
"lbs" => Ok(NonMetric::Pound),
"#" => Ok(NonMetric::Pound),
"stone" => Ok(NonMetric::Stone),
"stones" => Ok(NonMetric::Stone),
"st" => Ok(NonMetric::Stone),
// Temperature
"degree Fahrenheit" => Ok(NonMetric::Fahrenheit),
"degrees Fahrenheit" => Ok(NonMetric::Fahrenheit),
"degree fahrenheit" => Ok(NonMetric::Fahrenheit),
"degrees fahrenheit" => Ok(NonMetric::Fahrenheit),
"Fahrenheit" => Ok(NonMetric::Fahrenheit),
"fahrenheit" => Ok(NonMetric::Fahrenheit),
"°F" => Ok(NonMetric::Fahrenheit),
"F" => Ok(NonMetric::Fahrenheit),
// Area
"square inch" => Ok(NonMetric::SquareInch),
"square inches" => Ok(NonMetric::SquareInch),
"square in" => Ok(NonMetric::SquareInch),
"sq inch" => Ok(NonMetric::SquareInch),
"sq inches" => Ok(NonMetric::SquareInch),
"sq in" => Ok(NonMetric::SquareInch),
"inch²" => Ok(NonMetric::SquareInch),
"inches²" => Ok(NonMetric::SquareInch),
"in²" => Ok(NonMetric::SquareInch),
"\"²" => Ok(NonMetric::SquareInch),
"″²" => Ok(NonMetric::SquareInch),
"inch^2" => Ok(NonMetric::SquareInch),
"inches^2" => Ok(NonMetric::SquareInch),
"in^2" => Ok(NonMetric::SquareInch),
"\"^2" => Ok(NonMetric::SquareInch),
"square foot" => Ok(NonMetric::SquareFoot),
"square feet" => Ok(NonMetric::SquareFoot),
"square ft" => Ok(NonMetric::SquareFoot),
"sq foot" => Ok(NonMetric::SquareFoot),
"sq feet" => Ok(NonMetric::SquareFoot),
"sq ft" => Ok(NonMetric::SquareFoot),
"foot²" => Ok(NonMetric::SquareFoot),
"feet²" => Ok(NonMetric::SquareFoot),
"ft²" => Ok(NonMetric::SquareFoot),
"" => Ok(NonMetric::SquareFoot),
"′²" => Ok(NonMetric::SquareFoot),
"foot^2" => Ok(NonMetric::SquareFoot),
"feet^2" => Ok(NonMetric::SquareFoot),
"ft^2" => Ok(NonMetric::SquareFoot),
"'^2" => Ok(NonMetric::SquareFoot),
"sf" => Ok(NonMetric::SquareFoot),
"acre" => Ok(NonMetric::Acre),
"acres" => Ok(NonMetric::Acre),
"ac" => Ok(NonMetric::Acre),
"square mile" => Ok(NonMetric::SquareMile),
"square miles" => Ok(NonMetric::SquareMile),
"square mi" => Ok(NonMetric::SquareMile),
"sq mile" => Ok(NonMetric::SquareMile),
"sq miles" => Ok(NonMetric::SquareMile),
"sq mi" => Ok(NonMetric::SquareMile),
"mile²" => Ok(NonMetric::SquareMile),
"miles²" => Ok(NonMetric::SquareMile),
"mi²" => Ok(NonMetric::SquareMile),
"mile^2" => Ok(NonMetric::SquareMile),
"miles^2" => Ok(NonMetric::SquareMile),
"mi^2" => Ok(NonMetric::SquareMile),
// Volume
"cubic inch" => Ok(NonMetric::CubicInch),
"cubic inches" => Ok(NonMetric::CubicInch),
"cubic in" => Ok(NonMetric::CubicInch),
"cu inch" => Ok(NonMetric::CubicInch),
"cu inches" => Ok(NonMetric::CubicInch),
"cu in" => Ok(NonMetric::CubicInch),
"inch³" => Ok(NonMetric::CubicInch),
"inches³" => Ok(NonMetric::CubicInch),
"in³" => Ok(NonMetric::CubicInch),
"inch^3" => Ok(NonMetric::CubicInch),
"inches^3" => Ok(NonMetric::CubicInch),
"in^3" => Ok(NonMetric::CubicInch),
"cubic foot" => Ok(NonMetric::CubicFoot),
"cubic feet" => Ok(NonMetric::CubicFoot),
"cubic ft" => Ok(NonMetric::CubicFoot),
"cu foot" => Ok(NonMetric::CubicFoot),
"cu feet" => Ok(NonMetric::CubicFoot),
"cu ft" => Ok(NonMetric::CubicFoot),
"foot³" => Ok(NonMetric::CubicFoot),
"feet³" => Ok(NonMetric::CubicFoot),
"ft³" => Ok(NonMetric::CubicFoot),
"foot^3" => Ok(NonMetric::CubicFoot),
"feet^3" => Ok(NonMetric::CubicFoot),
"ft^3" => Ok(NonMetric::CubicFoot),
// Fluid volume
"imperial fluid ounce" => Ok(NonMetric::ImperialFluidOunce),
"imperial fluid ounces" => Ok(NonMetric::ImperialFluidOunce),
"imp fl oz" => Ok(NonMetric::ImperialFluidOunce),
"imp fl. oz." => Ok(NonMetric::ImperialFluidOunce),
"imp oz. fl." => Ok(NonMetric::ImperialFluidOunce),
"imperial pint" => Ok(NonMetric::ImperialPint),
"imperial pints" => Ok(NonMetric::ImperialPint),
"imp pt" => Ok(NonMetric::ImperialPint),
"imp p" => Ok(NonMetric::ImperialPint),
"imperial quart" => Ok(NonMetric::ImperialQuart),
"imperial quarts" => Ok(NonMetric::ImperialQuart),
"imp qt" => Ok(NonMetric::ImperialQuart),
"imperial gallon" => Ok(NonMetric::ImperialGallon),
"imperial gallons" => Ok(NonMetric::ImperialGallon),
"imp gal" => Ok(NonMetric::ImperialGallon),
"US teaspoon" => Ok(NonMetric::USTeaspoon),
"US teaspoons" => Ok(NonMetric::USTeaspoon),
"US tsp." => Ok(NonMetric::USTeaspoon),
"US tsp" => Ok(NonMetric::USTeaspoon),
"us teaspoon" => Ok(NonMetric::USTeaspoon),
"us teaspoons" => Ok(NonMetric::USTeaspoon),
"us tsp." => Ok(NonMetric::USTeaspoon),
"us tsp" => Ok(NonMetric::USTeaspoon),
"teaspoon" => Ok(NonMetric::USTeaspoon),
"teaspoons" => Ok(NonMetric::USTeaspoon),
"tsp." => Ok(NonMetric::USTeaspoon),
"tsp" => Ok(NonMetric::USTeaspoon),
"US tablespoon" => Ok(NonMetric::USTablespoon),
"US tablespoons" => Ok(NonMetric::USTablespoon),
"US Tbsp." => Ok(NonMetric::USTablespoon),
"US Tbsp" => Ok(NonMetric::USTablespoon),
"us tablespoon" => Ok(NonMetric::USTablespoon),
"us tablespoons" => Ok(NonMetric::USTablespoon),
"us tbsp." => Ok(NonMetric::USTablespoon),
"us tbsp" => Ok(NonMetric::USTablespoon),
"tablespoon" => Ok(NonMetric::USTablespoon),
"tablespoons" => Ok(NonMetric::USTablespoon),
"Tbsp." => Ok(NonMetric::USTablespoon),
"Tbsp" => Ok(NonMetric::USTablespoon),
"tbsp." => Ok(NonMetric::USTablespoon),
"tbsp" => Ok(NonMetric::USTablespoon),
"US fluid ounce" => Ok(NonMetric::USFluidOunce),
"US fluid ounces" => Ok(NonMetric::USFluidOunce),
"US fl oz" => Ok(NonMetric::USFluidOunce),
"US fl. oz." => Ok(NonMetric::USFluidOunce),
"US oz. fl." => Ok(NonMetric::USFluidOunce),
"us fluid ounce" => Ok(NonMetric::USFluidOunce),
"us fluid ounces" => Ok(NonMetric::USFluidOunce),
"us fl oz" => Ok(NonMetric::USFluidOunce),
"us fl. oz." => Ok(NonMetric::USFluidOunce),
"us oz. fl." => Ok(NonMetric::USFluidOunce),
"US cup" => Ok(NonMetric::USCup),
"US cups" => Ok(NonMetric::USCup),
"us cup" => Ok(NonMetric::USCup),
"us cups" => Ok(NonMetric::USCup),
"US liquid pint" => Ok(NonMetric::USLiquidPint),
"US liquid pints" => Ok(NonMetric::USLiquidPint),
"US pint" => Ok(NonMetric::USLiquidPint),
"US pints" => Ok(NonMetric::USLiquidPint),
"US pt" => Ok(NonMetric::USLiquidPint),
"US p" => Ok(NonMetric::USLiquidPint),
"us liquid pint" => Ok(NonMetric::USLiquidPint),
"us liquid pints" => Ok(NonMetric::USLiquidPint),
"us pint" => Ok(NonMetric::USLiquidPint),
"us pints" => Ok(NonMetric::USLiquidPint),
"us pt" => Ok(NonMetric::USLiquidPint),
"us p" => Ok(NonMetric::USLiquidPint),
"US liquid quart" => Ok(NonMetric::USLiquidQuart),
"US liquid quarts" => Ok(NonMetric::USLiquidQuart),
"US quart" => Ok(NonMetric::USLiquidQuart),
"US quarts" => Ok(NonMetric::USLiquidQuart),
"US qt" => Ok(NonMetric::USLiquidQuart),
"us liquid quart" => Ok(NonMetric::USLiquidQuart),
"us liquid quarts" => Ok(NonMetric::USLiquidQuart),
"us quart" => Ok(NonMetric::USLiquidQuart),
"us quarts" => Ok(NonMetric::USLiquidQuart),
"us qt" => Ok(NonMetric::USLiquidQuart),
"US gallon" => Ok(NonMetric::USGallon),
"US gallons" => Ok(NonMetric::USGallon),
"US gal" => Ok(NonMetric::USGallon),
"us gallon" => Ok(NonMetric::USGallon),
"us gallons" => Ok(NonMetric::USGallon),
"us gal" => Ok(NonMetric::USGallon),
// Ambiguous units
"fluid ounce" => Err(ParseError::AmbiguousUnit(input, "imperial", "US")),
"fluid ounces" => Err(ParseError::AmbiguousUnit(input, "imperial", "US")),
"fl oz" => Err(ParseError::AmbiguousUnit(input, "imp", "US")),
"fl. oz." => Err(ParseError::AmbiguousUnit(input, "imp", "US")),
"oz. fl." => Err(ParseError::AmbiguousUnit(input, "imp", "US")),
"pint" => Err(ParseError::AmbiguousUnit(input, "imperial", "US")),
"pints" => Err(ParseError::AmbiguousUnit(input, "imperial", "US")),
"pt" => Err(ParseError::AmbiguousUnit(input, "imp", "US")),
"p" => Err(ParseError::AmbiguousUnit(input, "imp", "US")),
"quart" => Err(ParseError::AmbiguousUnit(input, "imperial", "US")),
"quarts" => Err(ParseError::AmbiguousUnit(input, "imperial", "US")),
"qt" => Err(ParseError::AmbiguousUnit(input, "imp", "US")),
"gallon" => Err(ParseError::AmbiguousUnit(input, "imperial", "US")),
"gallons" => Err(ParseError::AmbiguousUnit(input, "imperial", "US")),
"gal" => Err(ParseError::AmbiguousUnit(input, "imp", "US")),
// Unknown unit
_ => Err(ParseError::UnknownUnit(input)),
}
}
#[derive(Debug, PartialEq)]
enum Token {
Number(String),
Unit(String),
}
enum TokState {
Neutral,
Number,
Unit(bool),
}
fn tokenize(input: &str) -> Vec<Token> {
let mut tokens = Vec::new();
let mut token = String::new();
let mut state = TokState::Neutral;
for c in input.chars() {
match state {
TokState::Neutral => {
if c.is_ascii_digit() || c == '-' {
token.push(c);
state = TokState::Number;
} else if !c.is_whitespace() {
token.push(c);
state = TokState::Unit(false);
}
}
TokState::Number => {
if c.is_ascii_digit() ||
c.is_whitespace() ||
c == '.' {
token.push(c);
} else {
tokens.push(Token::Number(token.trim().to_string()));
state = TokState::Unit(false);
token = String::new();
token.push(c);
}
}
TokState::Unit(after_caret) => {
if !after_caret && (c.is_ascii_digit() || c == '-') {
tokens.push(Token::Unit(token.trim().to_string()));
state = TokState::Number;
token = String::new();
token.push(c);
} else if c == '^' {
token.push(c);
state = TokState::Unit(true);
} else if c.is_whitespace() {
token.push(c);
state = TokState::Unit(false);
} else {
token.push(c);
}
}
}
}
match state {
TokState::Neutral => { assert!(token.len() == 0); }
TokState::Number => { tokens.push(Token::Number(token.trim().to_string())); }
TokState::Unit(_) => { tokens.push(Token::Unit(token.trim().to_string())); }
}
tokens
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn parsing() {
assert_eq!(parse(""), Ok(vec![]));
assert_eq!(parse("5 ft"), Ok(vec![
NonMetricQuantity { amount: 5.0, unit: NonMetric::Foot },
]));
assert_eq!(parse("5 ft 8 in"), Ok(vec![
NonMetricQuantity { amount: 5.0, unit: NonMetric::Foot },
NonMetricQuantity { amount: 8.0, unit: NonMetric::Inch },
]));
assert_eq!(parse("20 000 lbs"), Ok(vec![
NonMetricQuantity { amount: 20_000.0, unit: NonMetric::Pound },
]));
assert_eq!(parse("12.0."), Err(ParseError::NotValidNumber("12.0.".to_string())));
assert_eq!(parse("ft"), Err(ParseError::UnexpectedUnit("ft".to_string())));
assert_eq!(parse("5 tf"), Err(ParseError::UnknownUnit("tf".to_string())));
assert_eq!(parse("12"), Err(ParseError::ExpectedUnit));
assert_eq!(parse("1 gallon"), Err(ParseError::AmbiguousUnit("gallon".to_string(), "imperial", "US")));
}
#[test]
fn numbers() {
assert_eq!(parse_number("".to_string()), Err(ParseError::NotValidNumber("".to_string())));
assert_eq!(parse_number("1".to_string()), Ok(1.0));
assert_eq!(parse_number("1.0".to_string()), Ok(1.0));
assert_eq!(parse_number("0.1".to_string()), Ok(0.1));
assert_eq!(parse_number("0.1.".to_string()), Err(ParseError::NotValidNumber("0.1.".to_string())));
assert_eq!(parse_number("-10".to_string()), Ok(-10.0));
assert_eq!(parse_number("10\t00\u{1680}000".to_string()), Ok(10_00_000.0));
}
#[test]
fn units() {
// Length
test_units(NonMetric::Inch, &[
"inch",
"inches",
"in",
"\"",
"",
]);
test_units(NonMetric::Foot, &[
"foot",
"feet",
"ft",
"'",
"",
]);
test_units(NonMetric::Yard, &[
"yard",
"yards",
"yd",
]);
test_units(NonMetric::Mile, &[
"mile",
"miles",
"mi",
"m",
]);
// Weight
test_units(NonMetric::Ounce, &[
"ounce",
"ounces",
"oz",
]);
test_units(NonMetric::Pound, &[
"pound",
"pounds",
"lb",
"lbs",
"#",
]);
test_units(NonMetric::Stone, &[
"stone",
"stones",
"st",
]);
// Temperature
test_units(NonMetric::Fahrenheit, &[
"degree Fahrenheit",
"degrees Fahrenheit",
"degree fahrenheit",
"degrees fahrenheit",
"Fahrenheit",
"fahrenheit",
"°F",
"F",
]);
// Area
test_units(NonMetric::SquareInch, &[
"square inch",
"square inches",
"square in",
"sq inch",
"sq inches",
"sq in",
"inch²",
"inches²",
"in²",
"\"²",
"″²",
"inch^2",
"inches^2",
"in^2",
"\"^2",
]);
test_units(NonMetric::SquareFoot, &[
"square foot",
"square feet",
"square ft",
"sq foot",
"sq feet",
"sq ft",
"foot²",
"feet²",
"ft²",
"",
"′²",
"foot^2",
"feet^2",
"ft^2",
"'^2",
"sf",
]);
test_units(NonMetric::Acre, &[
"acre",
"acres",
"ac",
]);
test_units(NonMetric::SquareMile, &[
"square mile",
"square miles",
"square mi",
"sq mile",
"sq miles",
"sq mi",
"mile²",
"miles²",
"mi²",
"mile^2",
"miles^2",
"mi^2",
]);
// Volume
test_units(NonMetric::CubicInch, &[
"cubic inch",
"cubic inches",
"cubic in",
"cu inch",
"cu inches",
"cu in",
"inch³",
"inches³",
"in³",
"inch^3",
"inches^3",
"in^3",
]);
test_units(NonMetric::CubicFoot, &[
"cubic foot",
"cubic feet",
"cubic ft",
"cu foot",
"cu feet",
"cu ft",
"foot³",
"feet³",
"ft³",
"foot^3",
"feet^3",
"ft^3",
]);
// Fluid volume
test_units(NonMetric::ImperialFluidOunce, &[
"imperial fluid ounce",
"imperial fluid ounces",
"imp fl oz",
"imp fl. oz.",
"imp oz. fl.",
]);
test_units(NonMetric::ImperialPint, &[
"imperial pint",
"imperial pints",
"imp pt",
"imp p",
]);
test_units(NonMetric::ImperialQuart, &[
"imperial quart",
"imperial quarts",
"imp qt",
]);
test_units(NonMetric::ImperialGallon, &[
"imperial gallon",
"imperial gallons",
"imp gal",
]);
test_units(NonMetric::USTeaspoon, &[
"US teaspoon",
"US teaspoons",
"US tsp.",
"US tsp",
"us teaspoon",
"us teaspoons",
"us tsp.",
"us tsp",
"teaspoon",
"teaspoons",
"tsp.",
"tsp",
]);
test_units(NonMetric::USTablespoon, &[
"US tablespoon",
"US tablespoons",
"US Tbsp.",
"US Tbsp",
"us tablespoon",
"us tablespoons",
"us tbsp.",
"us tbsp",
"tablespoon",
"tablespoons",
"Tbsp.",
"Tbsp",
"tbsp.",
"tbsp",
]);
test_units(NonMetric::USFluidOunce, &[
"US fluid ounce",
"US fluid ounces",
"US fl oz",
"US fl. oz.",
"US oz. fl.",
"us fluid ounce",
"us fluid ounces",
"us fl oz",
"us fl. oz.",
"us oz. fl.",
]);
test_units(NonMetric::USCup, &[
"US cup",
"US cups",
"us cup",
"us cups",
]);
test_units(NonMetric::USLiquidPint, &[
"US liquid pint",
"US liquid pints",
"US pint",
"US pints",
"US pt",
"US p",
"us liquid pint",
"us liquid pints",
"us pint",
"us pints",
"us pt",
"us p",
]);
test_units(NonMetric::USLiquidQuart, &[
"US liquid quart",
"US liquid quarts",
"US quart",
"US quarts",
"US qt",
"us liquid quart",
"us liquid quarts",
"us quart",
"us quarts",
"us qt",
]);
test_units(NonMetric::USGallon, &[
"US gallon",
"US gallons",
"US gal",
"us gallon",
"us gallons",
"us gal",
]);
}
fn test_units(unit: NonMetric, spellings: &[&str]) {
for spelling in spellings {
assert_eq!(parse_unit(spelling.to_string()), Ok(unit));
}
}
#[test]
fn ambiguous_units() {
test_ambiguous_units(NonMetric::ImperialFluidOunce, NonMetric::USFluidOunce, &[
"fluid ounce",
"fluid ounces",
"fl oz",
"fl. oz.",
"oz. fl.",
]);
test_ambiguous_units(NonMetric::ImperialPint, NonMetric::USLiquidPint, &[
"pint",
"pints",
"pt",
"p",
]);
test_ambiguous_units(NonMetric::ImperialQuart, NonMetric::USLiquidQuart, &[
"quart",
"quarts",
"qt",
]);
test_ambiguous_units(NonMetric::ImperialGallon, NonMetric::USGallon, &[
"gallon",
"gallons",
"gal",
]);
}
fn test_ambiguous_units(unit1: NonMetric, unit2: NonMetric, spellings: &[&str]) {
for spelling in spellings {
let parsed = parse_unit(spelling.to_string());
if let Err(ParseError::AmbiguousUnit(unit_name, prefix1, prefix2)) = parsed {
assert_eq!(&unit_name, spelling);
let suggestion1 = format!("{prefix1} {unit_name}");
let suggestion2 = format!("{prefix2} {unit_name}");
assert_eq!(parse_unit(suggestion1), Ok(unit1));
assert_eq!(parse_unit(suggestion2), Ok(unit2));
} else {
panic!("units passed to test_ambiguous_units() must be ambiguous");
}
}
}
#[test]
fn unknown_unit() {
assert_eq!(parse_unit("hutenosa".to_string()), Err(ParseError::UnknownUnit("hutenosa".to_string())));
}
#[test]
fn tokens() {
assert_eq!(tokenize(""), vec![]);
assert_eq!(tokenize("10"), vec![Token::Number("10".to_string())]);
assert_eq!(tokenize(" 10 "), vec![Token::Number("10".to_string())]);
assert_eq!(tokenize("10 000"), vec![Token::Number("10 000".to_string())]);
assert_eq!(tokenize("10\t000"), vec![Token::Number("10\t000".to_string())]);
assert_eq!(tokenize("10\u{1680}000"), vec![Token::Number("10\u{1680}000".to_string())]);
assert_eq!(tokenize("10.0.1"), vec![Token::Number("10.0.1".to_string())]);
assert_eq!(tokenize("ft"), vec![Token::Unit("ft".to_string())]);
assert_eq!(
tokenize("10 ft"),
vec![
Token::Number("10".to_string()),
Token::Unit("ft".to_string()),
]
);
assert_eq!(
tokenize("5 ft 7 in"),
vec![
Token::Number("5".to_string()),
Token::Unit("ft".to_string()),
Token::Number("7".to_string()),
Token::Unit("in".to_string()),
]
);
assert_eq!(
tokenize("5\"7'"),
vec![
Token::Number("5".to_string()),
Token::Unit("\"".to_string()),
Token::Number("7".to_string()),
Token::Unit("'".to_string()),
]
);
assert_eq!(
tokenize(" 2.2lbs "),
vec![
Token::Number("2.2".to_string()),
Token::Unit("lbs".to_string()),
]
);
assert_eq!(
tokenize("sq ft"),
vec![
Token::Unit("sq ft".to_string()),
]
);
assert_eq!(
tokenize("sq ft2"),
vec![
Token::Unit("sq ft".to_string()),
Token::Number("2".to_string()),
]
);
assert_eq!(
tokenize("ft^2"),
vec![
Token::Unit("ft^2".to_string()),
]
);
assert_eq!(
tokenize("ft^22"),
vec![
Token::Unit("ft^22".to_string()),
]
);
assert_eq!(
tokenize("ft^2 2"),
vec![
Token::Unit("ft^2".to_string()),
Token::Number("2".to_string()),
]
);
assert_eq!(
tokenize("ft^2 s^-1 lb 2"),
vec![
Token::Unit("ft^2 s^-1 lb".to_string()),
Token::Number("2".to_string()),
]
);
}
}