allow unicode letters in literals

nilswende · Oct 25, 2024 · 2312132 · 2312132
1 parent e32d254
commit 2312132
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 9 deletions.
diff --git a/src/main/java/com/wn/dbml/compiler/lexer/Char.java b/src/main/java/com/wn/dbml/compiler/lexer/Char.java
@@ -4,20 +4,12 @@ final class Char {
 	private Char() {
 	}
 
-	public static boolean isDigit(int c) {
-		return '0' <= c && c <= '9';
-	}
-
-	public static boolean isLetter(int c) {
-		return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z';
-	}
-
 	public static boolean isLinebreak(int c) {
 		return c == '\n' || c == '\r';
 	}
 
 	public static boolean isWordChar(int c) {
-		return isLetter(c) || isDigit(c) || c == '_';
+		return Character.isLetter(c) || '0' <= c && c <= '9' || c == '_';
 	}
 
 	public static boolean isHexDigit(int c) {

diff --git a/src/test/java/com/wn/dbml/compiler/LexerTest.java b/src/test/java/com/wn/dbml/compiler/LexerTest.java
@@ -274,4 +274,29 @@ void testUnknown() {
 		assertEquals(List.of(ILLEGAL), types);
 	}
 
+	@Test
+	void testUmlaut() {
+		var dbml = "table üser {";
+		var lexer = getLexer(dbml);
+
+		var tokenList = lexer.tokenList();
+		var types = tokenList.stream().map(Token::getType).toList();
+
+		assertEquals(List.of(TABLE, SPACE, LITERAL, SPACE, LBRACE, EOF), types);
+		assertEquals(1, lexer.getPosition().line());
+		assertEquals(12, lexer.getPosition().column());
+	}
+
+	@Test
+	void testNonWordChar() {
+		var dbml = "table ü&ser {";
+		var lexer = getLexer(dbml);
+
+		var tokenList = lexer.tokenList();
+		var types = tokenList.stream().map(Token::getType).toList();
+
+		assertEquals(List.of(TABLE, SPACE, LITERAL, ILLEGAL), types);
+		assertEquals(1, lexer.getPosition().line());
+		assertEquals(8, lexer.getPosition().column());
+	}
 }
diff --git a/src/test/java/com/wn/dbml/compiler/ParserTest.java b/src/test/java/com/wn/dbml/compiler/ParserTest.java
@@ -851,4 +851,17 @@ void testParseColumnDefaultInvalidDecimal() {
 		var e = assertThrows(ParsingException.class, () -> parse(dbml));
 		assertTrue(e.getMessage().startsWith("[3:24] unexpected token 'LITERAL'"), e.getMessage());
 	}
+
+	@Test
+	void testUmlaut() {
+		var dbml = """
+				Table Üser {
+					name varchar
+				}""";
+		var database = parse(dbml);
+
+		var schema = getDefaultSchema(database);
+		var table = schema.getTable("Üser");
+		assertNotNull(table);
+	}
 }