initial commit, add scanner

src/main.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+    "fmt"
+    "scanner"
+)
+
+func main() {
+    source := "/* comment*/def foo:int[]..[](x1, x2ńńńńńńń) { return =  0xff + 077 + 12.34 + 56 + \"hello \n\rw\torld\" + 'hello world' + x1 * y2 + x3 / y3 && y || !y == y != {m, n} ? c)//hello world"
+    scanner := scanner.CreateScanner(source)
+    token := scanner.Scan()
+    for !scanner.EOF() {
+        fmt.Println(token.Value())
+        token = scanner.Scan()
+    }
+}

src/scanner/scanner_test.go

@@ -0,0 +1,51 @@
+package scanner
+
+import(
+    "testing"
+)
+
+func TestScanIdentifier(t *testing.T) {
+    source := "foo"
+    scanner := CreateScanner(source)
+
+    token := scanner.Scan()
+
+    expected := "foo"
+    actual := token.Value()
+
+    if (actual != expected) {
+        t.Error("Failed at scanning identifier. Expected:", expected, ", got :", actual)
+    }
+}
+
+func TestScanOctNumber(t *testing.T) {
+    source := "077"
+    scanner := CreateScanner(source)
+
+    token := scanner.Scan()
+
+    expected := "63"
+    actual := token.Value()
+
+    if (actual != expected) {
+        t.Error("Failed at scanning identifier. Expected:", expected, ", got :", actual)
+    }
+
+    token = scanner.Scan()
+}
+
+func TestScanHexNumber(t *testing.T) {
+    source := "0xff"
+    scanner := CreateScanner(source)
+
+    token := scanner.Scan()
+
+    expected := "255"
+    actual := token.Value()
+
+    if (actual != expected) {
+        t.Error("Failed at scanning identifier. Expected:", expected, ", got :", actual)
+    }
+
+    token = scanner.Scan()
+}

tools/dump_identifier.py

@@ -0,0 +1,42 @@
+import argparse
+import os
+
+def main():
+    parser = argparse.ArgumentParser(description="Dump Identifier Start")
+    parser.add_argument('-f', '--data_file', required=True, help='unicode data file')
+    parser.add_argument('-p', '--part', action='store_true', default=False, help='if include identifier part') 
+    args = parser.parse_args()
+
+    unicode_data = args.data_file
+    if unicode_data is None:
+        print('Need to provide unicode data path')
+        return
+
+    # Uppercase letter (Lu)
+    # Lowercase letter (Ll)
+    # Titlecase letter (Lt)
+    # Modifier letter (Lm)
+    # Other letter (lo)
+    # Letter number (Nl)
+    #
+    # Non-spacing mark (Mn)
+    # Combining spacing mark (Mc)
+    # Decimal number (Nd)
+    # Connector punctation (Pc)
+    # ZWNJ, ZWJ
+    if args.part:
+        categories = set(["Lu", "Ll", "Lt", "Lm", "Lo", "Nl", "Mn", "Mc", "Nd", "Pc", "ZERO WIDTH NON-JOINER", "ZERO WIDTH JOINER"])
+    else:
+        categories = set(["Lu", "Ll", "Lt", "Lm", "Lo", "Nl"])
+
+    with open(unicode_data, 'r') as data_file:
+        for line in data_file:
+            fields = line.split(";")
+            for field in fields:
+                if field in categories:
+                    decimal = int(fields[0], 16)
+                    print(str(decimal) + ', ', end="")
+                    break
+
+if __name__ == "__main__":
+    main()