matrix-dsl

A simple DSL for fast generation of Matrix Bots or Application Services
git clone git://archive.git.mtrnord.blog/Nordgedanken/matrix-dsl.git
Log | Files | Refs | LICENSE

lexer.go (3062B)

      1 package lexer
      2 
      3 import (
      4 	"bytes"
      5 	"fmt"
      6 	oLexer "github.com/alecthomas/participle/lexer"
      7 	"io"
      8 	"strconv"
      9 	"strings"
     10 	"text/scanner"
     11 	"unicode/utf8"
     12 )
     13 
     14 var MatrixDefinition = &matrixDefinition{}
     15 
     16 type matrixDefinition struct{}
     17 
     18 func (d *matrixDefinition) Lex(r io.Reader) oLexer.Lexer {
     19 	return Lex(r)
     20 }
     21 
     22 func (d *matrixDefinition) Symbols() map[string]rune {
     23 	return map[string]rune{
     24 		"EOF":       scanner.EOF,
     25 		"Char":      scanner.Char,
     26 		"Ident":     scanner.Ident,
     27 		"Int":       scanner.Int,
     28 		"Float":     scanner.Float,
     29 		"String":    scanner.String,
     30 		"RawString": scanner.RawString,
     31 		"Comment":   scanner.Comment,
     32 	}
     33 }
     34 
     35 // textScannerLexer is a Lexer based on text/scanner.Scanner
     36 type textScannerLexer struct {
     37 	scanner  *scanner.Scanner
     38 	filename string
     39 }
     40 
     41 // Lex an io.Reader with text/scanner.Scanner.
     42 //
     43 // This provides very fast lexing of source code compatibile with Go tokens.
     44 //
     45 // Note that this differs from text/scanner.Scanner in that string tokens will be unquoted.
     46 func Lex(r io.Reader) oLexer.Lexer {
     47 	lexer := lexWithScanner(r, &scanner.Scanner{})
     48 	lexer.scanner.Error = func(s *scanner.Scanner, msg string) {
     49 		// This is to support single quoted strings. Hacky.
     50 		if msg != "illegal char literal" {
     51 			oLexer.Panic(oLexer.Position(lexer.scanner.Pos()), msg)
     52 		}
     53 	}
     54 	return oLexer.Upgrade(lexer)
     55 }
     56 
     57 // LexWithScanner creates a Lexer from a user-provided scanner.Scanner.
     58 //
     59 // Useful if you need to customise the Scanner.
     60 //
     61 // Note that if this function is used, single-quoted strings are not supported. See the source for
     62 // Lex() for how to achieve this.
     63 
     64 func LexWithScanner(r io.Reader, scan *scanner.Scanner) oLexer.Lexer {
     65 	return oLexer.Upgrade(lexWithScanner(r, scan))
     66 }
     67 
     68 func lexWithScanner(r io.Reader, scan *scanner.Scanner) *textScannerLexer {
     69 	lexer := &textScannerLexer{
     70 		filename: oLexer.NameOfReader(r),
     71 		scanner:  scan,
     72 	}
     73 	lexer.scanner.Init(r)
     74 	return lexer
     75 }
     76 
     77 // LexBytes returns a new default lexer over bytes.
     78 func LexBytes(b []byte) oLexer.Lexer {
     79 	return Lex(bytes.NewReader(b))
     80 }
     81 
     82 // LexString returns a new default lexer over a string.
     83 func LexString(s string) oLexer.Lexer {
     84 	return Lex(strings.NewReader(s))
     85 }
     86 
     87 func (t *textScannerLexer) Next() oLexer.Token {
     88 	typ := t.scanner.Scan()
     89 	text := t.scanner.TokenText()
     90 	pos := oLexer.Position(t.scanner.Position)
     91 	pos.Filename = t.filename
     92 	out := oLexer.Token{
     93 		Type:  typ,
     94 		Value: text,
     95 		Pos:   pos,
     96 	}
     97 	out.Pos.Filename = t.filename
     98 	// Unquote strings.
     99 	switch out.Type {
    100 	case scanner.Char:
    101 		// FIXME(alec): This is pretty hacky...we convert a single quoted char into a double
    102 		// quoted string in order to support single quoted strings.
    103 		out.Value = fmt.Sprintf("\"%s\"", out.Value[1:len(out.Value)-1])
    104 		fallthrough
    105 	case scanner.String:
    106 		s, err := strconv.Unquote(out.Value)
    107 		if err != nil {
    108 			oLexer.Panic(out.Pos, err.Error())
    109 		}
    110 		out.Value = s
    111 		if out.Type == scanner.Char && utf8.RuneCountInString(s) > 1 {
    112 			out.Type = scanner.String
    113 		}
    114 	case scanner.RawString:
    115 		out.Value = out.Value[1 : len(out.Value)-1]
    116 	}
    117 	return out
    118 }