1
0
mirror of https://github.com/gusaul/grpcox.git synced 2024-11-17 14:56:26 +00:00
grpcox/vendor/github.com/jhump/protoreflect/desc/protoparse/lexer.go
2019-03-25 10:47:41 +07:00

765 lines
16 KiB
Go

package protoparse
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"strconv"
"strings"
"unicode/utf8"
)
type runeReader struct {
rr *bufio.Reader
unread []rune
err error
}
func (rr *runeReader) readRune() (r rune, size int, err error) {
if rr.err != nil {
return 0, 0, rr.err
}
if len(rr.unread) > 0 {
r := rr.unread[len(rr.unread)-1]
rr.unread = rr.unread[:len(rr.unread)-1]
return r, utf8.RuneLen(r), nil
}
r, sz, err := rr.rr.ReadRune()
if err != nil {
rr.err = err
}
return r, sz, err
}
func (rr *runeReader) unreadRune(r rune) {
rr.unread = append(rr.unread, r)
}
func lexError(l protoLexer, pos *SourcePos, err string) {
pl := l.(*protoLex)
if pl.err == nil {
pl.err = ErrorWithSourcePos{Underlying: errors.New(err), Pos: pos}
}
}
type protoLex struct {
filename string
input *runeReader
err error
res *fileNode
lineNo int
colNo int
offset int
prevSym terminalNode
}
func newLexer(in io.Reader) *protoLex {
return &protoLex{input: &runeReader{rr: bufio.NewReader(in)}}
}
var keywords = map[string]int{
"syntax": _SYNTAX,
"import": _IMPORT,
"weak": _WEAK,
"public": _PUBLIC,
"package": _PACKAGE,
"option": _OPTION,
"true": _TRUE,
"false": _FALSE,
"inf": _INF,
"nan": _NAN,
"repeated": _REPEATED,
"optional": _OPTIONAL,
"required": _REQUIRED,
"double": _DOUBLE,
"float": _FLOAT,
"int32": _INT32,
"int64": _INT64,
"uint32": _UINT32,
"uint64": _UINT64,
"sint32": _SINT32,
"sint64": _SINT64,
"fixed32": _FIXED32,
"fixed64": _FIXED64,
"sfixed32": _SFIXED32,
"sfixed64": _SFIXED64,
"bool": _BOOL,
"string": _STRING,
"bytes": _BYTES,
"group": _GROUP,
"oneof": _ONEOF,
"map": _MAP,
"extensions": _EXTENSIONS,
"to": _TO,
"max": _MAX,
"reserved": _RESERVED,
"enum": _ENUM,
"message": _MESSAGE,
"extend": _EXTEND,
"service": _SERVICE,
"rpc": _RPC,
"stream": _STREAM,
"returns": _RETURNS,
}
func (l *protoLex) cur() *SourcePos {
return &SourcePos{
Filename: l.filename,
Offset: l.offset,
Line: l.lineNo + 1,
Col: l.colNo + 1,
}
}
func (l *protoLex) prev() *SourcePos {
if l.prevSym == nil {
return &SourcePos{
Filename: l.filename,
Offset: 0,
Line: 1,
Col: 1,
}
}
return l.prevSym.start()
}
func (l *protoLex) Lex(lval *protoSymType) int {
if l.err != nil {
// if we are already in a failed state, bail
lval.err = l.err
return _ERROR
}
prevLineNo := l.lineNo
prevColNo := l.colNo
prevOffset := l.offset
var comments []*comment
pos := func() posRange {
return posRange{
start: &SourcePos{
Filename: l.filename,
Offset: prevOffset,
Line: prevLineNo + 1,
Col: prevColNo + 1,
},
end: l.cur(),
}
}
basic := func() basicNode {
return basicNode{
posRange: pos(),
leading: comments,
}
}
setPrev := func(n terminalNode) {
nStart := n.start().Line
if _, ok := n.(*basicNode); ok {
// if the node is a simple rune, don't attribute comments to it
// HACK: adjusting the start line makes leading comments appear
// detached so logic below will naturally associated trailing
// comment to previous symbol
nStart += 2
}
if l.prevSym != nil && len(n.leadingComments()) > 0 && l.prevSym.end().Line < nStart {
// we may need to re-attribute the first comment to
// instead be previous node's trailing comment
prevEnd := l.prevSym.end().Line
comments := n.leadingComments()
c := comments[0]
commentStart := c.start.Line
if commentStart == prevEnd {
// comment is on same line as previous symbol
n.popLeadingComment()
l.prevSym.pushTrailingComment(c)
} else if commentStart == prevEnd+1 {
// comment is right after previous symbol; see if it is detached
// and if so re-attribute
singleLineStyle := strings.HasPrefix(c.text, "//")
line := c.end.Line
groupEnd := -1
for i := 1; i < len(comments); i++ {
c := comments[i]
newGroup := false
if !singleLineStyle || c.start.Line > line+1 {
// we've found a gap between comments, which means the
// previous comments were detached
newGroup = true
} else {
line = c.end.Line
singleLineStyle = strings.HasPrefix(comments[i].text, "//")
if !singleLineStyle {
// we've found a switch from // comments to /*
// consider that a new group which means the
// previous comments were detached
newGroup = true
}
}
if newGroup {
groupEnd = i
break
}
}
if groupEnd == -1 {
// just one group of comments; we'll mark it as a trailing
// comment if it immediately follows previous symbol and is
// detached from current symbol
c1 := comments[0]
c2 := comments[len(comments)-1]
if c1.start.Line <= prevEnd+1 && c2.end.Line < nStart-1 {
groupEnd = len(comments)
}
}
for i := 0; i < groupEnd; i++ {
l.prevSym.pushTrailingComment(n.popLeadingComment())
}
}
}
l.prevSym = n
}
setString := func(val string) {
lval.str = &stringLiteralNode{basicNode: basic(), val: val}
setPrev(lval.str)
}
setIdent := func(val string, kind identKind) {
lval.id = &identNode{basicNode: basic(), val: val, kind: kind}
setPrev(lval.id)
}
setInt := func(val uint64) {
lval.ui = &intLiteralNode{basicNode: basic(), val: val}
setPrev(lval.ui)
}
setFloat := func(val float64) {
b := basic()
lval.f = &floatLiteralNode{val: val}
lval.f.setRange(&b, &b)
setPrev(lval.f)
}
setRune := func() {
b := basic()
lval.b = &b
setPrev(lval.b)
}
setError := func(err error) {
lval.err = err
l.err = err
}
for {
c, n, err := l.input.readRune()
if err == io.EOF {
// we're not actually returning a rune, but this will associate
// accumulated comments as a trailing comment on last symbol
// (if appropriate)
setRune()
return 0
} else if err != nil {
setError(err)
return _ERROR
}
prevLineNo = l.lineNo
prevColNo = l.colNo
prevOffset = l.offset
l.offset += n
if c == '\n' {
l.colNo = 0
l.lineNo++
continue
} else if c == '\r' {
continue
}
l.colNo++
if c == ' ' || c == '\t' {
continue
}
if c == '.' {
// tokens that start with a dot include type names and decimal literals
cn, _, err := l.input.readRune()
if err != nil {
setRune()
return int(c)
}
if cn == '_' || (cn >= 'a' && cn <= 'z') || (cn >= 'A' && cn <= 'Z') {
l.colNo++
token := []rune{c, cn}
token = l.readIdentifier(token)
setIdent(string(token), identTypeName)
return _TYPENAME
}
if cn >= '0' && cn <= '9' {
l.colNo++
token := []rune{c, cn}
token = l.readNumber(token, false, true)
f, err := strconv.ParseFloat(string(token), 64)
if err != nil {
setError(err)
return _ERROR
}
setFloat(f)
return _FLOAT_LIT
}
l.input.unreadRune(cn)
setRune()
return int(c)
}
if c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') {
// identifier
token := []rune{c}
token = l.readIdentifier(token)
str := string(token)
if strings.Contains(str, ".") {
setIdent(str, identQualified)
return _FQNAME
}
if t, ok := keywords[str]; ok {
setIdent(str, identSimpleName)
return t
}
setIdent(str, identSimpleName)
return _NAME
}
if c >= '0' && c <= '9' {
// integer or float literal
if c == '0' {
cn, _, err := l.input.readRune()
if err != nil {
setInt(0)
return _INT_LIT
}
if cn == 'x' || cn == 'X' {
cnn, _, err := l.input.readRune()
if err != nil {
l.input.unreadRune(cn)
setInt(0)
return _INT_LIT
}
if (cnn >= '0' && cnn <= '9') || (cnn >= 'a' && cnn <= 'f') || (cnn >= 'A' && cnn <= 'F') {
// hexadecimal!
l.colNo += 2
token := []rune{cnn}
token = l.readHexNumber(token)
ui, err := strconv.ParseUint(string(token), 16, 64)
if err != nil {
setError(err)
return _ERROR
}
setInt(ui)
return _INT_LIT
}
l.input.unreadRune(cnn)
l.input.unreadRune(cn)
setInt(0)
return _INT_LIT
} else {
l.input.unreadRune(cn)
}
}
token := []rune{c}
token = l.readNumber(token, true, true)
numstr := string(token)
if strings.Contains(numstr, ".") || strings.Contains(numstr, "e") || strings.Contains(numstr, "E") {
// floating point!
f, err := strconv.ParseFloat(numstr, 64)
if err != nil {
setError(err)
return _ERROR
}
setFloat(f)
return _FLOAT_LIT
}
// integer! (decimal or octal)
ui, err := strconv.ParseUint(numstr, 0, 64)
if err != nil {
setError(err)
return _ERROR
}
setInt(ui)
return _INT_LIT
}
if c == '\'' || c == '"' {
// string literal
str, err := l.readStringLiteral(c)
if err != nil {
setError(err)
return _ERROR
}
setString(str)
return _STRING_LIT
}
if c == '/' {
// comment
cn, _, err := l.input.readRune()
if err != nil {
setRune()
return int(c)
}
if cn == '/' {
l.colNo++
hitNewline, txt := l.skipToEndOfLineComment()
commentPos := pos()
commentPos.end.Col++
if hitNewline {
l.colNo = 0
l.lineNo++
}
comments = append(comments, &comment{posRange: commentPos, text: txt})
continue
}
if cn == '*' {
l.colNo++
if txt, ok := l.skipToEndOfBlockComment(); !ok {
setError(errors.New("block comment never terminates, unexpected EOF"))
return _ERROR
} else {
comments = append(comments, &comment{posRange: pos(), text: txt})
}
continue
}
l.input.unreadRune(cn)
}
setRune()
return int(c)
}
}
func (l *protoLex) readNumber(sofar []rune, allowDot bool, allowExp bool) []rune {
token := sofar
for {
c, _, err := l.input.readRune()
if err != nil {
break
}
if c == '.' {
if !allowDot {
l.input.unreadRune(c)
break
}
allowDot = false
cn, _, err := l.input.readRune()
if err != nil {
l.input.unreadRune(c)
break
}
if cn < '0' || cn > '9' {
l.input.unreadRune(cn)
l.input.unreadRune(c)
break
}
l.colNo++
token = append(token, c)
c = cn
} else if c == 'e' || c == 'E' {
if !allowExp {
l.input.unreadRune(c)
break
}
allowExp = false
cn, _, err := l.input.readRune()
if err != nil {
l.input.unreadRune(c)
break
}
if cn == '-' || cn == '+' {
cnn, _, err := l.input.readRune()
if err != nil {
l.input.unreadRune(cn)
l.input.unreadRune(c)
break
}
if cnn < '0' || cnn > '9' {
l.input.unreadRune(cnn)
l.input.unreadRune(cn)
l.input.unreadRune(c)
break
}
l.colNo++
token = append(token, c)
c = cn
cn = cnn
} else if cn < '0' || cn > '9' {
l.input.unreadRune(cn)
l.input.unreadRune(c)
break
}
l.colNo++
token = append(token, c)
c = cn
} else if c < '0' || c > '9' {
l.input.unreadRune(c)
break
}
l.colNo++
token = append(token, c)
}
return token
}
func (l *protoLex) readHexNumber(sofar []rune) []rune {
token := sofar
for {
c, _, err := l.input.readRune()
if err != nil {
break
}
if (c < 'a' || c > 'f') && (c < 'A' || c > 'F') && (c < '0' || c > '9') {
l.input.unreadRune(c)
break
}
l.colNo++
token = append(token, c)
}
return token
}
func (l *protoLex) readIdentifier(sofar []rune) []rune {
token := sofar
for {
c, _, err := l.input.readRune()
if err != nil {
break
}
if c == '.' {
cn, _, err := l.input.readRune()
if err != nil {
l.input.unreadRune(c)
break
}
if cn != '_' && (cn < 'a' || cn > 'z') && (cn < 'A' || cn > 'Z') {
l.input.unreadRune(cn)
l.input.unreadRune(c)
break
}
l.colNo++
token = append(token, c)
c = cn
} else if c != '_' && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') && (c < '0' || c > '9') {
l.input.unreadRune(c)
break
}
l.colNo++
token = append(token, c)
}
return token
}
func (l *protoLex) readStringLiteral(quote rune) (string, error) {
var buf bytes.Buffer
for {
c, _, err := l.input.readRune()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return "", err
}
if c == '\n' {
l.colNo = 0
l.lineNo++
return "", errors.New("encountered end-of-line before end of string literal")
}
l.colNo++
if c == quote {
break
}
if c == 0 {
return "", errors.New("null character ('\\0') not allowed in string literal")
}
if c == '\\' {
// escape sequence
c, _, err = l.input.readRune()
if err != nil {
return "", err
}
l.colNo++
if c == 'x' || c == 'X' {
// hex escape
c, _, err := l.input.readRune()
if err != nil {
return "", err
}
l.colNo++
c2, _, err := l.input.readRune()
if err != nil {
return "", err
}
var hex string
if (c2 < '0' || c2 > '9') && (c2 < 'a' || c2 > 'f') && (c2 < 'A' || c2 > 'F') {
l.input.unreadRune(c2)
hex = string(c)
} else {
l.colNo++
hex = string([]rune{c, c2})
}
i, err := strconv.ParseInt(hex, 16, 32)
if err != nil {
return "", fmt.Errorf("invalid hex escape: \\x%q", hex)
}
buf.WriteByte(byte(i))
} else if c >= '0' && c <= '7' {
// octal escape
c2, _, err := l.input.readRune()
if err != nil {
return "", err
}
var octal string
if c2 < '0' || c2 > '7' {
l.input.unreadRune(c2)
octal = string(c)
} else {
l.colNo++
c3, _, err := l.input.readRune()
if err != nil {
return "", err
}
if c3 < '0' || c3 > '7' {
l.input.unreadRune(c3)
octal = string([]rune{c, c2})
} else {
l.colNo++
octal = string([]rune{c, c2, c3})
}
}
i, err := strconv.ParseInt(octal, 8, 32)
if err != nil {
return "", fmt.Errorf("invalid octal escape: \\%q", octal)
}
if i > 0xff {
return "", fmt.Errorf("octal escape is out range, must be between 0 and 377: \\%q", octal)
}
buf.WriteByte(byte(i))
} else if c == 'u' {
// short unicode escape
u := make([]rune, 4)
for i := range u {
c, _, err := l.input.readRune()
if err != nil {
return "", err
}
l.colNo++
u[i] = c
}
i, err := strconv.ParseInt(string(u), 16, 32)
if err != nil {
return "", fmt.Errorf("invalid unicode escape: \\u%q", string(u))
}
buf.WriteRune(rune(i))
} else if c == 'U' {
// long unicode escape
u := make([]rune, 8)
for i := range u {
c, _, err := l.input.readRune()
if err != nil {
return "", err
}
l.colNo++
u[i] = c
}
i, err := strconv.ParseInt(string(u), 16, 32)
if err != nil {
return "", fmt.Errorf("invalid unicode escape: \\U%q", string(u))
}
if i > 0x10ffff || i < 0 {
return "", fmt.Errorf("unicode escape is out of range, must be between 0 and 0x10ffff: \\U%q", string(u))
}
buf.WriteRune(rune(i))
} else if c == 'a' {
buf.WriteByte('\a')
} else if c == 'b' {
buf.WriteByte('\b')
} else if c == 'f' {
buf.WriteByte('\f')
} else if c == 'n' {
buf.WriteByte('\n')
} else if c == 'r' {
buf.WriteByte('\r')
} else if c == 't' {
buf.WriteByte('\t')
} else if c == 'v' {
buf.WriteByte('\v')
} else if c == '\\' {
buf.WriteByte('\\')
} else if c == '\'' {
buf.WriteByte('\'')
} else if c == '"' {
buf.WriteByte('"')
} else if c == '?' {
buf.WriteByte('?')
} else {
return "", fmt.Errorf("invalid escape sequence: %q", "\\"+string(c))
}
} else {
buf.WriteRune(c)
}
}
return buf.String(), nil
}
func (l *protoLex) skipToEndOfLineComment() (bool, string) {
txt := []rune{'/', '/'}
for {
c, _, err := l.input.readRune()
if err != nil {
return false, string(txt)
}
if c == '\n' {
return true, string(txt)
}
l.colNo++
txt = append(txt, c)
}
}
func (l *protoLex) skipToEndOfBlockComment() (string, bool) {
txt := []rune{'/', '*'}
for {
c, _, err := l.input.readRune()
if err != nil {
return "", false
}
if c == '\n' {
l.colNo = 0
l.lineNo++
} else {
l.colNo++
}
txt = append(txt, c)
if c == '*' {
c, _, err := l.input.readRune()
if err != nil {
return "", false
}
if c == '/' {
l.colNo++
txt = append(txt, c)
return string(txt), true
}
l.input.unreadRune(c)
}
}
}
func (l *protoLex) Error(s string) {
if l.err == nil {
l.err = ErrorWithSourcePos{Underlying: errors.New(s), Pos: l.prevSym.start()}
}
}