Implement parser

README.md

@@ -1,3 +1,5 @@
 # go-toml
 
 Go library for the [TOML](https://github.com/mojombo/toml) format.
+
+This library supports TOML version [e3656ad493400895f4460f1244a25f8f8e31a32a](https://github.com/mojombo/toml/tree/e3656ad493400895f4460f1244a25f8f8e31a32a)

src/toml/lexer.go

@@ -39,6 +39,7 @@ const (
 	tokenDate
 	tokenKeyGroup
 	tokenComma
+	tokenEOL
 )
 
 type token struct {
@@ -196,6 +197,9 @@ func lexRvalue(l *lexer) stateFn {
 		case ',':
 			return lexComma
 		case '\n':
+			l.ignore()
+			l.pos += 1
+			l.emit(tokenEOF)
 			return lexVoid
 		}
 

src/toml/parser.go

@@ -3,27 +3,165 @@
 package toml
 
 import (
-	"strings"
+	"fmt"
+	"strconv"
+	"time"
 )
 
-// createSubTree takes a tree and a key andcreate the necessary intermediate
+
-// subtrees to create a subtree at that point. In-place.
+type parser struct {
-//
+	flow chan token
-// e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b]
+	tree *TomlTree
-// and tree[a][b][c]
+	tokensBuffer []token
-func createSubTree(tree *TomlTree, key string) {
+	currentGroup string
-	subtree := tree
-	for _, intermediate_key := range strings.Split(key, ".") {
-		_, exists := (*subtree)[intermediate_key]
-		if !exists {
-			(*subtree)[intermediate_key] = make(TomlTree)
 }
-		subtree = (*subtree)[intermediate_key].(*TomlTree)
+
+type parserStateFn func(*parser) parserStateFn
+
+func (p *parser) run() {
+	for state := parseStart; state != nil; {
+		state = state(p)
 	}
 }
 
+func (p *parser) peek() *token {
+	if len(p.tokensBuffer) != 0 {
+		return &(p.tokensBuffer[0])
+	}
 
-func parse(chan token) *TomlTree {
+	tok, ok := <- p.flow
+	if !ok {
+		return nil
+	}
+	p.tokensBuffer = append(p.tokensBuffer, tok)
+	return &tok
+}
+
+func (p *parser) assume(typ tokenType) {
+	tok := p.getToken()
+	if tok == nil {
+		panic(fmt.Sprintf("was expecting token %s, but token stream is empty", typ))
+	}
+	if tok.typ != typ {
+		panic(fmt.Sprintf("was expecting token %s, but got %s", typ, tok.typ))
+	}
+}
+
+func (p *parser) getToken() *token {
+	if len(p.tokensBuffer) != 0 {
+		tok := p.tokensBuffer[0]
+		p.tokensBuffer = p.tokensBuffer[1:]
+		return &tok
+	}
+	tok, ok := <- p.flow
+	if !ok {
+		return nil
+	}
+	return &tok
+}
+
+func parseStart(p *parser) parserStateFn {
+	tok := p.peek()
+
+	// end of stream, parsing is finished
+	if tok == nil {
+		return nil
+	}
+
+	switch tok.typ {
+	case tokenLeftBracket:
+		return parseGroup
+	case tokenKey:
+		return parseAssign
+	default:
+		panic("unexpected token")
+	}
+	return nil
+}
+
+func parseGroup(p *parser) parserStateFn {
+	p.getToken() // discard the [
+	key := p.getToken()
+	if key.typ != tokenKey {
+		panic(fmt.Sprintf("unexpected token %s, was expecting a key", key))
+	}
+	p.tree.createSubTree(key.val)
+	p.assume(tokenRightBracket)
+	p.currentGroup = key.val
+	return parseStart(p)
+}
+
+func parseAssign(p *parser) parserStateFn {
+	key := p.getToken()
+	p.assume(tokenEqual)
+	value := parseRvalue(p)
+	p.tree.Set(key.val, value)
+	return parseStart(p)
+}
+
+func parseRvalue(p *parser) interface{} {
+	tok := p.getToken()
+	if tok == nil {
+		panic("expecting a value")
+	}
+
+	switch tok.typ {
+	case tokenString:
+		return tok.val
+	case tokenTrue:
+		return true
+	case tokenFalse:
+		return false
+	case tokenInteger:
+		val, err := strconv.ParseInt(tok.val, 10, 64)
+		if err != nil { panic(err) }
+		return val
+	case tokenFloat:
+		val, err := strconv.ParseFloat(tok.val, 64)
+		if err != nil { panic(err) }
+		return val
+	case tokenDate:
+		val, err := time.Parse(time.RFC3339, tok.val)
+		if err != nil { panic(err) }
+		return val
+	case tokenLeftBracket:
+		return parseArray(p)
+	}
+
+	panic("never reached")
+
+	return nil
+}
+
+func parseArray(p *parser) []interface{} {
+	array := make([]interface{}, 0)
+	for {
+		follow := p.peek()
+		if follow == nil { panic("unterminated array") }
+		if follow.typ == tokenRightBracket {
+			p.getToken()
+			return array
+		}
+		val := parseRvalue(p)
+		array = append(array, val)
+		follow = p.peek()
+		if follow == nil { panic("unterminated array") }
+		if follow.typ != tokenRightBracket && follow.typ != tokenComma {
+			panic("missing comma")
+		}
+	}
+	return array
+}
+
+
+func parse(flow chan token) *TomlTree {
 	result := make(TomlTree)
-	return &result
+	parser := &parser {
+		flow: flow,
+		tree: &result,
+		tokensBuffer: make([]token, 0),
+		currentGroup: "",
+	}
+	parser.run()
+	return parser.tree
 }

src/toml/parser_test.go

@@ -3,11 +3,44 @@ package toml
 import "testing"
 
 
+func assertTree(t *testing.T, tree *TomlTree, ref map[string]interface{}) {
+	for k, v := range ref {
+		if tree.Get(k) != v {
+			t.Log("was expecting", v, "at", k, "but got", tree.Get(k))
+			t.Fail()
+		}
+	}
+}
+
 func testCreateSubTree(t *testing.T) {
 	tree := make(TomlTree)
-	createSubTree(&tree, "a.b.c")
+	tree.createSubTree("a.b.c")
 	tree.Set("a.b.c", 42)
 	if tree.Get("a.b.c") != 42 {
 		t.Fail()
 	}
 }
+
+
+func testSimpleKV(t *testing.T) {
+	tree := Load("a = 42")
+	assertTree(t, tree, map[string]interface{}{
+		"a": 42,
+	})
+
+	tree = Load("a = 42\nb = 21")
+	assertTree(t, tree, map[string]interface{}{
+		"a": 42,
+		"b": 21,
+	})
+}
+
+func testSimpleIntegers(t *testing.T) {
+	tree := Load("a = +42\nb = -21\nc = +4.2\nd = -2.1")
+	assertTree(t, tree, map[string]interface{}{
+		"a": 42,
+		"b": -21,
+		"c": 4.2,
+		"d": -4.2,
+	})
+}

src/toml/toml.go

@@ -39,8 +39,24 @@ func (t *TomlTree) Set(key string, value interface{}) {
 	(*subtree)[keys[len(key) - 1]] = value
 }
 
-
+// createSubTree takes a tree and a key andcreate the necessary intermediate
-func Load() TomlTree {
+// subtrees to create a subtree at that point. In-place.
-	result := make(TomlTree)
+//
-	return result
+// e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b]
+// and tree[a][b][c]
+func (t *TomlTree) createSubTree(key string) {
+	subtree := t
+	for _, intermediate_key := range strings.Split(key, ".") {
+		_, exists := (*subtree)[intermediate_key]
+		if !exists {
+			(*subtree)[intermediate_key] = make(TomlTree)
+		}
+		subtree = (*subtree)[intermediate_key].(*TomlTree)
+	}
+}
+
+
+func Load(content string) *TomlTree {
+	_, flow := lex(content)
+	return parse(flow)
 }