diff --git a/decode.go b/decode.go
index df43327..58bca71 100644
--- a/decode.go
+++ b/decode.go
@@ -19,7 +19,7 @@ func parseInteger(b []byte) (int64, error) {
 		case 'o':
 			return parseIntOct(b)
 		default:
-			return 0, fmt.Errorf("invalid base: '%c'", b[1])
+			return 0, newDecodeError(b[1:2], "invalid base: '%c'", b[1])
 		}
 	}
 	return parseIntDec(b)
diff --git a/errors.go b/errors.go
index 80db004..8d31594 100644
--- a/errors.go
+++ b/errors.go
@@ -22,6 +22,24 @@ type DecodeError struct {
 	human string
 }
 
+// internal version of DecodeError that is used as the base to create a
+// DecodeError with full context.
+type decodeError struct {
+	highlight []byte
+	message   string
+}
+
+func (de *decodeError) Error() string {
+	return de.message
+}
+
+func newDecodeError(highlight []byte, format string, args ...interface{}) error {
+	return &decodeError{
+		highlight: highlight,
+		message:   fmt.Sprintf(format, args...),
+	}
+}
+
 // Error returns the error message contained in the DecodeError.
 func (e *DecodeError) Error() string {
 	return e.message
@@ -45,15 +63,18 @@ func (e *DecodeError) Position() (row int, column int) {
 //
 // The function copies all bytes used in DecodeError, so that document and
 // highlight can be freely deallocated.
-func decodeErrorFromHighlight(document []byte, highlight []byte, message string) error {
+func wrapDecodeError(document []byte, de *decodeError) error {
+	if de == nil {
+		return nil
+	}
 	err := &DecodeError{
-		message: message,
+		message: de.message,
 	}
 
-	offset := unsafe.SubsliceOffset(document, highlight)
+	offset := unsafe.SubsliceOffset(document, de.highlight)
 
 	err.line, err.column = positionAtEnd(document[:offset])
-	before, after := linesOfContext(document, highlight, offset, 3)
+	before, after := linesOfContext(document, de.highlight, offset, 3)
 
 	var buf strings.Builder
 
@@ -74,7 +95,7 @@ func decodeErrorFromHighlight(document []byte, highlight []byte, message string)
 	if len(before) > 0 {
 		buf.Write(before[0])
 	}
-	buf.Write(highlight)
+	buf.Write(de.highlight)
 	if len(after) > 0 {
 		buf.Write(after[0])
 	}
@@ -84,7 +105,7 @@ func decodeErrorFromHighlight(document []byte, highlight []byte, message string)
 	if len(before) > 0 {
 		buf.WriteString(strings.Repeat(" ", len(before[0])))
 	}
-	buf.WriteString(strings.Repeat("~", len(highlight)))
+	buf.WriteString(strings.Repeat("~", len(de.highlight)))
 	buf.WriteString(" ")
 	buf.WriteString(err.message)
 
diff --git a/errors_test.go b/errors_test.go
index 9d0c5a9..e1354e2 100644
--- a/errors_test.go
+++ b/errors_test.go
@@ -136,7 +136,10 @@ before `, "highlighted", ``},
 			doc := b.Bytes()
 			hl := doc[start:end]
 
-			err := decodeErrorFromHighlight(doc, hl, e.msg)
+			err := wrapDecodeError(doc, &decodeError{
+				highlight: hl,
+				message:   e.msg,
+			})
 			derr := err.(*DecodeError)
 			assert.Equal(t, strings.Trim(e.expected, "\n"), derr.String())
 		})
diff --git a/unmarshaler.go b/unmarshaler.go
index bb1aff2..b95ce5a 100644
--- a/unmarshaler.go
+++ b/unmarshaler.go
@@ -66,6 +66,17 @@ func (d *decoder) arrayIndex(append bool, v reflect.Value) int {
 }
 
 func (d *decoder) FromParser(p *parser, v interface{}) error {
+	err := d.fromParser(p, v)
+	if err != nil {
+		de, ok := err.(*decodeError)
+		if ok {
+			err = wrapDecodeError(p.data, de)
+		}
+	}
+	return err
+}
+
+func (d *decoder) fromParser(p *parser, v interface{}) error {
 	r := reflect.ValueOf(v)
 	if r.Kind() != reflect.Ptr {
 		return fmt.Errorf("need to target a pointer, not %s", r.Kind())