From 50b9012e10d2fa74294547c25642b4a69fed4bda Mon Sep 17 00:00:00 2001
From: Christopher Durham <cad97@cad97.com>
Date: Sat, 27 Jan 2018 20:29:14 -0500
Subject: Add minimal docs to most public symbols

---
 src/lexer/mod.rs | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'src/lexer/mod.rs')

diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index 2f8d3a402..65a994327 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -17,6 +17,7 @@ use self::strings::{is_string_literal_start, scan_byte_char_or_string, scan_char
 mod comments;
 use self::comments::{scan_comment, scan_shebang};
 
+/// Break a string up into its component tokens
 pub fn tokenize(text: &str) -> Vec<Token> {
     let mut text = text;
     let mut acc = Vec::new();
@@ -28,6 +29,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
     }
     acc
 }
+/// Get the next token from a string
 pub fn next_token(text: &str) -> Token {
     assert!(!text.is_empty());
     let mut ptr = Ptr::new(text);
-- 
cgit v1.2.3