TDLexing:
CEDAR
DEFINITIONS = {
ROPE: TYPE = Rope.ROPE;
ROPEList: TYPE = LIST OF ROPE;
Lexer: TYPE = REF LexerPrivate;
LexerPrivate:
TYPE =
RECORD [
in: IO.STREAM,
filterComments: BOOL,
GetIndex: IndexGetter,
tokenStack: LIST OF Token ← NIL,
afterLast, beforeNext: INT ← 0
];
Token:
TYPE =
RECORD [
kind: IO.TokenKind,
rope: ROPE,
afterPrev, before, after:
INT
The token occupies indices [before .. after) in input stream.
];
MakeLexer: PROC [in: IO.STREAM, GetIndex: IndexGetter ← NIL--means VanillaIndexGetter--, filterComments: BOOL ← TRUE] RETURNS [l: Lexer];
IndexGetter:
TYPE =
PROC [in:
IO.
STREAM]
RETURNS [index:
INT];
rawIn and cookedIn differ by the potential comment filtering.
VanillaIndexGetter: IndexGetter; --IO.GetIndex[rawIn]--
GetToken: PROC [l: Lexer] RETURNS [token: Token];
GetPosition:
PROC [l: Lexer, side: WhichPosition]
RETURNS [position:
INT];
WhichPosition: TYPE = {afterLast, beforeNext};
ReturnToken: PROC [l: Lexer, token: Token];
GetRope: PROC [l: Lexer, kind: IO.TokenKind] RETURNS [rope: ROPE];
GetKwd: PROC [l: Lexer] RETURNS [kwd: ATOM];
NextIs:
PROC [l: Lexer, rope:
ROPE]
RETURNS [is:
BOOL];
Discards it.
NextIsSeq: PROC [l: Lexer, seq: ROPEList] RETURNS [is: BOOL];
NextIsA:
PROC [l: Lexer, ropes: ROPEList]
RETURNS [is:
BOOL];
Doesn't discard.
SkipTo: PROC [l: Lexer, oneOf: ROPEList];
ReservedWord: PROC [id: ROPE] RETURNS [reserved: BOOL];
PredefinedType: PROC [id: ROPE] RETURNS [is: BOOL];
SyntaxError: ERROR;
}.