DIRECTORY IO, Rope; TDLexing: CEDAR DEFINITIONS = { ROPE: TYPE = Rope.ROPE; ROPEList: TYPE = LIST OF ROPE; Lexer: TYPE = REF LexerPrivate; LexerPrivate: TYPE = RECORD [ in: IO.STREAM, filterComments: BOOL, GetIndex: IndexGetter, tokenStack: LIST OF Token _ NIL, afterLast, beforeNext: INT _ 0 ]; Token: TYPE = RECORD [ kind: IO.TokenKind, rope: ROPE, afterPrev, before, after: INT ]; MakeLexer: PROC [in: IO.STREAM, GetIndex: IndexGetter _ NIL--means VanillaIndexGetter--, filterComments: BOOL _ TRUE] RETURNS [l: Lexer]; IndexGetter: TYPE = PROC [in: IO.STREAM] RETURNS [index: INT]; VanillaIndexGetter: IndexGetter; --IO.GetIndex[rawIn]-- GetToken: PROC [l: Lexer] RETURNS [token: Token]; GetPosition: PROC [l: Lexer, side: WhichPosition] RETURNS [position: INT]; WhichPosition: TYPE = {afterLast, beforeNext}; ReturnToken: PROC [l: Lexer, token: Token]; GetRope: PROC [l: Lexer, kind: IO.TokenKind] RETURNS [rope: ROPE]; GetKwd: PROC [l: Lexer] RETURNS [kwd: ATOM]; NextIs: PROC [l: Lexer, rope: ROPE] RETURNS [is: BOOL]; NextIsSeq: PROC [l: Lexer, seq: ROPEList] RETURNS [is: BOOL]; NextIsA: PROC [l: Lexer, ropes: ROPEList] RETURNS [is: BOOL]; SkipTo: PROC [l: Lexer, oneOf: ROPEList]; ReservedWord: PROC [id: ROPE] RETURNS [reserved: BOOL]; PredefinedType: PROC [id: ROPE] RETURNS [is: BOOL]; SyntaxError: ERROR; }. ήTDLexing.mesa Mike Spreitzer August 12, 1986 11:46:01 pm PDT The token occupies indices [before .. after) in input stream. rawIn and cookedIn differ by the potential comment filtering. Discards it. Doesn't discard. Κ«– "cedar" style˜code™ K™.—K˜KšΟk œœ˜K˜šΟnœœ œ˜K˜Kšœœœ˜Kš œ œœœœ˜K˜Kšœœœ˜šœœœ˜Kšœœœ˜Kšœœ˜Kšžœ˜Kšœ œœ œ˜ Kšœœ˜K˜—K˜šœœœ˜Kšœœ ˜Kšœœ˜ šœ˜K™=—Kšœ˜—K˜Kšž œœœœžœΟcœœœœ ˜‰K˜š œ œœœœœ œ˜>Kšœ=™=—K˜KšžœŸ˜7K˜Kšžœœ œ˜1šž œœ!œ œ˜JKšœœ˜.—K˜Kšž œœ˜+K˜Kš žœœœ œœ˜BKšžœœ œœ˜,š žœœœœœ˜7K™ —Kšž œœœœ˜=šžœœœœ˜=K™—Kšžœœ˜)K˜Kš ž œœœœ œ˜7Kš žœœœœœ˜3K˜Kšž œœ˜K˜K˜——…—:Γ