RussellScannerImpl.mesa
This module is a scanner for Russell 84.
Last Edited by: Demers, March 5, 1984 4:34:50 pm PST
DIRECTORY
RussellSyntax USING[TokType],
IO USING[STREAM, GetChar, PeekChar, EndOf, Backup, TokenKind, SkipWhitespace, EndOfStream, GetCedarToken, DEL, NUL, SP],
Rope USING[FromRefText],
Atom USING[MakeAtomFromRefText],
RefText USING[InlineAppendChar, Length];
RussellScannerImpl: CEDAR PROGRAM
IMPORTS IO, Rope, Atom, RefText
EXPORTS RussellSyntax
= BEGIN
OPEN
RussellSyntax
;
TokTypeFromAtom: PRIVATE PROCEDURE [name: ATOM] RETURNS [TokType]
= BEGIN
RETURN[ SELECT name FROM
$tuple => tokKWtuple ,
$func => tokKWfunc ,
$proc => tokKWproc ,
$ref => tokKWref ,
$prod => tokKWprod ,
$union => tokKWunion ,
$if => tokKWif ,
$else => tokKWelse ,
$fi => tokKWfi ,
$do => tokKWdo ,
$od => tokKWod ,
$open => tokKWopen ,
$in => tokKWin ,
$ni => tokKWni ,
$lambda => tokKWlambda ,
$type => tokKWtype ,
$safetype => tokKWsafetype ,
ENDCASE => tokId ]
END ; -- TokTypeFromAtom
GetRussellToken: PUBLIC PROCEDURE [source: IO.STREAM, buffer: REF TEXT] RETURNS [tokType: TokType, tokVal: REF ANY ← NIL, charsRead: INT]
= BEGIN
c: CHAR;
nRead: INT;
nSkipped: INT;
tokKind: IO.TokenKind;
tokName: ATOM;
scanBuf: REF TEXT ← buffer;
BEGIN
Consume leading whitespace / comments to lookahead character.
nRead ← IO.SkipWhitespace[stream~source
!IO.EndOfStream => GOTO endOfFile ];
c ← source.GetChar[
!IO.EndOfStream => GOTO endOfFile ];
nRead ← nRead + 1;
Switch based on first nonwhite character of token
SELECT c FROM
IN ['a .. 'z], IN ['A .. 'Z] => {
source.Backup[c];
nRead ← nRead - 1;
[tokenKind~tokKind, token~scanBuf, charsSkipped~nSkipped]
← IO.GetCedarToken[stream~source, buffer~buffer];
nRead ← nRead + nSkipped;
IF tokKind # tokenERROR
THEN nRead ← nRead + RefText.Length[scanBuf];
IF tokKind # tokenID
THEN RETURN[tokType~tokError, charsRead~nRead] ;
tokName ← Atom.MakeAtomFromRefText[scanBuf];
RETURN[tokType~TokTypeFromAtom[tokName], tokVal~tokName, charsRead~nRead]
} ;
IN ['0 .. '9], '', '" => {
source.Backup[c];
nRead ← nRead - 1;
[tokenKind~tokKind, token~scanBuf, charsSkipped~nSkipped]
← IO.GetCedarToken[stream~source, buffer~buffer];
nRead ← nRead + nSkipped;
IF tokKind # tokenERROR
THEN nRead ← nRead + RefText.Length[scanBuf];
SELECT tokKind FROM
tokenDECIMAL, tokenOCTAL, tokenHEX =>
RETURN[tokType~tokIntConst, tokVal~Rope.FromRefText[scanBuf], charsRead~nRead] ;
tokenCHAR =>
RETURN[tokType~tokCharConst, tokVal~Rope.FromRefText[scanBuf], charsRead~nRead] ;
tokenROPE =>
RETURN[tokType~tokStringConst, tokVal~Rope.FromRefText[scanBuf], charsRead~nRead] ;
ENDCASE =>
RETURN[tokType~tokError, charsRead~nRead] ;
} ;
'< => {
IF (NOT source.EndOf[])
THEN BEGIN
SELECT source.PeekChar[] FROM
'* => {
[] ← source.GetChar[];
nRead ← nRead+1;
RETURN[tokType~tokLRecTupleBrak, charsRead~nRead];
} ;
ENDCASE =>
NULL ;
END ;
RETURN[tokType~tokLTupleBrak, charsRead~nRead];
} ;
'* => {
IF (NOT source.EndOf[])
THEN BEGIN
SELECT source.PeekChar[] FROM
'> => {
[] ← source.GetChar[];
nRead ← nRead+1;
RETURN[tokType~tokRRecTupleBrak, charsRead~nRead];
} ;
ENDCASE =>
NULL ;
END ;
} ;
'= => {
IF (NOT source.EndOf[])
THEN BEGIN
SELECT source.PeekChar[] FROM
'> => {
[] ← source.GetChar[];
nRead ← nRead+1;
RETURN[tokType~tokGuardArrow, charsRead~nRead];
} ;
ENDCASE =>
NULL ;
END ;
} ;
'- => {
IF (NOT source.EndOf[])
THEN BEGIN
SELECT source.PeekChar[] FROM
'> => {
[] ← source.GetChar[];
nRead ← nRead+1;
RETURN[tokType~tokFunctionArrow, charsRead~nRead];
} ;
ENDCASE =>
NULL ;
END ;
} ;
'. =>
RETURN[tokType~tokSelect, charsRead~nRead] ;
', =>
RETURN[tokType~tokListSep, charsRead~nRead] ;
'| =>
RETURN[tokType~tokConcat, charsRead~nRead] ;
'? =>
RETURN[tokType~tokQuery, charsRead~nRead] ;
'~ =>
RETURN[tokType~tokIsBoundTo, charsRead~nRead] ;
'> =>
RETURN[tokType~tokRTupleBrak, charsRead~nRead] ;
'; =>
RETURN[tokType~tokStmtSep, charsRead~nRead] ;
': =>
RETURN[tokType~tokHasType, charsRead~nRead] ;
'[ =>
RETURN[tokType~tokLBrak, charsRead~nRead] ;
'] =>
RETURN[tokType~tokRBrak, charsRead~nRead] ;
'{ =>
RETURN[tokType~tokLUnionBrak, charsRead~nRead] ;
'} =>
RETURN[tokType~tokRUnionBrak, charsRead~nRead] ;
'( =>
RETURN[tokType~tokLParen, charsRead~nRead] ;
') =>
RETURN[tokType~tokRParen, charsRead~nRead] ;
'# =>
RETURN[tokType~tokGuardedExpSep, charsRead~nRead] ;
IO.DEL =>
RETURN[tokType~tokError, charsRead~nRead] ;
ENDCASE =>
NULL ;
Default otherwise unrecognized tokens to operator identifiers.
scanBuf.length ← 0;
scanBuf ← RefText.InlineAppendChar[scanBuf, c ];
DO
c ← source.GetChar[ !IO.EndOfStream => EXIT ];
nRead ← nRead+1;
SELECT c FROM
IN [IO.NUL .. IO.SP], IN ['a .. 'z], IN ['A .. 'Z], IN ['0 .. '9], '', '", '<, '., ',, '|, '?, '~, '>, ';, ':, '[, '], '{, '}, '(, '), '#, IO.DEL => {
source.Backup[c];
nRead ← nRead-1;
EXIT
} ;
ENDCASE =>
scanBuf ← RefText.InlineAppendChar[scanBuf, c ] ;
ENDLOOP ;
RETURN[tokType~tokId, tokVal~Atom.MakeAtomFromRefText[scanBuf], charsRead~nRead] ;
EXITS
endOfFile => RETURN[tokType~tokEOF, charsRead~nRead] ;
END
END ; -- GetRussellToken
END .