
<<RussellScannerImpl.mesa>>
    <<This module is a scanner for Russell 84.>>
    <<Last Edited by: Demers, March 5, 1984 4:34:50 pm PST>>
DIRECTORY
    RussellSyntax USING[TokType],
    IO USING[STREAM, GetChar, PeekChar, EndOf, Backup, TokenKind, SkipWhitespace, EndOfStream, GetCedarToken, DEL, NUL, SP],
    Rope USING[FromRefText],
    Atom USING[MakeAtomFromRefText],
    RefText USING[InlineAppendChar, Length];
RussellScannerImpl: CEDAR PROGRAM
    IMPORTS IO, Rope, Atom, RefText
    EXPORTS RussellSyntax
    = BEGIN
        OPEN
            RussellSyntax
            ;
    TokTypeFromAtom: PRIVATE PROCEDURE [name: ATOM] RETURNS [TokType]
        = BEGIN
            RETURN[ SELECT name FROM
                $tuple => tokKWtuple ,
                $func => tokKWfunc ,
                $proc => tokKWproc ,
                $ref => tokKWref ,
                $prod => tokKWprod ,
                $union => tokKWunion ,
                $if => tokKWif ,
                $else => tokKWelse ,
                $fi => tokKWfi ,
                $do => tokKWdo ,
                $od => tokKWod ,
                $open => tokKWopen ,
                $in => tokKWin ,
                $ni => tokKWni ,
                $lambda => tokKWlambda ,
                $type => tokKWtype ,
                $safetype => tokKWsafetype ,
                ENDCASE => tokId ]
            END ; -- TokTypeFromAtom
    GetRussellToken: PUBLIC PROCEDURE [source: IO.STREAM, buffer: REF TEXT] RETURNS [tokType: TokType, tokVal: REF ANY _ NIL, 
    charsRead: INT]
        = BEGIN
        c: CHAR;
            nRead: INT;
            nSkipped: INT;
            tokKind: IO.TokenKind;
            tokName: ATOM;
            scanBuf: REF TEXT _ buffer;
        BEGIN
        <<Consume leading whitespace / comments to lookahead character.>>
            nRead _ IO.SkipWhitespace[stream~source
                !IO.EndOfStream => GOTO endOfFile ];
            c _ source.GetChar[
                !IO.EndOfStream => GOTO endOfFile ];
            nRead _ nRead + 1;
        <<Switch based on first nonwhite character of token>>
            SELECT c FROM
            IN ['a .. 'z], IN ['A .. 'Z] => {
                source.Backup[c];
                nRead _ nRead - 1;
                [tokenKind~tokKind, token~scanBuf, charsSkipped~nSkipped]
                    _ IO.GetCedarToken[stream~source, buffer~buffer];
                nRead _ nRead + nSkipped;
                IF tokKind # tokenERROR
                    THEN nRead _ nRead + RefText.Length[scanBuf];
                IF tokKind # tokenID
                    THEN RETURN[tokType~tokError, charsRead~nRead] ;
                tokName _ Atom.MakeAtomFromRefText[scanBuf];
                RETURN[tokType~TokTypeFromAtom[tokName], tokVal~tokName, charsRead~nRead]
                } ;
            IN ['0 .. '9], '', '" => {
                source.Backup[c];
                nRead _ nRead - 1;
                [tokenKind~tokKind, token~scanBuf, charsSkipped~nSkipped]
                    _ IO.GetCedarToken[stream~source, buffer~buffer];
                nRead _ nRead + nSkipped;
                IF tokKind # tokenERROR
                    THEN nRead _ nRead + RefText.Length[scanBuf];
                SELECT tokKind FROM
                    tokenDECIMAL, tokenOCTAL, tokenHEX =>
                        RETURN[tokType~tokIntConst, tokVal~Rope.FromRefText[scanBuf], charsRead~nRead] ;
                    tokenCHAR =>
                        RETURN[tokType~tokCharConst, tokVal~Rope.FromRefText[scanBuf], charsRead~nRead] ;
                    tokenROPE =>
                        RETURN[tokType~tokStringConst, tokVal~Rope.FromRefText[scanBuf], charsRead~nRead] ;
                    ENDCASE =>
                        RETURN[tokType~tokError, charsRead~nRead] ;
                } ;
            '< => {
                IF (NOT source.EndOf[])
                    THEN BEGIN
                        SELECT source.PeekChar[] FROM
                            '* => {
                                [] _ source.GetChar[];
                                nRead _ nRead+1;
                                RETURN[tokType~tokLRecTupleBrak, charsRead~nRead];
                                } ;
                            ENDCASE =>
                                NULL ;
                        END ;
                RETURN[tokType~tokLTupleBrak, charsRead~nRead];
                } ;
            '* => {
                IF (NOT source.EndOf[])
                    THEN BEGIN
                        SELECT source.PeekChar[] FROM
                            '> => {
                                [] _ source.GetChar[];
                                nRead _ nRead+1;
                                RETURN[tokType~tokRRecTupleBrak, charsRead~nRead];
                                } ;
                            ENDCASE =>
                                NULL ;
                        END ;
                } ;
            '= => {
                IF (NOT source.EndOf[])
                    THEN BEGIN
                        SELECT source.PeekChar[] FROM
                            '> => {
                                [] _ source.GetChar[];
                                nRead _ nRead+1;
                                RETURN[tokType~tokGuardArrow, charsRead~nRead];
                                } ;
                            ENDCASE =>
                                NULL ;
                        END ;
                } ;
            '- => {
                IF (NOT source.EndOf[])
                    THEN BEGIN
                        SELECT source.PeekChar[] FROM
                            '> => {
                                [] _ source.GetChar[];
                                nRead _ nRead+1;
                                RETURN[tokType~tokFunctionArrow, charsRead~nRead];
                                } ;
                            ENDCASE =>
                                NULL ;
                        END ;
                } ;
            '. =>
                RETURN[tokType~tokSelect, charsRead~nRead] ;
                ', =>
                    RETURN[tokType~tokListSep, charsRead~nRead] ;
                '| =>
                    RETURN[tokType~tokConcat, charsRead~nRead] ;
                '? =>
                    RETURN[tokType~tokQuery, charsRead~nRead] ;
                '~ =>
                    RETURN[tokType~tokIsBoundTo, charsRead~nRead] ;
                '> =>
                    RETURN[tokType~tokRTupleBrak, charsRead~nRead] ;
                '; =>
                    RETURN[tokType~tokStmtSep, charsRead~nRead] ;
                ': =>
                    RETURN[tokType~tokHasType, charsRead~nRead] ;
                '[ =>
                    RETURN[tokType~tokLBrak, charsRead~nRead] ;
                '] =>
                    RETURN[tokType~tokRBrak, charsRead~nRead] ;
                '{ =>
                    RETURN[tokType~tokLUnionBrak, charsRead~nRead] ;
                '} =>
                    RETURN[tokType~tokRUnionBrak, charsRead~nRead] ;
                '( =>
                    RETURN[tokType~tokLParen, charsRead~nRead] ;
                ') =>
                    RETURN[tokType~tokRParen, charsRead~nRead] ;
                '# =>
                    RETURN[tokType~tokGuardedExpSep, charsRead~nRead] ;
            IO.DEL =>
                RETURN[tokType~tokError, charsRead~nRead] ;
            ENDCASE =>
                NULL ;
        <<Default otherwise unrecognized tokens to operator identifiers.>>
            scanBuf.length _ 0;
            scanBuf _ RefText.InlineAppendChar[scanBuf, c ];
            DO
                c _ source.GetChar[ !IO.EndOfStream => EXIT ];
                nRead _ nRead+1;
                SELECT c FROM
                    IN [IO.NUL .. IO.SP], IN ['a .. 'z], IN ['A .. 'Z], IN ['0 .. '9], '', '", '<, '., ',, '|, '?, '~, '>, ';, ':, '[, '], '{, '}, '(, 
                    '), '#, IO.DEL => {
                        source.Backup[c];
                        nRead _ nRead-1;
                        EXIT
                        } ;
                    ENDCASE =>
                        scanBuf _ RefText.InlineAppendChar[scanBuf, c ] ;
                ENDLOOP ;
            RETURN[tokType~tokId, tokVal~Atom.MakeAtomFromRefText[scanBuf], charsRead~nRead] ;
        EXITS
            endOfFile => RETURN[tokType~tokEOF, charsRead~nRead] ;
            END
        END ; -- GetRussellToken
        END .