<<>>
<<LexerImpl.mesa>>
<<Copyright Ó 1992, 1993 by Xerox Corporation.  All rights reserved.>>
<<Hopcroft,  July 27, 1989 0:18:47 am PDT>>
<<Coolidge, July 12, 1990 3:50 pm PDT>>
<<Laurie Horton, November 8, 1991 4:03 pm PST>>
<<Willie-s, March 29, 1993 2:53 pm PST>>

    DIRECTORY
        Ascii USING [SP, CR, LF, FF, TAB, BS, BEL],
        Commander USING[CommandProc, Register],
        Convert USING [CardFromRope, IntFromRope, RealFromRope],
        PFS USING [PathFromRope, StreamOpen],
        PFSNames USING [PATH],
        IO USING[Close, EndOfStream, GetChar, GetTokenRope, int, IDProc, PeekChar, PutF1, PutRope, RIS, rope, STREAM, TokenKind],
        Lexer USING [],
        Rope USING[Cat, Concat, Fetch, FromChar, Length, ROPE, Substr];


    LexerImpl: CEDAR PROGRAM
    IMPORTS Commander, Convert, IO, PFS, Rope EXPORTS Lexer ~ {

    <<   TokenKind: TYPE  = { tokenERROR, tokenID, tokenDECIMAL, tokenOCTAL, tokenHEX, tokenREAL, tokenROPE, tokenCHAR, tokenATOM, 
    tokenSINGLE, tokenDOUBLE, tokenTRIPLE, tokenCOMMENT, tokenEOF, tokenNUMBER };>>

        State: TYPE = { LeadingSpace, StartComment, Comment, EndComment, Char, String, CharStringIdentifier, FloatIntHexOct, Identifier, 
        Punctuation,  FloatHexOctal, FloatOctal, Hex, FloatInt, IntModUORL, IntModU, IntModL, Float, Fraction,  FractionOrDot, 
        ExponentSign, Exponent, FloatModFOrL, Done, Error };

        inputStream: IO.STREAM;
        haveAChar: BOOL;
        thisChar: CHAR;
        <<currentChar: CHAR;>>

        GetNextChar:  PROC [] RETURNS [CHAR] ~ {
            RETURN[inputStream.GetChar[]]
            <<c:CHAR _ PeekNextChar[];>>
            <<haveAChar _ FALSE;>>
            <<RETURN [c]>>
            };

        PeekNextChar: PROC [] RETURNS [CHAR] ~ {
            RETURN[inputStream.PeekChar[]]
            <<IF haveAChar THEN >>
                <<RETURN [currentChar]>>
            <<ELSE {>>
                <<currentChar _ inputStream.GetChar[];>>
                <<haveAChar _ TRUE;>>
                <<RETURN [currentChar]>>
                <<}>>
            };

        IsPunctuation: PROC [c:CHAR] RETURNS [BOOL] ~ {
            RETURN [SELECT c FROM
                '!, '*, '%, '/, '^, '&, '(, '), ',, '., ':, '?, '[, '], '{, '}, '~, '+, '-, '<, '>, ';, '=, '', '", '| => TRUE
                ENDCASE => FALSE]
            };

        IsDecimalDigit: PROC [c:CHAR] RETURNS [BOOL]
            = INLINE { RETURN [c IN ['0 .. '9]] };

        Is1To9: PROC [c:CHAR] RETURNS [BOOL]
            = INLINE { RETURN [c IN ['1 .. '9]] };

        IsHexDigit: PROC [c:CHAR] RETURNS [BOOL]
            = INLINE { RETURN [c IN ['0 .. '9] OR c IN ['A .. 'F] OR c IN ['a .. 'f]] };

        IsOctalDigit: PROC [c:CHAR] RETURNS [BOOL]
            = INLINE { RETURN [c IN ['0 .. '7]] };

        IsAlphaNum: PROC [c:CHAR] RETURNS [BOOL]
            = INLINE { RETURN [c IN ['0 .. '9] OR c IN ['a .. 'z] OR c IN ['A .. 'Z] OR c='_] };

        IsNonDigit: PROC [c:CHAR] RETURNS [BOOL]
            = INLINE { RETURN [c IN ['a .. 'z] OR c IN ['A .. 'Z] OR c='_] };

        IsWhite: PROC [c:CHAR] RETURNS [BOOL]
            = INLINE { RETURN [c = Ascii.SP OR c = Ascii.CR OR c = Ascii.LF OR c = Ascii.TAB] };

        GetCTokenInit: PUBLIC PROC [] RETURNS [] ~ {
            haveAChar Ź FALSE;
            };

        GetCTokenRope: PUBLIC PROC [inStream:IO.STREAM, junk:BOOL] RETURNS [tokenKind:IO.TokenKind,token:Rope.ROPE,charsSkipped:INT] ~ {
            tokenValue:REF ANY;
            [tokenKind, token, charsSkipped, tokenValue] Ź GetCTokenRopeAndValue[inStream, junk];
            RETURN [tokenKind, token, charsSkipped]
            };

        GetCTokenRopeAndValue: PUBLIC PROC [inStream:IO.STREAM, junk:BOOL] 
        RETURNS [tokenKind:IO.TokenKind,token:Rope.ROPE,charsSkipped:INT, 
        tokenValue:REF ANY] ~ {
            state: State Ź LeadingSpace;
            tokenValid: BOOL Ź FALSE;
            inputStream Ź inStream;
            token Ź NIL;
            charsSkipped Ź 0;

            WHILE state # Done DO 
                BEGIN
                    ENABLE IO.EndOfStream => {
                        IF tokenValid THEN
                            GOTO GoodExit
                        ELSE
                            GOTO BadExit
                        };

                    SELECT state FROM
                        LeadingSpace => {
                            tokenKind Ź tokenEOF;
                            tokenValid Ź TRUE;
                            WHILE IsWhite[PeekNextChar[]] DO
                                [] Ź  GetNextChar[]
                                ENDLOOP;
                            tokenValid Ź FALSE;
                            state Ź SELECT thisChar Ź PeekNextChar[] FROM
                                '' => Char,
                                '" => String,
                                'L => CharStringIdentifier,
                                '/ => StartComment,
                                '. => FractionOrDot,
                                ENDCASE => 
                                    SELECT TRUE FROM
                                        IsDecimalDigit[thisChar] => FloatIntHexOct,
                                        IsNonDigit[thisChar] => Identifier,
                                        IsPunctuation[thisChar] => Punctuation,
                                        ENDCASE => Error;
                            };
                        Char => {
                            [] Ź GetNextChar[];
                            token Ź "'";
                            IF (thisChar Ź GetNextChar[]) =  '\\ THEN
                                token Ź token.Concat[EscapeChar[]]
                            ELSE
                                token Ź token.Concat[Rope.FromChar[thisChar]];
                            tokenKind Ź tokenCHAR;
                            IF GetNextChar[] # '' THEN 
                                state Ź Error
                            ELSE {
                                token Ź token.Concat["'"];
                                tokenValid Ź TRUE;
                                state Ź Done
                                }
                            };
                        String => {
                            [] Ź GetNextChar[];
                            token Ź "\"";
                            WHILE PeekNextChar[] # '"  DO
                                thisChar Ź GetNextChar[];
                                IF thisChar = '\\ THEN 
                                    token Ź token.Concat[EscapeChar[]]
                                ELSE
                                    token Ź token.Concat[Rope.FromChar[thisChar]]
                                ENDLOOP;
                            tokenKind Ź tokenROPE;
                            IF GetNextChar[] # '" THEN
                                state Ź Error
                            ELSE {
                                token Ź token.Concat["\""];
                                tokenValid Ź TRUE;
                                state Ź Done
                                }
                            };
                        CharStringIdentifier => {
                            [] Ź GetNextChar[];
                            token Ź "L";
                            tokenKind Ź tokenID;
                            tokenValid Ź TRUE;
                            thisChar Ź PeekNextChar[];
                            SELECT thisChar FROM
                                '' => {
                                    tokenValid Ź FALSE;
                                    state Ź Char
                                    };
                                '" => {
                                    tokenValid Ź FALSE;
                                    state Ź String
                                    };
                                ENDCASE => {
                                    tokenKind Ź tokenID;
                                    token Ź "L";
                                    WHILE IsAlphaNum[PeekNextChar[]] DO
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]]
                                    ENDLOOP;
                                    state Ź Done; 
                                    };
                            };
                        Identifier => {
                            tokenKind Ź tokenID;
                            tokenValid Ź TRUE;
                            [] Ź GetNextChar[];
                            token Ź Rope.FromChar[thisChar];
                            WHILE IsAlphaNum[PeekNextChar[]] DO
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]]
                            ENDLOOP;
                            state Ź Done
                            };
                        StartComment => {
                            tokenKind Ź tokenSINGLE;
                            token Ź "/";
                            tokenValid Ź TRUE;
                            [] Ź GetNextChar[];
                            IF (thisChar Ź PeekNextChar[]) = '* THEN {
                                tokenValid Ź FALSE;
                                [] Ź GetNextChar[];
                                state Ź Comment;
                                }
                            ELSE IF thisChar = '= THEN {
                                [] Ź GetNextChar[];
                                token Ź "/=";
                                tokenKind Ź tokenDOUBLE;
                                state Ź Done
                                }
                            ELSE 
                                state Ź Done
                            };
                        Comment => {
                            WHILE  PeekNextChar[] # '* DO
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]]
                                ENDLOOP;
                            state Ź EndComment;
                            };
                        EndComment => {
                            [] Ź GetNextChar[];
                            IF PeekNextChar[] = '/ THEN {
                                [] Ź GetNextChar[];
                                tokenKind Ź tokenCOMMENT;
                                tokenValid Ź TRUE;
                                state Ź Done;
                                }
                            ELSE {
                                token Ź token.Concat["*"];
                                state Ź Comment;
                                };
                            };
                        Punctuation => {
                            thisChar Ź GetNextChar[];
                            token Ź Rope.FromChar[thisChar];
                            tokenKind Ź tokenSINGLE;
                            tokenValid Ź TRUE;
                            state Ź Done;
                            SELECT thisChar FROM
                                '!, '*, '%, '/, '=, '^ => {
                                    IF PeekNextChar[] = '= THEN {
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                        tokenKind Ź tokenDOUBLE;
                                        };
                                    };
                                '| => {
                                    IF PeekNextChar[] = '= OR  PeekNextChar[] = '| THEN {
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]];  
                                        tokenKind Ź tokenDOUBLE;
                                        }
                                    };
                                '& => {
                                    IF PeekNextChar[] = '= OR  PeekNextChar[] = '& THEN {
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]];  
                                        tokenKind Ź tokenDOUBLE;
                                        };
                                    };
                                '(, '), ',, '., ':, '; , '?, '[, '], '{, '}, '~ => tokenKind Ź tokenSINGLE;
                                '+ => {
                                    IF PeekNextChar[] = '= OR  PeekNextChar[] = '+ THEN {
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                        tokenKind Ź tokenDOUBLE;
                                        };
                                    };
                                '- => {
                                    IF PeekNextChar[] = '= OR  PeekNextChar[] = '- OR PeekNextChar[] = '> THEN {
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                        tokenKind Ź tokenDOUBLE;
                                        };
                                    };
                                '< => 
                                    IF PeekNextChar[] = '= THEN {
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                        tokenKind Ź tokenDOUBLE;
                                        }
                                    ELSE IF PeekNextChar[]= '< THEN {
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                        tokenKind Ź tokenDOUBLE;
                                        IF PeekNextChar[] = '= THEN {
                                            token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                            }
                                        };
                                '> => 
                                    IF PeekNextChar[]='= THEN {
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                        tokenKind Ź tokenDOUBLE;
                                        }
                                    ELSE IF PeekNextChar[]= '> THEN {
                                        token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                        tokenKind Ź tokenDOUBLE;
                                        IF PeekNextChar[] = '= THEN {
                                            token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                            }
                                        };
                                ENDCASE => {
                                    tokenValid Ź FALSE;
                                    state Ź Error
                                    }
                            };
                        FloatIntHexOct => {
                            thisChar Ź PeekNextChar[];
                            state Ź 
                                IF thisChar = '0 THEN
                                    FloatHexOctal
                                ELSE IF thisChar = '. THEN
                                    Fraction
                                ELSE IF Is1To9[thisChar] THEN
                                    FloatInt
                                ELSE
                                    Error;
                            };
                        FloatHexOctal => {
                            token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            tokenKind Ź tokenOCTAL;
                            tokenValid Ź TRUE;
                            thisChar Ź PeekNextChar[];
                            IF thisChar = 'x THEN {
                                tokenValid Ź FALSE;
                                state Ź Hex
                                }
                            ELSE IF thisChar = '8 OR thisChar = '9 THEN {
                                tokenKind Ź tokenREAL;
                                state Ź Float
                                }
                            ELSE IF IsOctalDigit[thisChar] THEN
                                state Ź FloatOctal
                            ELSE IF thisChar = '. THEN {
                                tokenKind Ź tokenREAL;
                                state Ź Fraction
                                }
                            ELSE IF thisChar = 'E OR thisChar = 'e THEN {
                                tokenValid Ź FALSE;
                                state Ź ExponentSign
                                }
                            ELSE {
                                tokenKind Ź tokenOCTAL;
                                state Ź IntModUORL;
                                }
                            };
                        FloatOctal => {
                            token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            thisChar Ź PeekNextChar[];
                            IF thisChar = '8 OR thisChar = '9 THEN {
                                tokenKind Ź tokenREAL;
                                state Ź Float
                                }
                            ELSE IF IsOctalDigit[thisChar] THEN
                                state Ź FloatOctal
                            ELSE IF thisChar = '. THEN {
                                tokenKind Ź tokenREAL;
                                state Ź Fraction
                                }
                            ELSE IF thisChar = 'E OR thisChar = 'e THEN {
                                tokenValid Ź FALSE;
                                state Ź ExponentSign
                                }
                            ELSE {
                                state Ź IntModUORL;
                                tokenKind Ź tokenOCTAL
                                };
                            };
                        Hex => {
                            token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            tokenKind Ź tokenHEX;
                            WHILE IsHexDigit[PeekNextChar[]] DO
                                tokenValid Ź TRUE;
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                ENDLOOP;
                            state Ź IntModUORL;
                            };
                        FloatInt => {
                            tokenKind Ź tokenDECIMAL;
                            tokenValid Ź TRUE;
                            token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            thisChar Ź PeekNextChar[];
                            IF thisChar = '. THEN
                                state Ź Fraction
                            ELSE IF thisChar = 'e OR thisChar = 'E THEN {
                                tokenValid Ź FALSE;
                                state Ź ExponentSign
                                }
                            ELSE IF NOT IsDecimalDigit[thisChar] THEN {
                                tokenKind Ź tokenDECIMAL;
                                state Ź IntModUORL;
                                };
                            };
                        IntModUORL => {
                            thisChar Ź PeekNextChar[];
                            IF thisChar = 'u OR thisChar = 'U THEN {
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                state Ź IntModL;
                                }
                            ELSE IF thisChar = 'l OR thisChar = 'L THEN {
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                state Ź IntModU;
                                }
                            ELSE
                                state Ź Done;
                            };
                        IntModU => {
                            thisChar Ź PeekNextChar[];
                            IF thisChar = 'u OR thisChar = 'U THEN 
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            state Ź Done
                            };
                        IntModL => {
                            thisChar Ź PeekNextChar[];
                            IF thisChar = 'l OR thisChar = 'L THEN 
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            state Ź Done
                            };
                        Float => {
                            token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            thisChar Ź PeekNextChar[];
                            IF thisChar = '. THEN
                                state Ź Fraction
                            ELSE IF thisChar = 'e OR thisChar = 'E THEN {
                                tokenValid Ź FALSE;
                                state Ź ExponentSign
                                }
                            ELSE IF NOT IsDecimalDigit[thisChar] THEN {
                                tokenKind Ź tokenREAL;
                                state Ź FloatModFOrL;
                                };
                            };
                        FractionOrDot => {
                            token Ź ".";
                            [] Ź GetNextChar[];
                            tokenKind Ź tokenSINGLE;
                            tokenValid Ź TRUE;
                            thisChar Ź PeekNextChar[];
                            IF IsDecimalDigit[thisChar] THEN {
                                tokenKind Ź tokenREAL;
                                state Ź Fraction
                                }
                            ELSE {
                                tokenKind Ź tokenSINGLE;
                                state Ź Done
                                }
                            };
                        Fraction => {
                            token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            tokenKind Ź tokenREAL;
                            thisChar Ź PeekNextChar[];
                            IF thisChar = 'e OR thisChar = 'E THEN {
                                tokenValid Ź FALSE;
                                state Ź ExponentSign
                                }
                            ELSE IF NOT IsDecimalDigit[thisChar] THEN {
                                state Ź FloatModFOrL;
                                };
                            };
                        ExponentSign => {
                            tokenKind Ź tokenREAL;
                            token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            thisChar Ź PeekNextChar[];
                            IF thisChar = '+ OR thisChar = '- THEN 
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            IF IsDecimalDigit[PeekNextChar[]] THEN {
                                tokenValid Ź TRUE;
                                state Ź Exponent
                                }
                            ELSE
                                state Ź Error;
                            };
                        Exponent => {
                            WHILE IsDecimalDigit[PeekNextChar[]] DO
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                                ENDLOOP;
                            tokenKind Ź tokenREAL;
                            state Ź FloatModFOrL;
                            };
                        FloatModFOrL => {
                            thisChar Ź PeekNextChar[];
                            IF thisChar = 'f OR thisChar = 'F OR thisChar = 'l OR thisChar = 'L THEN
                                token Ź token.Concat[Rope.FromChar[GetNextChar[]]];
                            state Ź Done;
                            };
                        Error =>  {
                            [] Ź GetNextChar[];
                            tokenKind Ź tokenERROR;
                            state Ź Done;
                        };
                        ENDCASE =>  state Ź Error;
                    END
                REPEAT
                    GoodExit =>  {};
                    BadExit => tokenKind Ź tokenERROR
                ENDLOOP;
            SELECT tokenKind FROM
                tokenDECIMAL =>
                    tokenValue Ź NEW[INT Ź Convert.IntFromRope[token,10]];
                tokenHEX => {
                    tokenValue Ź NEW[INT Ź Convert.IntFromRope[token.Substr[2,token.Length[]-2],16]];
                    tokenKind Ź tokenDECIMAL
                    };
                tokenOCTAL => {
                    tokenValue Ź NEW[INT Ź Convert.IntFromRope[token,8]];
                    tokenKind Ź tokenDECIMAL
                    };
                tokenREAL =>
                    tokenValue Ź NEW[REAL Ź Convert.RealFromRope[token]];
                tokenCHAR =>
                    tokenValue Ź NEW[CHAR Ź Rope.Fetch[token,1]];
                tokenROPE =>
                    tokenValue Ź NEW[Rope.ROPE Ź token.Substr[1,token.Length[]-2]];
                ENDCASE => {};
            RETURN [tokenKind, token, 0, tokenValue]
            };

        EscapeChar: PROC [] RETURNS [Rope.ROPE] ~ {
            first,second,third:Rope.ROPE;
            number:Rope.ROPE;
            newChar:CHAR;
            newCharValid:BOOL Ź TRUE;
            c:CHAR Ź GetNextChar[];
            SELECT c FROM
                'n => newChar Ź Ascii.LF;
                't => newChar Ź Ascii.TAB;
                <<Vertical Tab must be finished>>
                'v => newChar Ź '^;
                'b => newChar Ź Ascii.BS;
                'r => newChar Ź Ascii.CR;
                'f => newChar Ź Ascii.FF;
                'a => newChar Ź Ascii.BEL;
                '\\ => newChar Ź '\\;
                '' => newChar Ź '';
                '" => newChar Ź '";
                '? => newChar Ź '?;
                'x => {
                    first Ź Rope.FromChar[GetNextChar[]];
                    IF IsHexDigit[PeekNextChar[]] THEN
                        second Ź Rope.FromChar[GetNextChar[]]
                    ELSE
                        second Ź NIL;
                   number Ź  Rope.Concat[first, second];
                    newChar Ź VAL[CARDINAL[Convert.CardFromRope[number,16]]]
                    };
                Ascii.LF => newCharValid Ź FALSE;
                ENDCASE =>  {
                    IF IsOctalDigit[c] THEN {
                        first Ź Rope.FromChar[c];
                        IF IsOctalDigit[PeekNextChar[]] THEN {
                            second Ź Rope.FromChar[GetNextChar[]];
                            IF IsOctalDigit[PeekNextChar[]] THEN
                                third Ź Rope.FromChar[GetNextChar[]]
                            ELSE
                                third Ź NIL
                            }
                        ELSE 
                            second Ź third Ź NIL;
                        number Ź Rope.Cat[first,second,third];
                        newChar Ź VAL[CARDINAL[Convert.CardFromRope[number,8]]]
                        }
                    ELSE
                        newCharValid Ź FALSE;
                    };
                IF newCharValid THEN
                    RETURN [Rope.FromChar[newChar]]
                ELSE
                    RETURN [NIL];
            };


        Test: Commander.CommandProc ~ {
            charsSkipped:INT;
            sourceStream:IO.STREAM;
            token:Rope.ROPE;
            tokenKind:IO.TokenKind Ź tokenID;
            commandLineStream:IO.STREAM Ź IO.RIS[cmd.commandLine];
            filename:PFSNames.PATH Ź PFS.PathFromRope[IO.GetTokenRope[commandLineStream,IO.IDProc].token];
            IO.Close[commandLineStream];
            sourceStream Ź PFS.StreamOpen[filename];
            GetCTokenInit[];
            WHILE tokenKind # tokenEOF DO
                [tokenKind,token,charsSkipped] Ź GetCTokenRope[sourceStream, TRUE ! 
                IO.EndOfStream => GOTO EOS];
                SELECT tokenKind FROM
                    tokenID => 
                        IO.PutF1[cmd.out,"Identifier = /%g/\n",IO.rope[token]];
                    tokenROPE => 
                        IO.PutF1[cmd.out,"String  constant= /%g/\n",IO.rope[token]];
                    tokenCHAR => 
                        IO.PutF1[cmd.out,"Character constant = /%g/\n",IO.rope[token]];
                    tokenSINGLE => 
                        IO.PutF1[cmd.out,"Single punctuation = /%g/\n",IO.rope[token]];
                    tokenDOUBLE => 
                        IO.PutF1[cmd.out,"Double punctuation = /%g/\n",IO.rope[token]];
                    <<tokenTRIPLE => >>
                        <<IO.PutF[cmd.out,"Triple punctuation = /%g/\n",IO.rope[token]];>>
                    tokenCOMMENT => 
                        IO.PutF1[cmd.out,"Comment = /%g/\n",IO.rope[token]];
                    tokenHEX => 
                        IO.PutF1[cmd.out,"Hexidecimal constant = /%g/\n",IO.rope[token]];
                    tokenDECIMAL =>
                        IO.PutF1[cmd.out,"Integer constant = /%g/\n",IO.rope[token]];
                    tokenREAL => 
                        IO.PutF1[cmd.out,"Real constant = /%g/\n",IO.rope[token]];
                    tokenOCTAL =>
                        IO.PutF1[cmd.out,"Octal constant = /%g/\n",IO.rope[token]];
                    tokenEOF => 
                        IO.PutRope[cmd.out,"End of file\n"];
                    tokenERROR =>
                        IO.PutRope[cmd.out,"Misformed token\n"];
                    ENDCASE => 
                        IO.PutF1[cmd.out,"Bad token kind %g\n",IO.int[ORD[tokenKind]]];
                ENDLOOP;
            EXITS
                EOS => IO.PutRope[cmd.out,"End of Stream\n"];
            };

        Commander.Register["Test",Test];
    }...