ExpressTokenImpl.mesa
Implementation of tokens for the Express package
Created Tuesday, July 17, 1984 9:50 pm PDT
Last edited by Eric Nickell, July 18, 1984 2:32:31 am PDT
DIRECTORY
ExpressToken,
IO USING [BreakProc, EndOfStream, GetTokenRope],
Rope USING [Equal, Length];
DefinedTokens:
TYPE ~
LIST
OF
RECORD [rope:
ROPE, kind:
ATOM];
rope is the name found in the input stream. NIL means that there is no text string that can be placed in the input stream that implicitly convert to said operation.
kind is the actual operator
proc, if it is not NIL, means that the code generated by ExpressProc will be a procedure call, and proc is the procedure to call
definedTokens: DefinedTokens ←
LIST[
["", $],
[NIL, $Id1],
[NIL, $Id2],
[NIL, $Id3],
[NIL, $Id4],
[NIL, $Id5],
[NIL, $Constant],
["+", $Add],
["-", $Subtract],
["*", $Multiply],
["/", $Divide],
["(", $LPar],
[")", $RPar],
["[", $LBracket],
["]", $RBracket],
[",", $Comma],
["IF", $If],
["THEN", $Then],
["ELSE", $Else],
["min", $min],
["max", $max],
["or", $or],
["and", $and],
["xor", $xor],
[">", $GT],
["<", $LT],
["=", $EQ],
[">=", $GE],
["<=", $LE],
["#", $NE],
["~", $not]
];
GetNextToken:
PUBLIC
PROC [ts: TokenStream, symbols: Symbols, cProcs: ClientProcList]
RETURNS [token: Token] ~ {
TokenBreak:
IO.BreakProc ~ {
RETURN[
SELECT char
FROM
IN ['\000 .. ' ] => sepr,
'>, '<, '=, '#, '~, '[, '], '(, '), '+, '-, '*, '/, '^, ', => break,
ENDCASE => other
];
};
Pos:
PROC [size:
INT]
RETURNS [
INT] ~ {
ts.position ← ts.position+size;
RETURN[size];
};
ids: INT ← 1; --Counts identifiers as they go by...
rope: ROPE;
size: INT;
First, see if there is anything in the buffer of the TokenStream
IF ts.buffer#
NIL
THEN {token ← ts.buffer.first; ts.buffer ←ts.buffer.rest; []←Pos[token.size]; RETURN};
Nothing in buffer...build one from IO stream
[rope, size] ←
IO.GetTokenRope[ts.stream, TokenBreak !
IO.EndOfStream => {
token ← ["<End of Stream>", $EndOfStream, Pos[size]];
GOTO ReturnEOS;
}];
size ← size+Rope.Length[rope];
Check against the client-supplied symbol table
FOR sym: Symbols ← symbols, sym.rest
UNTIL sym=
NIL
DO
IF Rope.Equal[sym.first, rope,
FALSE]
THEN
RETURN [[rope,
SELECT ids
FROM
1=>$Id1, 2=>$Id2, 3=>$Id3, 4=>$Id4, 5=>$Id5, ENDCASE=>ERROR, Pos[size]]];
ids ← ids+1;
ENDLOOP;
Check against the client-supplied proc names
FOR cProc: ClientProcList ← cProcs, cProc.rest
UNTIL cProc=
NIL
DO
IF Rope.Equal[cProc.first.userRope, rope, FALSE] THEN RETURN [[rope, $CProc, Pos[size]]];
ENDLOOP;
Check against the built-in symbol table
FOR builtIn: DefinedTokens ← definedTokens, builtIn.rest
UNTIL builtIn=
NIL
DO
IF Rope.Equal[builtIn.first.rope, rope, FALSE] THEN RETURN [[rope, builtIn.first.kind, Pos[size]]];
ENDLOOP;
The only other thing it could be is a number...assume so
RETURN[[rope, $Constant, Pos[size]]];
EXITS
ReturnEOS => RETURN;
};
PushToken:
PUBLIC
PROC [ts: TokenStream, token: Token] ~ {
ts.buffer ← CONS[token, ts.buffer];
ts.position ← ts.position - token.size;
};
RegisterToken:
PUBLIC
PROC [rope:
ROPE, kind:
ATOM] ~ {
Add a new token to the defined tokens list...
definedTokens ← CONS[ [rope, kind], definedTokens];
};
TokenStreamFromIOStream:
PUBLIC
PROC [s:
STREAM]
RETURNS [ts: TokenStream] ~ {
Turns a vanilla IO STREAM into a token stream. Wasn't that obvious already?
RETURN [NEW[TokenStreamRec ← [stream: s]]];
};