file MTScannerImpl.mesa
derived from Compiler>Scanner.mesa
last modified by Satterthwaite, June 6, 1983 4:06 pm
last edit by Schmidt, May 3, 1982 4:27 pm
Last Edited by: Nichols, July 13, 1983 6:13 pm
Last Edited by: HGM, January 31, 1985 9:12:02 pm PST
John Larson, July 29, 1986 7:43:53 pm PDT
DIRECTORY
IO: TYPE USING [GetIndex, STREAM],
Lex822: TYPE USING [LexToken, TokenType],
MTMiscOps: TYPE USING [Lookup, Table],
MTP1: TYPE --P1-- USING [Token, TValue, TerminalNode, nullTValue, Index, ScannerProc],
MTParseTable: TYPE ParseTable USING [ampmTok, atomTok, atSignTok, colonTok, commaTok, dateDummy, dayTok, dLitTok, dotTok, EOLTok, errorTok, lBracketTok, monthTok, nameDummy, qStringTok, rBracketTok, semiTok, TSymbol, zoneTok],
Rope: TYPE USING [Equal, ROPE, Fetch, Length];
MTScannerImpl:
CEDAR
PROGRAM
IMPORTS IO, Lex822, MTMiscOps, Rope
EXPORTS MTP1 ~ {
OPEN P1~~MTP1, MTParseTable;
ROPE: TYPE = Rope.ROPE;
Tables of rope constants for the date tokens.
ampmTable: MTMiscOps.Table ~ LIST["am", "pm"];
dayTable: MTMiscOps.Table ~ LIST["sun", "sunday", "mon", "monday", "tue", "tuesday", "wed", "wednesday", "thu", "thursday", "fri", "friday", "sat", "saturday"];
monthTable: MTMiscOps.Table ~ LIST["jan", "january", "feb", "february", "mar", "march", "apr", "april", "may", "jun", "june", "jul", "july", "aug", "august", "sep", "sept", "september", "oct", "october", "nov", "november", "dec", "december"];
zoneTable: MTMiscOps.Table ~
LIST[
"ut", "gmt", "est", "edt", "cst", "cdt", "mst", "mdt", "pst", "pdt", "bst"];
the scanner proper
stream: IO.STREAM ← NIL;
nTokens: CARDINAL; -- token count
nErrors: CARDINAL; -- lexical errors
eolSeen: BOOLEAN ← FALSE; -- stop the parse at the end of a header
firstTime: BOOLEAN; -- to force dummy token
ScanName:
PUBLIC P1.ScannerProc ~ {
tokenText, whiteSpace: ROPE;
tokType: Lex822.TokenType;
IF firstTime
THEN {
firstTime ← FALSE;
RETURN [[index~stream.GetIndex[], value~P1.nullTValue, class~nameDummy]];
};
IF eolSeen
THEN
RETURN [[index~stream.GetIndex[], value~P1.nullTValue, class~EOLTok]];
token.index ← stream.GetIndex[];
[tokenText, whiteSpace, tokType] ← Lex822.LexToken[stream];
token.value ← NEW[P1.TerminalNode ← [whiteSpace~whiteSpace, text~tokenText]];
token.class ←
SELECT tokType
FROM
atomTok => MTParseTable.atomTok,
domainLiteralTok => MTParseTable.dLitTok,
EOLTok => MTParseTable.EOLTok,
quotedStringTok => MTParseTable.qStringTok,
specialTok =>
SELECT tokenText.Fetch[0]
FROM
'. => MTParseTable.dotTok,
', => MTParseTable.commaTok,
': => MTParseTable.colonTok,
'; => MTParseTable.semiTok,
'@ => MTParseTable.atSignTok,
'< => MTParseTable.lBracketTok,
'> => MTParseTable.rBracketTok,
ENDCASE => MTParseTable.errorTok,
ENDCASE => MTParseTable.errorTok;
IF token.class = EOLTok
THEN
eolSeen ← TRUE
ELSE
IF token.class = MTParseTable.errorTok
THEN {
IF NOT eolSeen THEN nErrors ← nErrors+1
};
};
ScanDate:
PUBLIC P1.ScannerProc ~ {
tokenText, whiteSpace: ROPE;
tokType: Lex822.TokenType;
IF firstTime
THEN {
firstTime ← FALSE;
RETURN [[index~stream.GetIndex[], value~P1.nullTValue, class~dateDummy]];
};
IF eolSeen
THEN
RETURN [[index~stream.GetIndex[], value~P1.nullTValue, class~EOLTok]];
token.index ← stream.GetIndex[];
[tokenText, whiteSpace, tokType] ← Lex822.LexToken[stream];
token.value ← NEW[P1.TerminalNode ← [whiteSpace~whiteSpace, text~tokenText]];
token.class ←
SELECT tokType
FROM
atomTok =>
SELECT
TRUE
FROM
MTMiscOps.Lookup[tokenText, monthTable] => MTParseTable.monthTok,
MTMiscOps.Lookup[tokenText, dayTable] => MTParseTable.dayTok,
MTMiscOps.Lookup[tokenText, ampmTable] => MTParseTable.ampmTok,
MTMiscOps.Lookup[tokenText, zoneTable] => MTParseTable.zoneTok,
NumericZone[tokenText] => MTParseTable.zoneTok,
MilitaryZone[tokenText] => MTParseTable.zoneTok,
ENDCASE => MTParseTable.atomTok,
domainLiteralTok =>
MTParseTable.dLitTok,
EOLTok =>
MTParseTable.EOLTok,
quotedStringTok =>
MTParseTable.qStringTok,
specialTok =>
SELECT tokenText.Fetch[0]
FROM
'. => MTParseTable.dotTok,
', => MTParseTable.commaTok,
': => MTParseTable.colonTok,
'; => MTParseTable.semiTok,
'@ => MTParseTable.atSignTok,
'< => MTParseTable.lBracketTok,
'> => MTParseTable.rBracketTok,
ENDCASE => MTParseTable.errorTok,
ENDCASE => MTParseTable.errorTok;
IF token.class = EOLTok
THEN
eolSeen ← TRUE
ELSE
IF token.class = MTParseTable.errorTok
THEN {
IF NOT eolSeen THEN nErrors ← nErrors+1
};
};
MilitaryZone:
PROC [arg:
ROPE]
RETURNS[
BOOL] = {
-- A..Z except J
char: CHAR;
IF arg.Length[] # 1 THEN RETURN[FALSE];
char ← arg.Fetch[0];
IF char = 'J OR char = 'j THEN RETURN[FALSE];
IF ~(char IN['a..'z]) AND ~(char IN['A..'Z]) THEN RETURN[FALSE];
RETURN[TRUE]; };
NumericZone:
PROC [arg:
ROPE]
RETURNS[
BOOL] = {
-- +HHMM or -HHMM
char: CHAR;
IF arg.Length[] # 5 THEN RETURN[FALSE];
char ← arg.Fetch[0];
IF char # '+ AND char # '- THEN RETURN[FALSE];
FOR i:
INT
IN [1..4]
DO
char ← arg.Fetch[i];
IF ~(char IN['0..'9]) THEN RETURN[FALSE];
ENDLOOP;
RETURN[TRUE]; };
initialization/finalization
ScanInit:
PUBLIC
PROC[source:
IO.
STREAM] = {
stream ← source;
nTokens ← nErrors ← 0;
eolSeen ← FALSE;
firstTime ← TRUE;
};
ScanReset:
PUBLIC
PROC
RETURNS[
CARDINAL,
CARDINAL] = {
RETURN [nTokens, nErrors]};
}.