-- File: ScriptScanner.mesa - last edit by
-- Mitchell:  October 7, 1982 6:32 pm

DIRECTORY
IO USING [CreateInputStreamFromRope, GetInt, GetReal, NUL, SP, EndOfStream, GetChar, STREAM],
Inline USING [HighByte, LowByte],
Rope USING [Concat, FromChar, ROPE],
ScriptHash USING [Enter, Handle, Hash, nullHash, nullVal, Val],
ScriptParse USING [Error, Terminal, TerminalType];

ScriptScanner: PROGRAM
IMPORTS Inline, IO, Rope, ScriptHash, ScriptParse
EXPORTS ScriptParse = {

ROPE: TYPE = Rope.ROPE;

nullVal: ScriptHash.Val = ScriptHash.nullVal;

ScanState: TYPE = {start, version, token, stop};
ScanHandle: TYPE = REF ScanObject;
ScanObject: PUBLIC TYPE = RECORD [
index: LONG CARDINAL ← 0,
char: CHARACTER ← IO.NUL,
eof: BOOLEAN ← FALSE,
state: ScanState ← start,
endScript, links, true, false: ScriptHash.Hash,
token: ROPE,
tokenStream: IO.STREAM,  -- always a stream for ScanObject.token
univ, id: ScriptHash.Handle,
z: UNCOUNTED ZONE,
stream: IO.STREAM];

StreamForToken: PROC[s: ScanHandle] RETURNS [IO.STREAM] = {
RETURN[
(s.tokenStream ← IO.CreateInputStreamFromRope[s.token, s.tokenStream])];
};

NextChar: PROCEDURE [s: ScanHandle] = {
IF s.eof OR s.state = stop THEN s.char ← IO.NUL
ELSE {
DO
s.index ← s.index + 1;
 s.char ← s.stream.GetChar[ !
IO.EndOfStream => {
  s.index ← s.index - 1;
  s.eof ← TRUE;
  s.char ← IO.NUL;
  EXIT}];
IF s.char IN [40C..176C] THEN EXIT;
  ENDLOOP}};

TokenizeALine: PROCEDURE [scan: ScanHandle] = {
ReadAndScanALine[scan];
FOR index IN TokenVecX DO
tokenVec[index] ← ScanToken[scan];
IF tokenVec[index].type=null THEN RETURN;
ENDLOOP;
};

EDIT POINT

NextSymbol: PUBLIC PROC[scan: ScanHandle] RETURNS [t: ScriptParse.Terminal] = {
IF NOT anothertoken THEN TokenizeALine[scan];
t𡤌urrenttoken; updatetokenindex;
};

NextSymbol: PUBLIC PROCEDURE [scan: ScanHandle]
RETURNS [t: ScriptParse.Terminal] = {
IntSequence: PROCEDURE = {
number: CARDINAL;
set, char: CHARACTER;
DO
NextChar[scan];
 SELECT scan.char FROM
  '=, '> => EXIT;
  IN ['0..'9] => {
  number ← (scan.char - '0);
  DO
  NextChar[scan];
  SELECT scan.char FROM
  IN ['0..'9] => number ← (number * 10) + (scan.char - '0);
  IO.SP => EXIT;
  ENDCASE => ERROR ScriptParse.Error[scan, scan.index];
  ENDLOOP};
  IN ['A..'P] => {
  left, right: CHARACTER;
  left ← scan.char;
  NextChar[scan];
  right ← scan.char;
  IF right NOT IN ['A..'P] THEN ERROR ScriptParse.Error[scan, scan.index];
  number ← (left - 'A)*16 + (right - 'A)};
  ENDCASE => ERROR ScriptParse.Error[scan, scan.index];
 IF (char ← Inline.LowByte[number]) = 377C OR
  (set ← Inline.HighByte[number]) = 377C THEN
  ERROR ScriptParse.Error[scan, scan.index];
 IF set # curSet THEN {
  onlySet0 ← simpleChars ← FALSE;
  curSet ← set;
  {scan.char ← 377C; AppendToToken[scan]};
  {scan.char ← set; AppendToToken[scan]}};
 scan.char ← char;
 SELECT scan.char FROM
  '=, '> => simpleChars ← FALSE;
  IN [40C..176C] => NULL;
  ENDCASE => simpleChars ← FALSE;
 AppendToToken[scan];
 ENDLOOP};
Number: PROCEDURE [neg: BOOLEAN] = {
haveDot, haveE, okMinus: BOOLEAN ← FALSE;
scan.token.length ← 0;
IF neg THEN String.AppendChar[scan.token, '-];
DO
SELECT scan.char FROM
IN ['0..'9] => {okMinus ← FALSE; AppendToToken[scan]};
'. => {
  IF haveDot THEN EXIT;
  haveDot ← TRUE;
  okMinus ← FALSE;
  AppendToToken[scan]};
'E => {
  IF NOT haveDot OR haveE THEN EXIT;
  haveE ← TRUE;
  okMinus ← TRUE;
  AppendToToken[scan]};
'- => {IF NOT okMinus THEN EXIT; okMinus ← FALSE; AppendToToken[scan]};
  ENDCASE => EXIT;
 NextChar[scan];
 ENDLOOP;
SELECT TRUE FROM
haveDot => t.body ← real[IO.GetReal[StreamForToken[scan]]];
 NOT haveDot => t.body ← integer[IO.GetInt[StreamForToken[scan]]];
 ENDCASE => ERROR};
onlySet0, simpleChars: BOOLEAN;
curSet: CHARACTER;

SELECT scan.state FROM
start => {scan.state ← version; RETURN[[body: start[]]]};
version => {NextChar[scan]; scan.state ← token; RETURN[VersionID[scan]]};
stop => RETURN[[pos: scan.index, body: stop[]]];
ENDCASE => NULL; -- fall through and do normal stuff
DO
BEGIN
t.pos ← scan.index;
SELECT scan.char FROM
IO.NUL => {scan.state ← stop; RETURN[[pos: scan.index, body: stop[]]]};

IO.SP, ', => GOTO tryAgain;

'$ => {t.body ← dollar[]; GOTO almostDone};
'' => {t.body ← quote[]; GOTO almostDone};
'← => {t.body ← leftArrow[]; GOTO almostDone};
'% => {t.body ← percent[]; GOTO almostDone};
'| => {t.body ← bar[]; GOTO almostDone};
'* => {t.body ← times[]; GOTO almostDone};
'/ => {t.body ← divide[]; GOTO almostDone};
'+ => {t.body ← plus[]; GOTO almostDone};
'^ => {t.body ← upArrow[]; GOTO almostDone};
'[ => {t.body ← leftBracket[]; GOTO almostDone};
'] => {t.body ← rightBracket[]; GOTO almostDone};
'{ => {t.body ← leftBrace[]; GOTO almostDone};
'} => {t.body ← rightBrace[]; GOTO almostDone};
'( => {t.body ← leftParen[]; GOTO almostDone};
') => {t.body ← rightParen[]; GOTO almostDone};
'. => {t.body ← dot[]; GOTO almostDone};
'? => {t.body ← question[]; GOTO almostDone};

': => { -- colonEqual
  NextChar[scan];
  IF scan.char # '= THEN t.body ← colon[]
  ELSE t.body ← colonEqual[];
  GOTO almostDone};

'- => { -- minus or comment or negative integer
  NextChar[scan];
  SELECT scan.char FROM
  '- => { -- eat the comment
  DO
  NextChar[scan];
  IF scan.char = '- THEN {
   NextChar[scan]; IF scan.char = '- THEN EXIT};
  ENDLOOP;
  GOTO tryAgain};
  IN ['0..'9], '. => {Number[TRUE]; GOTO done};
  ENDCASE => {t.body ← minus[]; GOTO done}};

 '< => { -- string
  temp: ROPE;
  onlySet0 ← simpleChars ← TRUE;
  curSet ← 0C;
  scan.token.length ← 0;
  DO
  NextChar[scan];
  SELECT scan.char FROM
  '> => EXIT;
  '= => {
  IntSequence[];
  SELECT scan.char FROM
   '> => EXIT;
   '= => NULL;
   ENDCASE => ERROR ScriptParse.Error[scan, scan.index]};
  ENDCASE => {
  IF curSet # 0C THEN {
   save: CHARACTER ← scan.char;
   {scan.char ← 377C; AppendToToken[scan]};
   {scan.char ← 0C; AppendToToken[scan]};
   scan.char ← save};
  IF scan.char NOT IN [40C..176C] THEN simpleChars ← FALSE;
  AppendToToken[scan]};
  ENDLOOP;
  temp ← scan.z.NEW[StringBody[scan.token.length]];
  String.AppendString[temp, scan.token];
  t.body ← string[
  [allSet0: onlySet0, allSimple: simpleChars, string: temp]];
  GOTO almostDone};

 'a, 'b, 'c, 'd, 'e, 'f, 'g, 'h, 'i, 'j, 'k, 'l, 'm,
'n, 'o, 'p, 'q, 'r, 's, 't, 'u, 'v, 'w, 'x, 'u, 'z,
 'A, 'B, 'C, 'D, 'E, 'F, 'G, 'H, 'I, 'J, 'K, 'L, 'M,
 'N, 'O, 'P, 'Q, 'R, 'S, 'T, 'U, 'V, 'W, 'X, 'Y, 'Z => { -- id or universal
  allCaps: BOOLEAN ← scan.char IN ['A..'Z];
  scan.token.length ← 0;
  AppendToToken[scan];
  DO
  NextChar[scan];
  SELECT scan.char FROM
  IN ['a..'z] => {AppendToToken[scan]; allCaps ← FALSE};
  IN ['0..'9], IN ['A..'Z] => AppendToToken[scan];
  ENDCASE => EXIT;
  ENDLOOP;
  IF allCaps THEN {
  hash: ScriptHash.Hash = scan.univ.Enter[scan.token, nullVal];
  SELECT hash FROM
  scan.endScript => t.body ← stop[];
  scan.true => t.body ← boolean[TRUE];
  scan.false => t.body ← boolean[FALSE];
  scan.links => t.body ← links[];
  ENDCASE => t.body ← universal[hash]}
  ELSE t.body ← id[scan.id.Enter[scan.token, nullVal]];
  GOTO done};

 '0, '1, '2, '3, '4, '5, '6, '7, '8, '9 => { -- number
  Number[FALSE];
  GO TO done};

  ENDCASE => ERROR ScriptParse.Error[scan, scan.index];

EXITS
almostDone => {NextChar[scan]; EXIT};
done => EXIT;
 tryAgain => {NextChar[scan]};
END
ENDLOOP};

VersionID: PROCEDURE [scan: ScanHandle] RETURNS [t: ScriptParse.Terminal] = {
match: ROPE = "Interscript/Reference/83 "L;
FOR i: CARDINAL IN [0..match.length) DO
IF scan.char # match[i] THEN ERROR ScriptParse.Error[scan, i];
NextChar[scan];
ENDLOOP;
scan.index ← match.length;
RETURN[[body: versionId[]]]};

InitScan: PUBLIC PROCEDURE [
stream: IO.STREAM, univ, id: ScriptHash.Handle, z: UNCOUNTED ZONE]
RETURNS [scan: ScanHandle] = {
scan ← z.NEW[ScanObject ← [
z: z, univ: univ, id: id, stream: stream,
endScript: univ.Enter["ENDSCRIPT"L, nullVal],
links: univ.Enter["LINKS"L, nullVal],
true: univ.Enter["T"L, nullVal],
false: univ.Enter["F"L, nullVal],
token: z.NEW[StringBody[40]] ]]};

}. -- of ScriptScanImpl