<> <> DIRECTORY Bindings, ScriptScan; ScriptTokenizer: CEDAR PROGRAM IMPORTS Bindings EXPORTS ScriptScan = BEGIN c: REF TEXT; InitScan: PUBLIC PROC [stream: IO.STREAM, bindTbl: Bindings.BTHandle, z: ZONE] = BEGIN c _ RefText.New[256]; END; classMap: ARRAY CHAR OF CharClass _ [ NUL .. ControlZ: blank 'A .. 'Z: ucAlpha 'a .. 'z: lcAlpha '0 .. '9: digit '<,'>, '!, ... : separator ]; CharClass: TYPE ~ {lcAlpha, ucAlpha, digit, separator, blank}; { PARA$ justified_T font.size_ 3.7E2 { font | face _ } } sbuuuusblllllllllsubllllsllllsbdsdudbsbllllbsbllllbsbsulllllllllsbsbsllllllllllsss GetToken: PUBLIC PROC RETURNS [t: Token] = BEGIN <> <> <> <<>> '< => {UNTIL nextchar[stream]='> DO NULL ENDLOOP; END; END. <> <>