Copyright Ó 1981, 1982, 1983 , 1992by Xerox Corporation. All rights reserved.
File: TokensA.mesa - last edited by:
PXK :  8-Jan-82 15:38:28
SXW : 12-Jul-81 14:57:51
LXR : 23-Feb-83 10:25:34
JGS:  13-Jan-83 10:58:22
Mark: 18-May-81 17:16:13
AXD : 29-Sep-82 20:42:59
BJD : 11-Jan-83 10:33:40
Philip James, March 13, 1991 4:45 pm PST
Christian Jacobi, April 7, 1992 5:23 pm PDT
DIRECTORY
ARAccess USING [AppendChar],
Ascii USING [CR, LF, NUL, SP, TAB],
Convert USING [CardFromRope, Error],
Heap: TYPE USING [systemZone],
Inline USING [BITAND],
Rope USING [Fetch, Length, ROPE],
String USING [
AppendChar, CopyToNewString, InvalidNumber, Length,
StringBoundsFault, StringToLongNumber, StringToNumber],
StringLookUp: TYPE USING [InTable],
Token;
Window USING [Box, nullBox];
TokensA: CEDAR MONITOR IMPORTS ARAccess, Convert, --Heap, Inline, --Rope--, String, StringLookUp --EXPORTS Token =
BEGIN OPEN Ascii;
UnterminatedQuote: PUBLIC SIGNAL = CODE;
FilterState: TYPE = Token.FilterState;
StandardFilterState: TYPE = Token.StandardFilterState;
FilterProcType: TYPE = Token.FilterProcType;
GetCharProcType: TYPE = Token.GetCharProcType;
QuoteProcType: TYPE = Token.QuoteProcType;
SkipMode: TYPE = Token.SkipMode;
nonQuote: CHARACTER = Token.nonQuote;
initialLength: CARDINAL = 100; --initial length for Token strings
lengthIncrement: CARDINAL = 100; --amount of additional storage to allocate
on a bounds fault
savedString: Rope.ROPE ¬ NIL; -- keep one string of length initialLength
GetString: ENTRY PROC [length: CARDINAL] RETURNS [Rope.ROPE] = {
ENABLE UNWIND => NULL;
IF length = initialLength AND savedString # NIL THEN {
ls: Rope.ROPE ¬ savedString;
savedString ¬ NIL;
ls ¬ NIL;
ls.Length ← 0;
RETURN[ls]};
RETURN[NIL--Heap.systemZone.NEW[StringBody[length]]--]};
FreeString: ENTRY PROC [ls: Rope.ROPE] = {
ENABLE UNWIND => NULL;
IF ls # NIL --AND ls.maxlength = initialLength-- AND savedString = NIL THEN {
savedString ¬ ls; RETURN};
--Heap.systemZone.FREE[@ls]--};
Boolean: PUBLIC PROCEDURE [h: Token.Handle, signalOnError: BOOLEAN ← TRUE]
RETURNS [true: BOOLEAN ← FALSE] =
BEGIN
temp: Rope.ROPE ← NIL; -- ← [10];
BEGIN
InvalidChar: PROCEDURE [expected: CHARACTER] RETURNS [invalid: BOOLEAN] =
BEGIN
temp ← ARAccess.AppendChar[temp, h.break ← h.getChar[h]];
invalid ← Inline.BITAND[h.break, 337B] # expected;
END;
Skip[h, NIL, WhiteSpace, TRUE];
String.AppendChar[temp, h.break];
SELECT Inline.BITAND[h.break, 337B] FROM
'T =>
BEGIN
IF InvalidChar['R] OR InvalidChar['U] THEN GO TO invalid;
IF InvalidChar['E] THEN GO TO invalid; -- allow compiler to cross-jump
RETURN[TRUE];
END;
'F =>
BEGIN
IF InvalidChar['A] OR InvalidChar['L] OR InvalidChar['S] THEN
GO TO invalid;
IF InvalidChar['E] THEN GO TO invalid; -- allow compiler to cross-jump
RETURN[FALSE];
END;
ENDCASE => GO TO invalid;
EXITS invalid => IF signalOnError THEN SIGNAL SyntaxError[temp];
END;
END;
MAS: Can we get rid of this and just use Inline.LowHalf[LongNumber]
if HighHalf[] = 0 or -1? Two issues: speed and equivalence of results.
Number: PUBLIC PROC [
h: Token.Handle, radix: CARDINAL, signalOnError: BOOLEAN ¬ TRUE]
RETURNS [u: CARD] =
BEGIN
s: Rope.ROPE ¬ Filtered[h, NIL, NumberFilter];
IF s = NIL THEN {IF signalOnError THEN SIGNAL SyntaxError[s]; RETURN[0]};
u ¬ Convert.CardFromRope[
s, radix !
Convert.Error => {
IF signalOnError THEN SIGNAL SyntaxError[s]; u ¬ 0; CONTINUE};
UNWIND => FreeString[s]];
FreeString[s];
END;
Decimal: PUBLIC PROC [h: Token.Handle, signalOnError: BOOLEAN ¬ TRUE]
RETURNS [i: INTEGER] = {i ¬ Number[h, 10, signalOnError]};
Octal: PUBLIC PROC [h: Token.Handle, signalOnError: BOOLEAN ← TRUE]
RETURNS [c: CARDINAL] = {c ← Number[h, 8, signalOnError]};
LongNumber: PUBLIC PROC [
h: Token.Handle, radix: CARDINAL, signalOnError: BOOLEAN ← TRUE]
RETURNS [u: LONG UNSPECIFIED] =
BEGIN
s: Rope.ROPE ← Filtered[h, NIL, NumberFilter];
IF s = NIL THEN {IF signalOnError THEN SIGNAL SyntaxError[s]; RETURN[0]};
u ← String.StringToLongNumber[
s, radix !
String.InvalidNumber => {
IF signalOnError THEN SIGNAL SyntaxError[s]; u ← 0; CONTINUE};
UNWIND => FreeString[s]];
FreeString[s];
END;
LongDecimal: PUBLIC PROC [h: Token.Handle, signalOnError: BOOLEAN ← TRUE]
RETURNS [i: LONG INTEGER] = {i ← LongNumber[h, 10, signalOnError]};
LongOctal: PUBLIC PROC [h: Token.Handle, signalOnError: BOOLEAN ← TRUE]
RETURNS [c: LONG CARDINAL] = {c ← LongNumber[h, 8, signalOnError]};
Item: PUBLIC PROCEDURE [h: Token.Handle, temporary: BOOLEAN ¬ TRUE]
RETURNS [value: Rope.ROPE] =
BEGIN
value ¬ GetString[initialLength];
BEGIN
ENABLE {
UNWIND => FreeString[value];
String.StringBoundsFault =>
BEGIN
ns ← String.CopyToNewString[
s, Heap.systemZone, s.maxlength + lengthIncrement - s.length];
FreeString[s];
RESUME[value ← ns];
END
};
Skip[h, NIL, WhiteSpace, TRUE];
DO
IF WhiteSpaceInline[h.break] OR h.break = NUL THEN EXIT;
value ¬ ARAccess.AppendChar[value, h.break];
h.break ¬ h.getChar[h];
ENDLOOP;
IF value.Length = 0 THEN {FreeString[value]; RETURN[NIL]};
IF ~temporary THEN {
old: Rope.ROPE ¬ value;
value ¬ old;
value ← String.CopyToNewString[old, Heap.systemZone];
FreeString[old]};
END;
END;
WindowBox: PUBLIC PROCEDURE [h: Token.Handle]
RETURNS [box: Window.Box ← Window.nullBox] =
BEGIN
labels: ARRAY [0..4) OF Rope.ROPE ← ["X"L, "Y"L, "W"L, "H"L];
data: Token.StandardFilterState;
string: Rope.ROPE ← NIL;
MustBeBracketed: Token.FilterProcType = {RETURN[FALSE]};
string ← MaybeQuoted[h, @data, MustBeBracketed, Brackets];
IF String.Length[string] > 0 THEN {
sh: Token.Handle = StringToHandle[string];
DO
temp: Rope.ROPE ← Filtered[sh, @data, Alphabetic];
': was consumed and now is in sh.break
IF sh.break = Ascii.NUL THEN {FreeString[temp]; EXIT};
SELECT StringLookUp.InTable[temp, DESCRIPTOR[labels]] FROM
0 => box.place.x ← Decimal[sh, FALSE];
1 => box.place.y ← Decimal[sh, FALSE];
2 => box.dims.w ← Decimal[sh, FALSE];
3 => box.dims.h ← Decimal[sh, FALSE];
ENDCASE => {FreeString[temp]; EXIT};
FreeString[temp];
ENDLOOP;
[] ← FreeStringHandle[sh]};
FreeString[string];
END;
No default skip!
ZeroData: PROCEDURE [data: FilterState] = TRUSTED INLINE
BEGIN
pu: REF LONG UNSPECIFIED = LOOPHOLE[data];
IF pu # NIL THEN pu­ ¬ 0;
END;
Skip: PUBLIC PROCEDURE [
h: Token.Handle, data: FilterState, filter: FilterProcType,
skipInClass: BOOLEAN ¬ TRUE] =
BEGIN
ZeroData[data];
WHILE (h.break ¬ h.getChar[h]) # NUL AND
filter[h.break, data] = skipInClass DO ENDLOOP;
END;
Filtered: PUBLIC PROCEDURE [
h: Token.Handle, data: FilterState, filter: FilterProcType,
skip: SkipMode ¬ whiteSpace, temporary: BOOLEAN ¬ TRUE]
RETURNS [value: Rope.ROPE] =
BEGIN
value ¬ NIL;
ZeroData[data];
First handle (possible) skip
DO
IF (h.break ¬ h.getChar[h]) = NUL THEN RETURN;
IF skip = whiteSpace AND WhiteSpaceInline[h.break] THEN LOOP;
IF filter[h.break, data] THEN EXIT;
IF skip = none OR skip = whiteSpace THEN RETURN;
ENDLOOP;
Then accumulate all inClass characters
value ¬ GetString[initialLength];
DO
ENABLE {
UNWIND => FreeString[value];
String.StringBoundsFault =>
BEGIN
ns ← String.CopyToNewString[
s, Heap.systemZone, s.maxlength + lengthIncrement - s.length];
FreeString[s];
RESUME[value ← ns];
END
};
value ¬ ARAccess.AppendChar[value, h.break];
String.AppendChar[value, h.break];
IF (h.break ¬ h.getChar[h]) = NUL OR ~filter[h.break, data] THEN EXIT;
ENDLOOP;
IF ~temporary THEN {
old: Rope.ROPE ¬ value;
value ¬ old;
value ← String.CopyToNewString[old, Heap.systemZone];
FreeString[old]};
END;
closeQuote: CHAR ¬ 0c;
QuoteFilter: FilterProcType = {
IF c = NUL THEN {SIGNAL UnterminatedQuote; RETURN [FALSE]}
ELSE RETURN[c # closeQuote]};
MaybeQuoted: PUBLIC PROCEDURE [
h: Token.Handle, data: FilterState, filter: FilterProcType ¬ NonWhiteSpace,
isQuote: QuoteProcType ¬ Quote, skip: SkipMode ¬ whiteSpace,
temporary: BOOLEAN ¬ TRUE] RETURNS [value: Rope.ROPE] =
BEGIN
ApplyFilter: FilterProcType ¬ filter;
ZeroData[data];
value ¬ NIL;
First handle (possible) skip
DO
IF (h.break ¬ h.getChar[h]) = NUL THEN RETURN;
IF skip = whiteSpace AND WhiteSpaceInline[h.break] THEN LOOP;
IF (closeQuote ¬ isQuote[h.break]) # nonQuote THEN
IF (h.break ¬ h.getChar[h]) = closeQuote THEN {
h.break ¬ h.getChar[h];
IF h.break = closeQuote THEN EXIT; -- doubling close quote is literal quote character
RETURN}
ELSE {
IF h.break = NUL THEN {SIGNAL UnterminatedQuote; RETURN};
ApplyFilter ¬ QuoteFilter; EXIT};
IF filter[h.break, data] THEN EXIT;
IF skip = none OR skip = whiteSpace THEN RETURN;
ENDLOOP;
Then accumulate all inClass characters
value ¬ GetString[initialLength];
BEGIN
ENABLE {
UNWIND => FreeString[value];
String.StringBoundsFault =>
BEGIN
ns ← String.CopyToNewString[
s, Heap.systemZone, s.maxlength + lengthIncrement - s.length];
FreeString[s];
RESUME[value ← ns];
END
};
WHILE h.break # NUL DO
value ¬ ARAccess.AppendChar[value, h.break];
IF ~ApplyFilter[h.break ¬ h.getChar[h], data] THEN
BEGIN
IF ApplyFilter = QuoteFilter THEN {
h.break ¬ h.getChar[h]; -- get next character
IF h.break = closeQuote THEN LOOP}; -- of closeQuote, include character in token
EXIT;
END;
ENDLOOP;
IF ~temporary THEN {
old: Rope.ROPE ¬ value;
value ¬ old;
value ← String.CopyToNewString[old, Heap.systemZone];
FreeString[old]};
END;
END;
Standard filters
Alphabetic: PUBLIC FilterProcType =
BEGIN
RETURN[SELECT c FROM IN ['a..'z], IN ['A..'Z] => TRUE, ENDCASE => FALSE];
END;
AlphaNumeric: PUBLIC FilterProcType =
BEGIN
RETURN[
SELECT c FROM
IN ['a..'z], IN ['A..'Z], IN ['0..'9] => TRUE,
ENDCASE => FALSE];
END;
Delimited: PUBLIC FilterProcType =
first non-blank char IS the delimiter. Must be used with skip = nonToken
BEGIN
delimiter: REF CHARACTER = LOOPHOLE[data];
IF data = NIL THEN ERROR NilData;
IF data[0] # 0 THEN RETURN[c # delimiter­];
IF ~WhiteSpaceInline[c] AND c # NUL THEN delimiter­ ← c;
RETURN[FALSE];
END;
uninitialized: CARDINAL = 0;
inHost: CARDINAL = 1;
startName: CARDINAL = 2;
inName: CARDINAL = 3;
FileName: PUBLIC FilterProcType =
BEGIN
IF data = NIL THEN ERROR NilData;
SELECT c FROM
IN ['a..'z], IN ['A..'Z], IN ['0..'9], '>, '*, '!, ';, '#, '-,
'., '$, '+ =>
IF data[0] = uninitialized OR data[0] = startName THEN
data[0] ¬ inName;
'< =>
IF data[0] = uninitialized OR data[0] = startName THEN
data[0] ¬ inName
ELSE RETURN[FALSE];
'[ =>
IF data[0] = uninitialized THEN
data[0] ¬ inHost
ELSE RETURN[FALSE];
'] => IF data[0] = inHost THEN data[0] ¬ startName ELSE RETURN[FALSE];
ENDCASE => IF data[0] # inHost THEN RETURN[FALSE];
RETURN[TRUE];
END;
Line: PUBLIC FilterProcType =
BEGIN RETURN[SELECT c FROM CR, NUL => FALSE, ENDCASE => TRUE]; END;
Numeric: PUBLIC FilterProcType =
BEGIN RETURN[SELECT c FROM IN ['0..'9] => TRUE, ENDCASE => FALSE]; END;
Switches: PUBLIC FilterProcType = -- '~, '-, AlphaNumeric
BEGIN
RETURN[
SELECT c FROM
IN ['a..'z], IN ['A..'Z], IN ['0..'9], '~, '- => TRUE,
ENDCASE => FALSE];
END;
NumberFilter: FilterProcType =
BEGIN
RETURN[
SELECT c FROM
IN ['0..'9], '+, '-, 'B, 'b, 'D, 'd => TRUE,
ENDCASE => FALSE];
END;
NonWhiteSpace: PUBLIC FilterProcType =
BEGIN RETURN[c # NUL AND ~WhiteSpaceInline[c]]; END;
WhiteSpace: PUBLIC FilterProcType = BEGIN RETURN[WhiteSpaceInline[c]]; END;
WhiteSpaceInline: PROCEDURE [c: CHARACTER]
RETURNS [ --isWhiteSpace:-- BOOLEAN] = INLINE
BEGIN RETURN[SELECT c FROM SP, TAB, LF, CR => TRUE, ENDCASE => FALSE]; END;
Brackets: PUBLIC QuoteProcType = -- () [] {} <>
BEGIN
RETURN[
SELECT c FROM '( => '), '[ => '], '{ => '}, '< => '>, ENDCASE => nonQuote];
END;
Quote: PUBLIC QuoteProcType = -- '"
BEGIN RETURN[SELECT c FROM '', '" => c, ENDCASE => nonQuote]; END;
Type conversion
LSHandle: TYPE = REF LSObject;
LSObject: TYPE = MACHINE DEPENDENT RECORD [
object: Token.Object, s: Rope.ROPE, i: CARDINAL];
FreeStringHandle: PUBLIC PROCEDURE [h: Token.Handle]
RETURNS [nil: Token.Handle ¬ NIL] = {--Heap.systemZone.FREE[@h]--};
StringToHandle: PUBLIC PROCEDURE [s: Rope.ROPE, offset: CARDINAL ¬ 0]
RETURNS [h: Token.Handle] =
BEGIN
lsh: LSHandle ¬ NEW[LSObject ¬ [[getChar: StringGetChar, break: Ascii.NUL], s, offset]];
h ¬ NEW[Token.Object ¬ [StringGetChar, Ascii.NUL]];
END;
FreeTokenString: PUBLIC PROCEDURE [s: Rope.ROPE]
RETURNS [nil: Rope.ROPE ¬ NIL] = {FreeString[s]};
StringGetChar: Token.GetCharProcType = TRUSTED
BEGIN
lsh: LSHandle ¬ LOOPHOLE[h];
leng: CARD ¬ lsh.s.Length[];
IF lsh.i < leng THEN {
c ¬ lsh.s.Fetch[lsh.i]; lsh.i ¬ lsh.i + 1}
ELSE c ¬ Ascii.NUL;
END;
SIGNALs
NilData: PUBLIC SIGNAL = CODE;
SyntaxError: PUBLIC SIGNAL [s: Rope.ROPE] = CODE;
END...