

DIRECTORY
Ascii USING [NUL],
Rope USING [ROPE];

Token: CEDAR DEFINITIONS =
BEGIN


Handle: TYPE = REF Object;
Object: TYPE = RECORD [
getChar: GetCharProcType, break: CHARACTER м Ascii.NUL];
GetCharProcType: TYPE = PROCEDURE [h: Handle] RETURNS [c: CHARACTER];

FilterState: TYPE = REF StandardFilterState;
StandardFilterState: TYPE = ARRAY [0..2) OF UNSPECIFIED;
FilterProcType: TYPE = PROCEDURE [c: CHARACTER, data: FilterState]
RETURNS [inClass: BOOLEAN];

QuoteProcType: TYPE = PROCEDURE [c: CHARACTER] RETURNS [closing: CHARACTER];

SkipMode: TYPE = {none, whiteSpace, nonToken};

nonQuote: CHARACTER = Ascii.NUL;


Boolean: PROCEDURE [h: Handle, signalOnError: BOOLEAN м TRUE]
RETURNS [true: BOOLEAN];
Number: PROCEDURE [h: Handle, radix: CARDINAL, signalOnError: BOOLEAN м TRUE]
RETURNS [u: UNSPECIFIED];
Decimal: PROCEDURE [h: Handle, signalOnError: BOOLEAN м TRUE]
RETURNS [i: INTEGER];
Octal: PROCEDURE [h: Handle, signalOnError: BOOLEAN м TRUE]
RETURNS [c: CARDINAL];
LongNumber: PROCEDURE [
h: Handle, radix: CARDINAL, signalOnError: BOOLEAN м TRUE]
RETURNS [u: LONG UNSPECIFIED];
LongDecimal: PROCEDURE [h: Handle, signalOnError: BOOLEAN м TRUE]
RETURNS [i: LONG INTEGER];
LongOctal: PROCEDURE [h: Handle, signalOnError: BOOLEAN м TRUE]
RETURNS [c: LONG CARDINAL];


Item: PROCEDURE [
h: Handle, temporary: BOOLEAN м TRUE] RETURNS [value: Rope.ROPE];


Skip: PROCEDURE [
h: Handle, data: FilterState, filter: FilterProcType,
skipInClass: BOOLEAN м TRUE];

Filtered: PROCEDURE [
h: Handle, data: FilterState, filter: FilterProcType,
skip: SkipMode м whiteSpace, temporary: BOOLEAN м TRUE]
RETURNS [value: Rope.ROPE];

MaybeQuoted: PROCEDURE [
h: Handle, data: FilterState, filter: FilterProcType м NonWhiteSpace, 
isQuote: QuoteProcType м Quote,
skip: SkipMode м whiteSpace, temporary: BOOLEAN м TRUE]
RETURNS [value: Rope.ROPE];


Alphabetic: FilterProcType;
AlphaNumeric: FilterProcType;
Delimited: FilterProcType;
FileName: FilterProcType;  -- '[, '], '<, '>, '*, '!, ';, '#, '-, '., '$, '+, AlphaNumeric
Line: FilterProcType;
NetFilter: FilterProcType;
NonWhiteSpace: FilterProcType;  
Numeric: FilterProcType;
Switches: FilterProcType;  -- '~, '-, AlphaNumeric
WhiteSpace: FilterProcType;  -- SP, TAB, CR

Brackets: QuoteProcType;  -- () [] {} <>
Quote: QuoteProcType;  -- '"


FreeStringHandle: PROCEDURE [h: Token.Handle] RETURNS [nil: Token.Handle];
StringToHandle: PROCEDURE [s: Rope.ROPE, offset: CARDINAL м 0]
RETURNS [h: Token.Handle];
FreeTokenString: PROCEDURE [s: Rope.ROPE] RETURNS [nil: Rope.ROPE м NIL];


NilData: SIGNAL;
SyntaxError: SIGNAL [s: Rope.ROPE];
UnterminatedQuote: SIGNAL;

END...

   z  
File: Token.mesa - last edited by:
Copyright ╙ 1992 by Xerox Corporation.  All rights reserved.
PXK    :	 8-Nov-82 18:00:44
Mark:	18-May-81 17:12:04
LXR    :	14-Sep-82 11:02:45
BXM  :	 7-Oct-81 13:56:12
AXD    :	 1-Jul-83 16:49:33
Philip James, March 13, 1991 3:34 pm PST
Copyright (C) Xerox Corporation 1982. All rights reserved.
Format USING [NetFormat],
System USING [HostNumber, NetworkAddress, NetworkNumber, SocketNumber],
Window USING [Box];
TYPEs
Client GetCharProc can conceal instance data around hн passed to h.getChar.
A filter is called only once for each input character.
FilterStateн[0..1] are initialized to 0 by routines using filters
Delimited "tokens" may be parsed via state maintained in the FilterState.
PROCEDUREs
All Rope.ROPEs RETURNed are from the system heap.
All routines update h.break before RETURNing.
Skips leading white space!
WindowBox: PROCEDURE [h: Handle] RETURNS [Window.Box];
NetFormat: TYPE = Format.NetFormat;

HostNumber: PROCEDURE [
h: Handle, format: NetFormat _ octal, signalOnError: BOOLEAN _ TRUE]
RETURNS [host: System.HostNumber];
NetworkAddress: PROCEDURE [
h: Handle, format: NetFormat _ octal, signalOnError: BOOLEAN _ TRUE]
RETURNS [address: System.NetworkAddress];
NetworkNumber: PROCEDURE [
h: Handle, format: NetFormat _ octal, signalOnError: BOOLEAN _ TRUE]
RETURNS [networkNumber: System.NetworkNumber];
SocketNumber: PROCEDURE [
h: Handle, format: NetFormat _ octal, signalOnError: BOOLEAN _ TRUE]
RETURNS [socketNumber: System.SocketNumber];
No default skip!
Standard filters
skips leading white space. first char IS the delimiter. 
must be called with Skip = nonToken
Misc. utility
SIGNALs
Э╩╪  ХNewlineDelimiter
Ц(cedarcode) styleЩ ЪЬ"Щ"JЪЬ
╧eЬ1Щ<JЪЬЩJЪЬЩJЪЬЩJЪЬЩJЪЬЩJЪЬ(Щ(ЧIcodeШ JЪЬ:Щ:KШ Ъ╧k	Ш	KЪЬЮЬЮЬШKЪЬЮЬШJЪЬЮЬЩJЪЬЮЬ;ЩGJЪЬЮЬЩЧKШ Ъ╧nЬЮЬШKЪЮШЧШ JЪЬЩЧШ JЪЬKЩKKЪЬЮЬЮЬШЪЬЮЬЮЬШKЪЬ!Ю	Ь	ЮЬШ8ЧKЪ	ЬЮЬЮ	ЬЮЬЮ	ЬШEЧШ JЪЬ6Щ6JЪЬAЩAJЪЬIЩIKЪЬЮЬЮЬШ,KЪ	ЬЮЬЮЬЮЬЮЬШ8ЪЬЮЬЮ	ЬЮ	ЬШBKЪЮЬЮЬШЧЧШ KЪЬЮЬЮ	ЬЮ	ЬЮЬЮ	ЬШLЧШ KЪЬ
ЮЬ Ш.ЧШ KЪЬ
Ю	Ь	ЮЬШ ЧШ JЪЬ
Щ
JЪЬ1Щ1JЪЬ-Щ-ЧШ JЪЬЩЧШ ЪЯЬЮ	ЬЮЬЮЬШ=KЪЮЬЮЬШЧЪ
ЯЬЮ	ЬЮЬЮЬЮЬШMKЪЮЬЮЬШЧЪЯЬЮ	ЬЮЬЮЬШ=KЪЮЬЮЬШЧЪЯЬЮ	ЬЮЬЮЬШ;KЪЮЬЮЬШЧЪЯ
ЬЮ	ЬШKЪЬЮЬЮЬЮЬШ:KЪЮЬЮЬЮЬШЧЪЯЬЮ	ЬЮЬЮЬШAKЪЮЬЮЬЮЬШЧЪЯ	ЬЮ	ЬЮЬЮЬШ?KЪЮЬЮЬЮЬШЧJЪЯ	ЬЮ	ЬЮЬЩ6ЧШ JЪЬЮЬЩ#JЩ ЪЯ
ЬЮ	ЬЩJЪЬ5ЮЬЮЬЩDJЪЮЬЩ"ЧЪЯЬЮ	ЬЩJЪЬ5ЮЬЮЬЩDJЪЮЬ"Щ)ЧЪЯЬЮ	ЬЩJЪЬ5ЮЬЮЬЩDJЪЮЬ'Щ.ЧЪЯЬЮ	ЬЩJЪЬ5ЮЬЮЬЩDJЪЮЬ%Щ,ЧЧШ ЪЯЬЮ	ЬШKЪ	ЬЮЬЮЬЮЬ	Ю	ЬШAЧЧШ JЪЬЩЧШ ЪЯЬЮ	ЬШKШ5KЪЬЮЬЮЬШЧЧШ ЪЯЬЮ	ЬШKШ5KЪЬ(ЮЬЮЬШ7KЪЮЬ	Ю	ЬШЧЧШ ЪЯЬЮ	ЬШKШFKШKЪЬ(ЮЬЮЬШ7KЪЮЬ	Ю	ЬШЧЧШ JЪЬЩЧШ KЪЯ
ЬШKЪЯЬШKЪЯ	ЬШJЪЬ8Щ8JЪЬ#Щ#KЪЯЬ╧c?ШZKЪЯЬШKЪЯ	ЬШKЪЯЬШ KЪЯЬШKЪЯЬаШ2KЪЯ
ЬаШ+ЧШ KЪЯЬаШ(KЪЯЬаШЧШ JЪЬЩЧШ KЪЯЬЮ	ЬЮЬШJЪЯЬЮ	ЬЮ	Ь
ЮЬШ>KЪЮЬШЧKЪЯЬЮ	ЬЮ	ЬЮЬЮ	ЬЮЬШIЧШ JЪЬЩЧШ KЪЯЬЮЬШKЪЯЬЮЬЮ	ЬШ#KЪЯЬЮЬШЧШ KЪЮЬШЧKШ Ч ЕЧ    
F  Ш  