-- DFParserImpl.Mesa
-- last edit December 29, 1982 3:24 pm
-- last edit May 22, 1983 1:16 pm, Russ Atkinson
-- Pilot 6.0/ Mesa 7.0

DIRECTORY
CWF: TYPE USING [WF0, WF1, WF2, WF3, WF4],
Date: TYPE USING [StringToPacked],
DFSubr: TYPE USING [AppendToUsingSeq, Criterion, DF, DFEntryProcType, DFFileRecord, DFSeq,
 FreeUsingSeq, InterestingNestedDFProcType, NextDF,
 StripLongName, UsingEmpty, UsingSeq, ZoneType],
IO: TYPE USING[UserAbort],
LongString: TYPE USING [AppendChar, AppendString, EquivalentString, InvalidNumber],
Stream: TYPE USING [EndOfStream, GetChar, Handle],
Subr: TYPE USING [AbortMyself, AllocateString, CopyString, EndsIn, errorflg,
 FreeString, LongZone, strcpy, SubStrCopy, TTYProcs],
Time: TYPE USING [Invalid];

DFParserImpl: PROGRAM
IMPORTS CWF, Date, DFSubr, IO, LongString, Stream, Subr, Time
EXPORTS DFSubr = {

-- MDS usage (encouraged so that we won't run out of Resident VM)
-- this is a constant array
tokenString: ARRAY Token OF STRING = [
 "bad"L, "CameFrom"L, "Directory"L, "Exports"L,
 "Host"L, "Imports"L, "Include"L, "Of"L, "Public"L,
 "PublicOnly"L, "ReadOnly"L, "ReleaseAs"L, "Using"L,
 -- special characters
 "@"L, "{"L, "}"L, "["L, "]"L, "Comment"L,
 "~"L, "+"L, "~="L, ">"L, "filename or date"L, "EOF"L, "CR"L];
-- end of MDS (there are also string literals)

Token: TYPE = {tBad,
 -- keywords
 tCameFrom, tDirectory, tExports, tHost, tImports,
 tInclude, tOf, tPublic, tPublicOnly, tReadOnly, tReleaseAs, tUsing,
 -- special characters
 tAtSign, tOpenCurly, tCloseCurly, tOpenBracket, tCloseBracket, tComment,
 tTilde, tPlus, tNotEqual, tGreaterThan, tOther, tEOF, tCR};
 -- tOther is a catch all that includes filenames
 -- such as <dir>schmidt.df; dates, etc.
 

-- this outer procedure is here to hold state for the inner
-- parsing procedures and avoid using any MDS
-- making the code reentrant

-- if noremoteerrors is true, don't complain if a file doesn't
--  appear to have a remote place
-- if forceReadonly then make every entry in this DF file be ReadOnly
-- if omitNonPublic, then don't parse them into dfseq
-- dffilename is for error messages
-- if using ~= NIL then restrict parsing to those on using list
-- sh may not be a FileStream!

-- dfseq does not need to be local!

-- justexception is here to kludge over bugs in with
-- the spacing on Host and Directory: it is not used with Imports and Includes

ParseStream: PUBLIC PROC[sh: Stream.Handle, dfseq: DFSubr.DFSeq, dffilename: LONG STRING,
 using: DFSubr.UsingSeq, noremoteerrors, forceReadonly, omitNonPublic: BOOL,
 h: Subr.TTYProcs,
 interestingNestedDF: DFSubr.InterestingNestedDFProcType, dfEntryProc: DFSubr.DFEntryProcType,
 ancestor: LONG STRING, nLevel: CARDINAL] = {
peekch: CHAR ← ' ;  -- peeck character
tok: Token ← tBad;  -- next token
wholecomment: LONG STRING ← NIL; -- memory in longzone
streamposn: LONG CARDINAL ← 0;
justexception: BOOL ← FALSE;
longzone: UNCOUNTED ZONE ← Subr.LongZone[];

-- these strings are allocated dynamically to prevent running out of resident VM
-- beware: they are in a longzone
tokenstr: LONG STRING ← NIL;  -- receives text of next token (tok)
commentline: LONG STRING ← NIL; -- used by GetC to accumulate comments, includes trailing CR
directory: LONG STRING ← NIL;  -- the current directory
releasedir: LONG STRING ← NIL;  -- the current release directory
fullname: LONG STRING ← NIL;  -- the whole name
shortname: LONG STRING ← NIL;  -- the short name
tempdir: LONG STRING ← NIL;  -- explicit directory on a filename


 LittleCopyString: PROC[newstr, oldstr: LONG STRING] RETURNS[LONG STRING] = {
 RETURN[IF dfseq.zoneType ~= huge
  OR oldstr = NIL
  OR NOT LongString.EquivalentString[newstr, oldstr] THEN
   Subr.CopyString[newstr, dfseq.dfzone]
  ELSE oldstr];
 };
 
 CopyHost: PROC[host: LONG STRING] RETURNS[LONG STRING] = {
 IF dfseq.zoneType = huge THEN {
  IF LongString.EquivalentString[host, dfseq.indigoHost] THEN RETURN[dfseq.indigoHost];
  IF LongString.EquivalentString[host, dfseq.ivyHost] THEN RETURN[dfseq.ivyHost];
  };
 RETURN[Subr.CopyString[host, dfseq.dfzone]];
 };
 
 -- does not get the following token
 -- parses [host]<directory> or <directory>
 -- called only if tok = tOther or tOpenBracket
 HostAndOrDir: PROC[host, directory: LONG STRING] = {
 IF tok = tOpenBracket THEN { -- [ host ]
  GetToken[TRUE];
  CkTok[tok, tOther];
  Subr.strcpy[host, tokenstr];
  GetToken[TRUE];
  CkTok[tok, tCloseBracket];
  GetToken[TRUE];
  };
 -- the next string is the directory, possibly with <>
 CkTok[tok, tOther];
 Subr.strcpy[directory, tokenstr];
 IF directory.length = 0 THEN {
  IF NOT noremoteerrors THEN {
   CWF.WF0["Error - Null directory entry.\n"];
   Subr.errorflg ← TRUE;
   };
  RETURN;
  };
 IF directory[directory.length - 1] = '> THEN
  directory.length ← directory.length -1;
 IF directory[0] = '< THEN
  Subr.SubStrCopy[directory, directory, 1];
 };
 
 -- does not get the following token
 ParseFullName: PROC[host, directory, filename: LONG STRING]
  RETURNS[vers: CARDINAL] = {
 host.length ← 0;
 IF tok = tOpenBracket THEN {
  GetToken[TRUE];
  CkTok[tok, tOther];
  Subr.strcpy[host, tokenstr];
  GetToken[TRUE];
  CkTok[tok, tCloseBracket];
  GetToken[TRUE];
  };
 -- tok should now be <directory>file!vers
 CkTok[tok, tOther];
 vers ← DFSubr.StripLongName[tokenstr, NIL, directory, filename, FALSE
  ! LongString.InvalidNumber => {
   posn: LONG CARDINAL ← IF streamposn = 0 THEN 0
    ELSE streamposn - 1;
   CWF.WF2["Error - invalid number at position %lu in file %s.\n",
    @posn, dffilename];
   SIGNAL Subr.AbortMyself;
   }
  ];
 };
 
 CkTok: PROC[token: Token, shouldbe: Token] = {
 IF token ~= shouldbe THEN {
  posn: LONG CARDINAL ← IF streamposn = 0 THEN 0 ELSE streamposn - 1;
  CWF.WF4["Error - expecting a %s, found a %s at position %lu in file %s.\n",
   tokenString[token], tokenString[shouldbe], @posn, dffilename];
  };
};
 
 Misplaced: PROC[token: Token] = {
 posn: LONG CARDINAL ← IF streamposn = 0 THEN 0 ELSE streamposn - 1;
 CWF.WF3["Error - was not expecting the token '%s' at position %lu in file %s.\n",
  tokenString[token], @posn, dffilename];
 };
 
 SkipCR: PROC = {
 IF tok = tCR THEN
  GetToken[TRUE];
 };
 
 -- gets the following token, first on the next line
 GetCR: PROC = {
 -- forces next GetToken to the beginning of the line
 -- (and not return a CR)
 DO
  IF tok = tCR OR tok = tEOF THEN EXIT;
  GetToken[FALSE];
  ENDLOOP;
 GetToken[TRUE];
 };

 ParseDateField: PROC RETURNS[criterion: DFSubr.Criterion,
  wantdate: LONG CARDINAL] = {
 sdate: LONG STRING ← NIL;
 savetok: Token;
 wantdate ← 0;
 criterion ← none;
 GetToken[FALSE];
 IF tok = tOf THEN
  GetToken[FALSE];
 IF tok = tCR THEN {
  GetToken[TRUE];
  RETURN;
  };
 savetok ← tBad;
 sdate ← Subr.AllocateString[100];
 {ENABLE UNWIND => {Subr.FreeString[sdate]};
 WHILE tok = tOther OR tok = tNotEqual OR tok = tGreaterThan DO
  LongString.AppendString[sdate, tokenstr];
  LongString.AppendChar[sdate, ' ];
  savetok ← tok;
  GetToken[FALSE];
  ENDLOOP;
 -- tok may be tBad or tCR or be one of the following
 SELECT savetok FROM
 tNotEqual => criterion ← notequal;
 tGreaterThan => criterion ← update;
 tOther => {
  short: STRING ← [32]; -- must be short for Date.StringToPacked, dammitt!
  IF sdate.length > 32 THEN sdate.length ← 32;
  LongString.AppendString[short, sdate];
  wantdate ← Date.StringToPacked[short
  ! Time.Invalid => {
   CWF.WF3["Error - '%s' is an invalid time at position %lu in %s.\n",
    sdate, @streamposn, dffilename];
   wantdate ← 0;
   CONTINUE;
   }];
  };
 tCR => NULL; -- no date given
 ENDCASE => CWF.WF3["Error - expecting a date at position %lu in %s, \nfound '%s' instead.\n",
  @streamposn, dffilename, tokenstr];
 }; -- of ENABLE UNWIND
 Subr.FreeString[sdate];
 };

 -- this procedure is called ONCE by the outer procedure
 ParseStreamInternal: PROC = {
 nExports: CARDINAL;
 readonly, public, camefrom: BOOL ← FALSE;
 df: DFSubr.DF;
 lastdir, lastreleasedir: LONG STRING ← NIL;
 isatsign, istopmark, ispublicOnly, isnoremoteversion, exportsImports: BOOL;
 host: LONG STRING ← Subr.AllocateString[40];  -- the current host
 releaseHost: LONG STRING ← Subr.AllocateString[40]; -- the current release host
 localhost: LONG STRING ← Subr.AllocateString[40];
 localdirectory: LONG STRING ← Subr.AllocateString[100];
 localrhost: LONG STRING ← Subr.AllocateString[40];
 localrdirectory: LONG STRING ← Subr.AllocateString[100];

 {ENABLE UNWIND => {
  Subr.FreeString[host]; Subr.FreeString[releaseHost];
  Subr.FreeString[localhost]; Subr.FreeString[localdirectory];
  Subr.FreeString[localrhost]; Subr.FreeString[localrdirectory];
  };
 nExports ← 0;
 [] ← GetC[]; -- to set up peekch
 GetToken[TRUE]; -- first token
 exportsImports ← FALSE;
 DO
  -- tok is from last GetToken[]; should be first on next line
  IF h.in.UserAbort[] THEN SIGNAL Subr.AbortMyself;
  justexception ← FALSE;
  SELECT tok FROM
  tEOF => EXIT;
  tHost => {
   justexception ← TRUE;
   GetToken[TRUE];
   CkTok[tok, tOther];
   Subr.strcpy[host, tokenstr];
   GetCR[]; -- forces new line
   };
  tDirectory, tPublic, tExports, tReadOnly => {
   justexception ← TRUE;
   exportsImports ← public ← readonly ← camefrom ← FALSE;
   DO
    SELECT tok FROM
    tImports => GOTO isImports;
    tReadOnly => readonly ← TRUE;
    tExports, tPublic => public ← TRUE;
    tOpenBracket => EXIT;
    tDirectory => NULL;
    ENDCASE => Misplaced[tok];
    GetToken[TRUE];
    IF tok = tOther OR tok = tEOF OR tok = tOpenBracket THEN
     EXIT;
    ENDLOOP;
   HostAndOrDir[host, directory];
   -- parse CameFrom or ReleaseAs
   GetToken[TRUE];
   releaseHost.length ← releasedir.length ← 0;
   IF tok = tCameFrom OR tok = tReleaseAs THEN {
    camefrom ← (tok = tCameFrom);
    GetToken[TRUE];
    HostAndOrDir[releaseHost, releasedir];
    GetCR[]; -- gets new line & next token
    };
   EXITS
   isImports => exportsImports ← TRUE; -- "Exports Imports", goto tImports processing
   };
  tImports, tInclude => {
   -- if exportsImports is true, then this is an Exports Imports
   innerUsing: DFSubr.UsingSeq ← NIL;
   vers: CARDINAL;
   savecomment: LONG STRING ← NIL; -- memory in longzone
   createtime: LONG CARDINAL;
   criterion: DFSubr.Criterion;
   savetok: Token ← tok;
   skipIt, callIt, publicOk: BOOL ← FALSE;
   -- (Exports) Imports [host]<path>file!vers Of <date> CameFrom [host]<path>
   -- Imports (Exports) [host]<path>file!vers Of <date> CameFrom [host]<path>
   -- Include [host]<path>file!vers Of <date> (CameFrom|ReleaseAs [host]<path>)
   -- Using [ file list ]
   localhost.length ← 0;
   localdirectory.length ← 0;
   localrhost.length ← 0;
   localrdirectory.length ← 0;
   df ← NIL;
   GetToken[TRUE];
   savecomment ← wholecomment;
   wholecomment ← NIL;
   IF savetok = tExports THEN {
    exportsImports ← TRUE;
    GetToken[TRUE]; -- Imports Exports
    };
   vers ← ParseFullName[localhost, localdirectory, shortname];
   skipIt ← FALSE;
   -- skip it if there is a using list and it does not consume an element
   -- or if the Imports is not Public
   -- don't skip it if it is Include
   IF using ~= NIL THEN
    skipIt ← NOT Consume[using, shortname]
   ELSE skipIt ← (omitNonPublic AND NOT exportsImports AND savetok = tImports);
   IF exportsImports THEN nExports ← nExports + 1;
   IF NOT skipIt THEN {
    df ← DFSubr.NextDF[dfseq];
    IF df = NIL THEN EXIT;
    df.host ← CopyHost[localhost];
    df.directory ← Subr.CopyString[localdirectory, dfseq.dfzone];
    df.shortname ← Subr.CopyString[shortname, dfseq.dfzone];
    IF savecomment ~= NIL THEN
     df.comment ← Subr.CopyString[savecomment, dfseq.dfzone];
    };
   [criterion, createtime] ← ParseDateField[];
   SkipCR[];
   camefrom ← FALSE;
   IF tok = tCameFrom OR tok = tReleaseAs THEN {
    camefrom ← (tok = tCameFrom);
    GetToken[TRUE];
    HostAndOrDir[localrhost, localrdirectory];
    GetToken[TRUE];
    };
   IF NOT skipIt THEN {
    IF savetok = tImports THEN
     df.readonly ← df.publicOnly ← df.atsign ← TRUE
    ELSE -- tInclude --
     df.atsign ← TRUE;
    df.version ← vers;
    df.public ← exportsImports; -- from Exports Imports
    df.criterion ← criterion;
    df.createtime ← createtime;
     df.releaseDirectory ← IF localrdirectory.length > 0 THEN
     Subr.CopyString[localrdirectory, dfseq.dfzone] ELSE NIL;
    df.releaseHost ← IF localrhost.length > 0 THEN CopyHost[localrhost] ELSE NIL;
    df.cameFrom ← camefrom;
    };
   IF tok = tUsing AND savetok = tImports THEN {
    GetToken[TRUE];
    CkTok[tok, tOpenBracket];
    DO
     GetToken[TRUE];
     IF tok = tCloseBracket OR tok = tEOF THEN EXIT;
     CkTok[tok, tOther];
     innerUsing ← DFSubr.AppendToUsingSeq[innerUsing, tokenstr,
       dfseq.dfzone];
     ENDLOOP;
    IF NOT skipIt THEN df.using ← innerUsing;
    GetCR[];
    };
   IF savecomment ~= NIL THEN
    Subr.FreeString[savecomment, longzone];
   savecomment ← NIL;
   -- first call about this entry
   IF NOT skipIt AND dfEntryProc ~= NIL THEN
    dfEntryProc[dfEntry: df];
   -- call the user about this DF file
   -- call if
   -- 1) the using list we are driven by is not exhausted and
   --  a) there is an inner using list so we must check the intersection
   --   b) if we are looking at an Imports w/o Using list then this entry is Public,
   --  c) if this is an Included DF file
   -- 2) or if the using list is exhausted, the DF file was on the using list;
   -- 3) or if there is no using list and
   --  a) the DF file is Public
   --   b) if PublicOnly is false
   --  c) if the df file is an Includes
   publicOk ← exportsImports OR NOT omitNonPublic;
   callIt ← interestingNestedDF ~= NIL;
   IF using ~= NIL THEN
    callIt ← callIt
    AND ((NOT DFSubr.UsingEmpty[using]
       AND (savetok = tInclude OR publicOk OR innerUsing ~= NIL))
     OR NOT skipIt)
   ELSE callIt ← callIt AND (publicOk OR savetok = tInclude);
   IF callIt THEN
    interestingNestedDF[host: localhost, directory: localdirectory, shortname: shortname,
     ancestor: ancestor, immediateParent: dffilename, nLevel: nLevel,
     version: vers, createtime: createtime,
     driverUsingSeq: using, innerUsingSeq: innerUsing, dfEntry: df,
     entryIsReadonly: (savetok = tImports) OR forceReadonly,
     publicOnly: (savetok = tImports) OR omitNonPublic,
     criterion: criterion];
   IF skipIt THEN DFSubr.FreeUsingSeq[innerUsing];
   exportsImports ← public ← FALSE; -- reset for next line
   };
  tOther, tAtSign, tPlus, tTilde => {
   criterion: DFSubr.Criterion;
   createtime: LONG CARDINAL;
   skipIt, callIt, publicOk, isNewOnly: BOOL;
   vers: CARDINAL;
   -- +, ~, @ { PublicOnly } <Directory>Filename!vers createdate
  
   df ← NIL;
   skipIt ← FALSE;
   -- if no using list and this is exports only, then skip it
   IF omitNonPublic AND NOT public AND using = NIL THEN
    skipIt ← TRUE;
   IF public THEN nExports ← nExports + 1;
   isatsign ← istopmark ← ispublicOnly ← isNewOnly ← FALSE;
   DO
    SELECT tok FROM
    tAtSign => isatsign ← TRUE;
    tOpenCurly => NULL;
    tPublicOnly => ispublicOnly ← TRUE;
    tCloseCurly => NULL;
    tPlus => istopmark ← TRUE;
    tTilde => isNewOnly ← TRUE;
    tOther, tEOF => EXIT;
    ENDCASE => Misplaced[tok];
    GetToken[TRUE];
    ENDLOOP;
   IF tok = tEOF THEN LOOP;
   CkTok[tok, tOther];
   isnoremoteversion ← FALSE;
   Subr.strcpy[fullname, tokenstr];
   IF tokenstr[0] ~= '< AND directory.length = 0 THEN {
    IF NOT noremoteerrors THEN {
     Subr.errorflg ← TRUE;
     CWF.WF1[
    "Error - directory not specified for '%s'.\n", tokenstr];
     };
    isnoremoteversion ← TRUE;
    };
   -- fullname looks like <schmidt>model>junk.mesa!3 or junk.mesa!3
   vers ← DFSubr.StripLongName[fullname, NIL, tempdir,
    shortname, FALSE
    ! LongString.InvalidNumber => {
     posn: LONG CARDINAL ← IF streamposn = 0 THEN 0
      ELSE streamposn - 1;
     CWF.WF2["Error - invalid number at position %lu in file %s.\n",
      @posn, dffilename];
      SIGNAL Subr.AbortMyself;
      }
     ];
   -- skip it if not on using list
   -- we will not skip DF files that are referenced in included DF files (omitNonPublic = FALSE),
   -- we will skip DF files that are referenced in imported DF files (omitNonPublic = TRUE)
   IF using ~= NIL THEN
    skipIt ← NOT Consume[using, shortname];
   -- skip it if there is a using list and it does not consume an element
   IF NOT skipIt THEN {
    df ← DFSubr.NextDF[dfseq];
    IF df = NIL THEN EXIT;
    -- get previous comment saved
    IF wholecomment ~= NIL THEN
     df.comment ← Subr.CopyString[wholecomment, dfseq.dfzone];
    };
   -- free previous comment at this point
   -- any later and the parsedatefield/skipCR will loose a line
   IF wholecomment ~= NIL THEN
    Subr.FreeString[wholecomment, longzone];
   wholecomment ← NIL;
   [criterion, createtime] ← ParseDateField[];
   SkipCR[]; -- if we are looking at a CR, get next tok
   IF NOT skipIt THEN {
    lastdir ← df.directory ← IF isnoremoteversion THEN NIL
     ELSE IF tempdir.length = 0 THEN LittleCopyString[directory, lastdir]
     ELSE LittleCopyString[tempdir, lastdir];
    lastreleasedir ← df.releaseDirectory ← IF releasedir.length > 0 THEN
     LittleCopyString[releasedir, lastreleasedir] ELSE NIL;
    df.releaseHost ← IF releaseHost.length > 0 THEN CopyHost[releaseHost] ELSE NIL;
    df.host ← CopyHost[host];
    df.shortname ← Subr.CopyString[shortname, dfseq.dfzone];
    df.version ← vers;
    df.atsign ← isatsign;
    df.topmark ← istopmark;
    df.newOnly ← isNewOnly;
    df.readonly ← forceReadonly OR readonly;
    df.public ← public;
    df.publicOnly ← ispublicOnly;
    df.cameFrom ← camefrom;
    df.createtime ← createtime;
    df.criterion ← criterion;
    };
   -- first call about this entry
   IF NOT skipIt AND dfEntryProc ~= NIL THEN
    dfEntryProc[dfEntry: df];

   -- call the user about this DF file
   publicOk ← public OR NOT omitNonPublic;
   callIt ← isatsign AND interestingNestedDF ~= NIL AND Subr.EndsIn[shortname, ".df"];
   IF using ~= NIL THEN
    callIt ← callIt
     AND ((NOT DFSubr.UsingEmpty[using] AND publicOk OR NOT readonly)
      OR NOT skipIt)
   ELSE callIt ← callIt AND (publicOk OR NOT readonly);
   IF callIt THEN
    interestingNestedDF[host: host, directory: directory, shortname: shortname,
     ancestor: ancestor, immediateParent: dffilename, nLevel: nLevel,
     version: vers, createtime: createtime,
     driverUsingSeq: using, innerUsingSeq: NIL, dfEntry: df,
     entryIsReadonly: readonly OR forceReadonly,
     publicOnly: ispublicOnly OR omitNonPublic,
     criterion: criterion];
   };
  ENDCASE => {
   Misplaced[tok];
   GetToken[TRUE];
   };
  ENDLOOP;
 IF wholecomment ~= NIL THEN {
  IF dfseq.trailingcomment = NIL THEN
   dfseq.trailingcomment ← Subr.CopyString[wholecomment, dfseq.dfzone];
  Subr.FreeString[wholecomment, longzone];
  };
 }; -- of ENABLE UNWIND
 Subr.FreeString[host]; Subr.FreeString[releaseHost];
 Subr.FreeString[localhost]; Subr.FreeString[localdirectory];
 Subr.FreeString[localrhost]; Subr.FreeString[localrdirectory];
 -- don't give warning if no exports, since it may be nested Includes
 -- IF omitNonPublic AND nExports = 0 THEN
  -- CWF.WF1["Warning - %s is analyzed Exports Only, but it has no Exports!\n", dffilename];
 }; -- end of ParseStreamInternal
 
 -- returns results in tok and tokenstr
 GetToken: PROC[ignoreLeadingCR: BOOL] = {
 ch: CHAR;
 tok ← tBad;
 tokenstr.length ← 0;
 WHILE peekch = ' OR peekch = '\t OR peekch = ',
 OR peekch = '/ OR peekch = '-   -- questionable?
 OR (ignoreLeadingCR AND peekch = '\n) DO 
  [] ← GetC[];
  ENDLOOP;
 IF peekch IN ['A .. 'Z] OR peekch IN ['a .. 'z]
 OR peekch IN ['0 .. '9] OR peekch = '< THEN {
  tok ← tOther;
  DO
   ch ← GetC[];
   IF tokenstr.length >= tokenstr.maxlength THEN {
    CWF.WF1["String '%s' is too long.\n", tokenstr];
    RETURN;
    };
   tokenstr[tokenstr.length] ← ch;
   tokenstr.length ← tokenstr.length + 1;
   -- not ">", "-" as they are part of file names
   IF peekch = 0C
   OR peekch = '[
   OR peekch = ']
   OR peekch = '
   OR peekch = '\t
   OR peekch = '\n
   OR peekch = ',
   OR peekch = '+
   OR peekch = '@
   OR peekch = '{
   OR peekch = '}
   OR peekch = '/
   OR peekch = '~
   THEN
    EXIT;
   ENDLOOP;
  --now see if reserved or identifier or other
  IF tokenstr[0] = '< OR tokenstr.length > 10 THEN RETURN;
  -- claim: this is as good as hashing!!
  SELECT tokenstr.length FROM
  2 =>
   IF LongString.EquivalentString[tokenstr, "of"] THEN
    tok ← tOf;
  4 =>
   IF LongString.EquivalentString[tokenstr, "host"] THEN
    tok ← tHost;
  5 =>
   IF LongString.EquivalentString[tokenstr, "using"] THEN
    tok ← tUsing;
  6 =>
   IF LongString.EquivalentString[tokenstr, "public"] THEN
    tok ← tPublic;
  7 => {
   IF LongString.EquivalentString[tokenstr, "exports"] THEN
    tok ← tExports;
   IF LongString.EquivalentString[tokenstr, "imports"] THEN
    tok ← tImports;
   IF LongString.EquivalentString[tokenstr, "include"] THEN
    tok ← tInclude;
   };
  8 => {
   IF LongString.EquivalentString[tokenstr, "includes"] THEN
    tok ← tInclude;
   IF LongString.EquivalentString[tokenstr, "camefrom"] THEN
    tok ← tCameFrom;
   IF LongString.EquivalentString[tokenstr, "readonly"] THEN
    tok ← tReadOnly;
   };
  9 => {
   IF LongString.EquivalentString[tokenstr, "directory"] THEN
    tok ← tDirectory;
   IF LongString.EquivalentString[tokenstr, "releaseas"] THEN
    tok ← tReleaseAs;
   };
  10 =>
   IF LongString.EquivalentString[tokenstr, "publiconly"] THEN
    tok ← tPublicOnly;
  -- don't change # 10 without looking at IF test above
  ENDCASE;
  RETURN;
  };
 ch ← GetC[];
 LongString.AppendChar[tokenstr, ch];
 -- not alphanumeric, look for magic characters
 SELECT ch FROM
 -- one char delimiters
 '\n => tok ← tCR;
 '+ => tok ← tPlus;
 '@ => tok ← tAtSign;
 '{ => tok ← tOpenCurly;
 '} => tok ← tCloseCurly;
 '[ => tok ← tOpenBracket;
 '] => tok ← tCloseBracket;
 '# => tok ← tNotEqual;
 '> => tok ← tGreaterThan;
 0C => tok ← tEOF;
 -- two chars
 '~ => {
  IF peekch = '= THEN {
   [] ← GetC[];
   tok ← tNotEqual;
   }
  ELSE tok ← tTilde;
  };
 ENDCASE => tok ← tOther;
 };

 ReadC: PROC = {
 peekch ← Stream.GetChar[sh
  ! Stream.EndOfStream => {
   peekch ← 0C;
   CONTINUE;
   }
  ];
 streamposn ← streamposn + 1;
 };
 
 GetC: PROC RETURNS[ch: CHAR] = {
 ch ← peekch;
 IF ch = 0C THEN RETURN; -- STP only signals EndOfStream once, then hangs
 ReadC[];
 DO
  -- strips leading blanks from lines
  IF ch = '\n AND peekch = ' THEN {
   WHILE peekch = ' DO
    ReadC[];
    ENDLOOP;
   -- note that ch remains = \n
   };
  IF (ch = '/ AND peekch = '/)
  OR (ch = '- AND peekch = '-)
  OR (ch = '\n AND peekch = '\n AND NOT justexception) THEN {
   -- handle comment
   len: CARDINAL;
   savecomment: LONG STRING;
   commentline.length ← 0;
   LongString.AppendChar[commentline, ch];
   IF peekch ~= '\n THEN {
    -- get rest of comment line
    DO
     LongString.AppendChar[commentline, peekch];
     ReadC[];
     IF peekch = '\n OR peekch = 0C THEN EXIT;
     ENDLOOP;
    LongString.AppendChar[commentline, peekch];
    };
   ch ← peekch;
   IF ch ~= 0C THEN ReadC[];
   len ← commentline.length;
   IF wholecomment ~= NIL THEN len ← len + wholecomment.length;
   savecomment ← wholecomment;
   wholecomment ← Subr.AllocateString[len, longzone];
   IF savecomment ~= NIL THEN {
    Subr.strcpy[wholecomment, savecomment];
    Subr.FreeString[savecomment, longzone];
    };
   LongString.AppendString[wholecomment, commentline];
   -- CWF.WF1["Comment !%s!\n", wholecomment];
   }
  ELSE EXIT;
  ENDLOOP;
 };

 Cleanup: PROC = {
 Subr.FreeString[tempdir, longzone];
 Subr.FreeString[shortname, longzone];
 Subr.FreeString[fullname, longzone];
 Subr.FreeString[releasedir, longzone];
 Subr.FreeString[directory, longzone];
 Subr.FreeString[tokenstr, longzone];
 Subr.FreeString[commentline, longzone];
 };

-- these strings are allocated here to avoid running out of resident VM
commentline ← Subr.AllocateString[600, longzone];
tokenstr ← Subr.AllocateString[100, longzone];
directory ← Subr.AllocateString[100, longzone];
releasedir ← Subr.AllocateString[100, longzone];
fullname ← Subr.AllocateString[100, longzone];
shortname ← Subr.AllocateString[100, longzone];
tempdir ← Subr.AllocateString[100, longzone];
ParseStreamInternal[
 ! UNWIND => Cleanup[]];
Cleanup[];
};

Consume: PROC[usingseq: DFSubr.UsingSeq, shortname: LONG STRING] RETURNS[consumed: BOOL] = {
FOR i: CARDINAL IN [0 .. usingseq.size) DO
 IF usingseq[i] = NIL THEN LOOP;
 IF LongString.EquivalentString[usingseq[i], shortname] THEN {
  Subr.FreeString[usingseq[i], usingseq.zone];
  usingseq[i] ← NIL;
  RETURN[TRUE];
  };
 ENDLOOP;
RETURN[FALSE];
};

}.