
<<MakeCatalogImpl.Mesa>>
<<Copyright © 1985 by Xerox Corporation.  All rights reserved.>>
<<Last edited by Beach, January 24, 1985 9:35:53 am PST>>
<<>>
DIRECTORY
Ascii USING [Upper, Lower],
Basics USING [Comparison],
BasicTime USING [Now],
Commander USING [CommandProc, Register],
CommandTool USING [ArgumentVector, Parse],
Convert USING [RopeFromTime],
DefaultRemoteNames USING [Get],
DFUtilities USING [DirectoryItem, FileItem, ParseFromStream, ProcessItemProc],
EditSpan USING [Copy],
FS USING [ComponentPositions, EnumerateForNames, Error, ExpandName, NameProc, StreamOpen],
IO USING [BreakProc, Close, EndOf, EndOfStream, Error, GetChar, GetTokenRope, PutRope, RIS, SetIndex, SkipWhitespace, STREAM],
List USING [Nconc1],
NameSymbolTable USING [Name, MakeName],
NodeProps USING [PutProp, true],
OrderedSymbolTableRef USING [CompareProc, CreateTable, DuplicateKey, EnumerateIncreasing, Insert, Item, Size, Table],
ProcessExtras USING [CheckForAbort],
PutGet USING [FromFile],
Rope USING [Cat, Compare, Concat, Equal, Fetch, Find, FromChar, FromRefText, IsEmpty, Length, Match, ROPE, Substr],
TextNode USING [Level, StepForward, MakeNodeLoc, NarrowToTextNode, NodeItself, NodeRope, nullSpan, Root, Span, TextBody],
TiogaFileOps USING [AddLooks, CreateRoot, InsertAsLastChild, InsertNode, NodeBody, SetContents, SetFormat, Store],
UserCredentials USING [Get]
;

MakeCatalogImpl: CEDAR PROGRAM
IMPORTS Ascii, BasicTime, Commander, CommandTool, Convert, DefaultRemoteNames, DFUtilities, EditSpan, FS, IO, List, 
NameSymbolTable, NodeProps, OrderedSymbolTableRef, ProcessExtras, PutGet, Rope, TextNode, TiogaFileOps, UserCredentials
EXPORTS TiogaFileOps -- export NodeBody to convince Compiler we know that TiogaFileOps.Ref is really TextNode.Ref
~ BEGIN
ROPE: TYPE ~ Rope.ROPE;
Ref: TYPE ~ REF NodeBody;
NodeBody: PUBLIC TYPE ~ TextNode.TextBody;
MakeCatalogProc: Commander.CommandProc ~ {
systemHost: ROPE _ DefaultRemoteNames.Get[].systemHost;
serverName: ROPE _ Rope.Substr[systemHost, 1, systemHost.Length-2]; -- strip off the brackets
catalogName: ROPE;
catalogFileName: ROPE;
argv: CommandTool.ArgumentVector _ CommandTool.Parse[cmd];
quiet: BOOLEAN _ FALSE;
i: NAT _ 1;
WHILE i < argv.argc DO
IF Rope.Fetch[argv[i], 0] = '- THEN {
SELECT Ascii.Lower[Rope.Fetch[argv[i], 1]] FROM
's => {
i _ i+1;  -- consume following option as the server name
IF i >= argv.argc THEN
RETURN [$Failure, "-server option must be followed by a server name; please supply one."];
serverName _ argv[i];
};
'q => quiet _ TRUE;
ENDCASE => RETURN [$Failure, Rope.Concat[argv[i], " is an unrecognized option; use only -server or -quiet"]];
}
ELSE {
IF catalogName # NIL THEN 
RETURN [$Failure, "multiple catalog names supplied; only one allowed."];
catalogName _ argv[i];
};
i _ i+1;
ENDLOOP;
IF catalogName = NIL THEN RETURN;

catalogFileName _ MakeCatalog[serverName, catalogName, IF quiet THEN NIL ELSE cmd.out];

RETURN [NIL, Rope.Cat["\n", catalogFileName, " created."]];
};
MakeCatalog: PROC [serverName: ROPE, catalogName: ROPE, log: IO.STREAM _ NIL] RETURNS [catalogFileName: ROPE] ~ {
dfFilePattern: ROPE _ Rope.Cat["/", serverName, "/", catalogName, "/Top/*.DF!H"];
TryThisDFFile: FS.NameProc ~ {
<<PROC [fullFName: ROPE] RETURNS [continue: BOOLEAN];>>
base: ROPE;
ext: ROPE;
[base, ext] _ ParseFileName[fullFName];
IF NOT Rope.Equal[s1~ext, s2~"DF", case~FALSE] THEN RETURN;
ProcessExtras.CheckForAbort;
AddCatalogEntry[doc~catalogDoc, packageName~base, dfFileName~fullFName];
};
catalogDoc: CatalogDoc _ InitializeCatalogDoc[catalogName];
catalogDoc.log _ log;
FS.EnumerateForNames[pattern~dfFilePattern, proc~TryThisDFFile];
FinishCatalogDoc[catalogDoc];
RETURN [catalogDoc.fileName]
};
CatalogDoc: TYPE ~ REF CatalogDocRec;
CatalogDocRec: TYPE ~ RECORD[
log: IO.STREAM,  -- for recording our progress
fileName: ROPE,
root: Ref,
latestLevel1Node: Ref,
latestLevel2Node: Ref,
latestLevel3Node: Ref,
commandIndex: IndexTable,
keywordIndex: IndexTable];
InitializeCatalogDoc: PROC [catalogName: ROPE] RETURNS [doc: CatalogDoc] ~ {
UpperCase: PROC [r: ROPE] RETURNS [u: ROPE] ~ {
FOR i: INT IN [0..Rope.Length[r]) DO
u _ u.Concat[Rope.FromChar[Ascii.Upper[r.Fetch[i]]]];
ENDLOOP;
};

doc _ NEW[CatalogDocRec];
doc.commandIndex _ CreateIndexTable[];
doc.keywordIndex _ CreateIndexTable[];
doc.fileName _ Rope.Concat[catalogName, "Catalog.Tioga"];
doc.latestLevel1Node _ NIL;

doc.root _ TiogaFileOps.CreateRoot[];
NodeProps.PutProp[doc.root, $Postfix, Rope.FromRefText["(firstPageNumber) 0 .def (firstHeadersAfterPage) 0 .def"]];

AppendNode[level~1, doc~doc, r~doc.fileName, format~NIL];
NodeProps.PutProp[doc.latestLevel1Node, $Comment, NodeProps.true];

AppendNode[level~1, doc~doc, r~Rope.Cat["Last edited by ", UserCredentials.Get[].name, ", ", 
Convert.RopeFromTime[BasicTime.Now[]]], format~NIL];
NodeProps.PutProp[doc.latestLevel1Node, $Comment, NodeProps.true];

AppendNode[level~1, doc~doc, r~Rope.Concat[UpperCase[catalogName], " PACKAGE CATALOG"], format~"unleaded"];
TiogaFileOps.AddLooks[x~doc.latestLevel1Node, start~0, len~LAST[INT], look~'s, root~doc.root];
NodeProps.PutProp[doc.latestLevel1Node, $Mark, Rope.FromRefText["centerHeader"]];

AppendNode[level~1, doc~doc, r~"CEDAR 6.0  FOR INTERNAL XEROX USE ONLY", format~"unleaded"];
TiogaFileOps.AddLooks[x~doc.latestLevel1Node, start~0, len~LAST[INT], look~'s, root~doc.root];
NodeProps.PutProp[doc.latestLevel1Node, $Mark, Rope.FromRefText["centerFooter"]];

AppendNode[level~1, doc~doc, r~Rope.Concat["Package Catalog\nfor ", catalogName], format~"title"];

AppendNode[level~1, doc~doc, r~"c Copyright 1985 by Xerox Corporation.  All rights reserved.", format~"abstract"];
TiogaFileOps.AddLooks[x~doc.latestLevel1Node, start~0, len~1, look~'m, root~doc.root];
TiogaFileOps.AddLooks[x~doc.latestLevel1Node, start~0, len~LAST[INT], look~'s, root~doc.root];

AppendNode[level~1, doc~doc, r~"Abstract: This catalog is a list of interesting packages and tools.  The catalog is automatically 
created from the collection of maintainer-supplied entries.", format~"abstract"];
TiogaFileOps.AddLooks[x~doc.latestLevel1Node, start~0, len~8, look~'b, root~doc.root];

AppendNode[level~1, doc~doc, r~"XEROX\t\t\tXerox Corporation\n\t\t\t\tPalo Alto Research Center\n\t\t\t\t3333 Coyote Hill 
Road\n\t\t\t\tPalo Alto, California 94304\n\nFor Internal Xerox Use Only", format~"boilerplate"];
TiogaFileOps.AddLooks[x~doc.latestLevel1Node, start~0, len~5, look~'q, root~doc.root];
TiogaFileOps.AddLooks[x~doc.latestLevel1Node, start~5, len~LAST[INT], look~'o, root~doc.root];
TiogaFileOps.AddLooks[x~doc.latestLevel1Node, start~115, len~LAST[INT], look~'b, root~doc.root];
TiogaFileOps.AddLooks[x~doc.latestLevel1Node, start~115, len~LAST[INT], look~'x, root~doc.root];

AppendNode[level~1, doc~doc, r~"Catalog Components", format~"head"];
<<each component is added underneath this heading>>
};
FinishCatalogDoc: PROC [doc: CatalogDoc] ~ {
ProcessExtras.CheckForAbort;
IF NOT IsTableEmpty[doc.commandIndex] THEN AddIndex[doc, command];
IF NOT IsTableEmpty[doc.keywordIndex] THEN AddIndex[doc, keyword];
ProcessExtras.CheckForAbort;
[] _ TiogaFileOps.Store[filename~doc.fileName, x~doc.root];
};
AddCatalogEntry: PROC [doc: CatalogDoc, packageName: ROPE, dfFileName: ROPE] ~ {
problem: ROPE;
{
lastDirectoryPath: ROPE _ NIL;
packageDoc: ROPE _ Rope.Concat[packageName, "Doc.Tioga"];
documentationFileName: ROPE _ NIL;
documentationFiles: ROPE _ NIL;
listOfCommands: LIST OF REF ANY _ NIL;
TryThisDFItem: DFUtilities.ProcessItemProc ~ {
<<ProcessItemProc: TYPE = PROC [item: REF ANY] RETURNS [stop: BOOL _ FALSE];>>
WITH item SELECT FROM
directory: REF DFUtilities.DirectoryItem => {
lastDirectoryPath _ directory.path1;
};
file: REF DFUtilities.FileItem => {
base: ROPE;
ext: ROPE;
[base, ext] _ ParseFileName[file.name];
SELECT TRUE FROM
Rope.Equal["Load", ext, FALSE] => {
<<command name is the name of the .Load file>>
listOfCommands _ List.Nconc1[listOfCommands, base];
};
Rope.Equal["Tioga", ext, FALSE] AND Rope.Match["*Doc", base, FALSE] => {
IF NOT Rope.Match["*Documentation*", lastDirectoryPath, FALSE] THEN {
Log[doc.log, Rope.Cat["  ", file.name, " in the DF file is not in the proper Documentation directory; ignored\n"]];
RETURN;
};
documentationFiles _ IF documentationFiles.IsEmpty
THEN Rope.Cat[base, ".", ext]
ELSE Rope.Cat[documentationFiles, ", ", base, ".", ext];
IF Rope.Equal[Rope.Cat[base, ".", ext], packageDoc, FALSE] THEN
documentationFileName _ Rope.Concat[lastDirectoryPath, file.name];
RETURN;
};
ENDCASE => NULL;
};
ENDCASE => NULL;
};
dfStream: IO.STREAM _ FS.StreamOpen[fileName~dfFileName ! FS.Error => {problem _ error.explanation; GO TO FileProblem}];

ProcessExtras.CheckForAbort;

Log[doc.log, Rope.Concat[packageName, " ... "]];
DFUtilities.ParseFromStream[in~dfStream, proc~TryThisDFItem, filter~[comments~FALSE, filterA~all, filterB~public, filterC~all]];
IO.Close[dfStream];

ProcessExtras.CheckForAbort;

{ -- create the catalog entry
author: ROPE _ NIL;
abstractSpan: TextNode.Span _ TextNode.nullSpan;
keywords: ROPE;
singlePageDocument: BOOLEAN;
IF documentationFileName.IsEmpty THEN
Log[doc.log, "  no package documentation\n"]
ELSE 
[author, abstractSpan, keywords, singlePageDocument] _ ExtractStuffFromDocFile[doc, documentationFileName];
AppendNode[level~2, doc~doc, r~packageName, format~"head"];
AppendNode[level~3, doc~doc, r~Rope.Cat["DF file: ", packageName, ".df"], format~"indent"];
TiogaFileOps.AddLooks[x~doc.latestLevel3Node, start~0, len~8, look~'b, root~doc.root];
IF NOT author.IsEmpty THEN {
AppendNode[level~3, doc~doc, r~Rope.Cat["Author: ", author], format~"indent"];
TiogaFileOps.AddLooks[x~doc.latestLevel3Node, start~0, len~7, look~'b, root~doc.root];
};
IF NOT documentationFiles.IsEmpty AND
NOT (Rope.Equal[documentationFiles, packageDoc, FALSE] AND singlePageDocument) THEN {
<<the list of documentation files either has multiple files (since its not just PackageDoc.Tioga) or points to a long document 
(since its not a single page document)>>
AppendNode[level~3, doc~doc, r~Rope.Cat["Documentation: ", documentationFiles], format~"indent"];
TiogaFileOps.AddLooks[x~doc.latestLevel3Node, start~0, len~14, look~'b, root~doc.root];
};
IF NOT keywords.IsEmpty THEN {
CommaSeparated: IO.BreakProc ~ {
<<PROC [char: CHAR] RETURNS [CharClass];>>
RETURN [SELECT char FROM
', => sepr,
ENDCASE => other]
};
s: IO.STREAM _ IO.RIS[keywords];
firstBlank: INT _ Rope.Find[keywords, " "];
IO.SetIndex[self~s, index~firstBlank];  -- position past "Keywords:"
[] _ IO.SkipWhitespace[s];
WHILE NOT IO.EndOf[s] DO
ENABLE IO.Error, IO.EndOfStream => EXIT;
keyword: ROPE _ NIL;
keyword _ IO.GetTokenRope[stream~s, breakProc~CommaSeparated].token;
IF NOT keyword.IsEmpty THEN
InsertIndexTerm[doc.keywordIndex, keyword, packageName];
[] _ IO.GetChar[s];
[] _ IO.SkipWhitespace[s];
ENDLOOP;
IO.Close[s];
AppendNode[level~3, doc~doc, r~keywords, format~"indent"];
TiogaFileOps.AddLooks[x~doc.latestLevel3Node, start~0, len~10, look~'b, root~doc.root];
};
{
contents: ROPE _ NIL;
FOR l: LIST OF REF ANY _ listOfCommands, l.rest WHILE l # NIL DO
commandName: ROPE _ NARROW[l.first];
InsertIndexTerm[doc.commandIndex, commandName, packageName];
contents _ IF contents.IsEmpty
THEN Rope.Concat["Commands: ", commandName]
ELSE Rope.Cat[contents, ", ", commandName];
ENDLOOP;
IF NOT contents.IsEmpty THEN {
AppendNode[level~3, doc~doc, r~contents, format~"indent"];
TiogaFileOps.AddLooks[x~doc.latestLevel3Node, start~0, len~9, look~'b, root~doc.root];
};
};
IF abstractSpan # TextNode.nullSpan THEN {
[] _ EditSpan.Copy[dest~TextNode.MakeNodeLoc[doc.latestLevel3Node], source~abstractSpan, destRoot~doc.root, 
sourceRoot~TextNode.Root[abstractSpan.start.node]];
FOR node: Ref _ Forward[doc.latestLevel3Node], Forward[node] WHILE TextNode.Level[node] >= 3 DO
TiogaFileOps.SetFormat[node, "indent"];
ENDLOOP;
};
};
EXITS
FileProblem => {
Log[doc.log, Rope.Concat[packageName, " ... "]];
Log[doc.log, Rope.Cat["  ", problem, "\n"]];
};

};
};
ExtractStuffFromDocFile: PROC [doc: CatalogDoc, documentationFile: ROPE] RETURNS [author: ROPE, abstractSpan: TextNode.Span, 
keywords: ROPE, singlePageDocument: BOOLEAN _ FALSE] ~ {
RopeOf: PROC [n: Ref] RETURNS [ROPE] ~ {
RETURN [TextNode.NodeRope[n]];
};
problem: ROPE;
{
tiogaDocRoot: Ref _ TextNode.NarrowToTextNode[PutGet.FromFile[documentationFile ! FS.Error => {problem _ error.explanation; GOTO 
FileProblem}]];
node: Ref _ Forward[tiogaDocRoot];
authorFormat: NameSymbolTable.Name _ NameSymbolTable.MakeName["authors"];
abstractFormat: NameSymbolTable.Name _ NameSymbolTable.MakeName["abstract"];
boilerplateFormat: NameSymbolTable.Name _ NameSymbolTable.MakeName["boilerplate"];
WHILE node # NIL DO
SELECT TRUE FROM
authorFormat = node.typename => {
IF author.IsEmpty
THEN author _ RopeOf[node]
ELSE author _ Rope.Cat[author, "; ", RopeOf[node]];
};
abstractFormat = node.typename AND abstractSpan = TextNode.nullSpan => {
<<We only do this when abstractSpan is empty.>>
<<Documenation may contain several abstract format nodes>>
<<The following heuristics are used to determine the span of the abstract:>>
<<Begin the abstract if the node begins with "Abstract">>
<<Begin the abstract at the node following the copyright notice which is a node matching "Copyright *" or "* Copyright *">>
<<End the abstract if the node does not have abstract format>>
<<End the abstract if the node begins with "Keyword">>

shortContents: ROPE _ Rope.Substr[RopeOf[node], 0, 100];
IF Rope.Match["Abstract*", shortContents, FALSE] THEN {
abstractSpan.start _ abstractSpan.end _ [node, TextNode.NodeItself];
}
ELSE IF Rope.Match["Copyright*", shortContents, FALSE] OR
Rope.Match["* Copyright *", shortContents, FALSE] THEN {
<<look forward to see if next node has abstract format>>
node _ Forward[node];
IF abstractFormat = node.typename THEN 
abstractSpan.start _ abstractSpan.end _ [node, TextNode.NodeItself];
};
IF abstractSpan # TextNode.nullSpan THEN {
<<look forward to see how many abstract nodes there are in the abstract span>>
node _ Forward[node];
WHILE abstractFormat = node.typename AND NOT Rope.Match["Keyword*", Rope.Substr[RopeOf[node], 0, 100], FALSE] DO
abstractSpan.end _ [node, TextNode.NodeItself];
node _ Forward[node];
ENDLOOP;
};
<<We may have moved node forward to a non-abstract format node, check again>>
IF abstractFormat # node.typename THEN LOOP
ELSE IF Rope.Match["Keyword*", Rope.Substr[RopeOf[node], 0, 100], FALSE] THEN {
keywords _ RopeOf[node];
};
};
boilerplateFormat = node.typename => {
<<peek to see if there is any following nodes>>
singlePageDocument _ Forward[node] = NIL;
<<collect head nodes here for cumulative table of contents>>
EXIT;
};
ENDCASE;
node _ Forward[node];
ENDLOOP;
EXITS
FileProblem => {
Log[doc.log, Rope.Cat["  ", problem, "\n"]];
RETURN [NIL, TextNode.nullSpan, NIL, FALSE];
};
};
};
AddIndex: PROC [doc: CatalogDoc, kindOf: {command, keyword}] ~ {
indexTable: IndexTable _ IF kindOf = command THEN doc.commandIndex ELSE doc.keywordIndex;
lastTerm: ROPE;
PerIndexTerm: EnumerateProc ~ {
IF NOT Rope.Equal[term, lastTerm, FALSE]
THEN AppendNode[level~2, doc~doc, r~Rope.Cat[term, ": ", location], format~"item"]
ELSE doc.latestLevel2Node.rope _ Rope.Cat[doc.latestLevel2Node.rope, ", ", location];
lastTerm _ term;
RETURN [FALSE];
};
AppendNode[level~1, doc~doc, r~IF kindOf = command THEN "Command Index" ELSE "Keyword Index", format~"head"];
EnumerateIndexTerms[indexTable, PerIndexTerm];
};
ParseFileName: PROC [fileName: ROPE] RETURNS [base, ext: ROPE] ~ {
cp: FS.ComponentPositions;
fullName: ROPE;
[fullName, cp] _ FS.ExpandName[fileName];
base _ Rope.Substr[fullName, cp.base.start, cp.base.length];
ext _ Rope.Substr[fullName, cp.ext.start, cp.ext.length];
};
AppendNode: PROC [doc: CatalogDoc, level: NAT, r: ROPE, format: ROPE] ~ {
InnerAppendNode: PROC [subtree: Ref, latest: Ref, r: ROPE, format: ROPE] RETURNS [newNode: Ref] ~ {
IF subtree = NIL THEN ERROR;
newNode _ IF latest = NIL THEN
TiogaFileOps.InsertNode[x~subtree, child~TRUE]
ELSE 
TiogaFileOps.InsertAsLastChild[x~subtree, prevLast~latest];
TiogaFileOps.SetContents[x~newNode, txt~r];
TiogaFileOps.SetFormat[x~newNode, format~format];
};
SELECT level FROM
1 => {
doc.latestLevel1Node _ InnerAppendNode[doc.root, doc.latestLevel1Node, r, format];
doc.latestLevel2Node _ doc.latestLevel3Node _ NIL;
};
2 => {
doc.latestLevel2Node _ InnerAppendNode[doc.latestLevel1Node, doc.latestLevel2Node, r, format];
doc.latestLevel3Node _ NIL;
};
3 => {
doc.latestLevel3Node _ InnerAppendNode[doc.latestLevel2Node, doc.latestLevel3Node, r, format];
};
ENDCASE;
};
Forward: PROC [node: Ref] RETURNS [next: Ref] ~ {
RETURN [TextNode.NarrowToTextNode[TextNode.StepForward[node]]];
};
Log: PROC [log: IO.STREAM, msg: ROPE] ~ {
IF log # NIL THEN log.PutRope[msg];
};
IndexTable: TYPE ~ OrderedSymbolTableRef.Table;
CreateIndexTable: PROC [] RETURNS [IndexTable] ~ {
RETURN [OrderedSymbolTableRef.CreateTable[compareProc: CompareIndexKeys]];
};
IsTableEmpty: PROC [table: IndexTable] RETURNS [BOOLEAN] ~ {
RETURN [OrderedSymbolTableRef.Size[table] = 0];
};
Pair: TYPE ~ REF PairRec;
PairRec: TYPE ~ RECORD[
term: ROPE,
location: ROPE];
InsertIndexTerm: PROC [table: IndexTable, term: ROPE, location: ROPE] ~ {
OrderedSymbolTableRef.Insert[table, NEW[PairRec _ [term, location]] ! OrderedSymbolTableRef.DuplicateKey => CONTINUE];
};
CompareIndexKeys: OrderedSymbolTableRef.CompareProc ~ {
<<PROC [r1, r2: OrderedSymbolTableRef.Item] RETURNS [Basics.Comparison]>>
pair1: Pair _ NARROW[r1];
pair2: Pair _ NARROW[r2];
comparison: Basics.Comparison _ Rope.Compare[pair1.term, pair2.term, FALSE];
IF comparison = equal THEN comparison _ Rope.Compare[pair1.location, pair2.location, FALSE];
RETURN [comparison];
};
EnumerateProc: TYPE ~ PROC [term: ROPE, location: ROPE] RETURNS [BOOLEAN];
EnumerateIndexTerms: PROC [table: IndexTable, proc: EnumerateProc] ~ {
ApplyProc: PROC [item: OrderedSymbolTableRef.Item] RETURNS [BOOLEAN] ~ {
pair: Pair _ NARROW[item];
RETURN [proc[pair.term, pair.location]];
};
OrderedSymbolTableRef.EnumerateIncreasing[table, ApplyProc];
};
Commander.Register[
key~"MakeCatalog",
proc~MakeCatalogProc,
doc~"MakeCatalog [-quiet] [-server <ServerName>] <CatalogDirectoryName>     make a catalog of software packages stored as DF files 
in /<ServerName>/<CatalogDirectoryName>/Top/*.DF"];
END.