DIRECTORY
Ascii USING [Letter],
Basics,
Commander USING [CommandProc, Register, Handle],
CommandTool USING [ArgumentVector, Parse, ParseToList],
Convert,
List,
FS,
IO,
Real,
RefText,
Rope,
KeyNote,
KeyNotePseudoServer,
SymTab;


KeyNoteCommandToolImpl: CEDAR PROGRAM
IMPORTS Ascii, Commander, CommandTool, Convert, FS, List, IO, Real, RefText, Rope, SymTab, KeyNote, KeyNotePseudoServer = {

ROPE: TYPE = Rope.ROPE;

KeyBD: Commander.CommandProc = {
argv: CommandTool.ArgumentVector _ CommandTool.Parse[cmd];
databaseName: ROPE  _ argv[1];
fileNamesToMatch: ROPE _ argv[2];
stopListFileName: ROPE _ argv[3];
wvp: KeyNote.WordVerifierProc = {
EachCharInWord: Rope.ActionType = {
IF Ascii.Letter[ch: c] THEN 
quit _ FALSE
ELSE {
quit _ TRUE;
};
};
IF SymTab.Fetch[x:NARROW[clientData], key: word].found THEN RETURN[FALSE];
RETURN[~Rope.Map[base: word,  action: EachCharInWord]];
};
stream: IO.STREAM _ FS.StreamOpen[fileName: stopListFileName];
block: REF TEXT _ RefText.New[FS.FileInfo[stopListFileName].bytes];
handle: KeyNote.Handle;
[] _ IO.GetBlock[self: stream, block: block];
handle _ KeyNote.OpenDatabase[databaseName: databaseName,  fileNamesToMatch: NEW[KeyNote.FileNamesToMatchObject _ [pattern: fileNamesToMatch]],  wordVerifierProc: wvp, tokenRelevanceThreshhold: 2, clientDataForVerifierProc: RopeToSymTab[RefText.TrustTextAsRope[text: block]] ! KeyNote.Error => { msg _ Rope.Cat[ "Problems during database building", "    ",Convert.RopeFromAtom[ec], ":", explanation]; GO TO Die}  
 ];
KeyNote.CloseDatabase[handle];
IO.Close[self: stream];
EXITS	
Die => result _ $Failure;
Usage => RETURN[$Failure, "Usage: KeyBD TokenDatabaseName pattern stopListFileName"];
};
KeyWM: Commander.CommandProc = {
databaseName: ROPE;
handle: KeyNote.Handle;
resultList: KeyNote.ResultList;
displayOverAllWeight: BOOLEAN _ TRUE;
displayTokensInEachFile: BOOLEAN _ TRUE;
displayThreshhold: REAL _ 0.10;
argv: CommandTool.ArgumentVector _ CommandTool.Parse[cmd: cmd];
EliminateIrrelevantFiles: PROC [relevantAndIrrelevantFiles:  KeyNote.ResultList, threshHold: REAL] RETURNS [ relevantFiles: KeyNote.ResultList] = {
RemoveIrrelevantItem: PROC [ item: REF ANY, list: List.LORA] = {
IF Real.CompareREAL[NARROW[item, REF KeyNote.ResultObject].overAllWeight, threshHold] # less THEN 
relevantFiles _ List.Nconc1[list: relevantFiles, ref: item];
};
List.Map[relevantAndIrrelevantFiles, RemoveIrrelevantItem];
};
ProcessSwitches: PROC [arg: ROPE, argNext: ROPE] = {
FOR index: INT IN [0..Rope.Length[arg]) DO
SELECT Rope.Fetch[arg, index] FROM
'w, 'W => {displayOverAllWeight _ FALSE };
't, 'T => {displayTokensInEachFile _ FALSE };
'q, 'Q=> {
displayThreshhold _ Convert.RealFromRope[r:argNext];
};
ENDCASE;
ENDLOOP;
};
FOR i: NAT IN [1..argv.argc) DO
arg: ROPE = argv[i];
argNext: ROPE = IF (argv.argc-1)=i THEN NIL ELSE argv[i+1];
SELECT Rope.Fetch[arg, 0] FROM
'- => ProcessSwitches[arg, argNext! Convert.Error => { msg _ "illegal threshold"; GO TO Die} ];
'/, '[ => {
databaseName  _ NARROW[argv[i]];
FOR j: NAT IN [i+1..argv.argc) DO 
resultList _ CONS[argv[j], resultList];
ENDLOOP;
EXIT;
};
ENDCASE =>NULL;
ENDLOOP;
IF Real.CompareREAL[displayThreshhold,1.0] = greater OR
Real.CompareREAL[displayThreshhold,0.0] =  less
THEN {
msg _ "threshold must be between 0 and 1"; GO TO Die };
handle _ KeyNote.OpenDatabase[databaseName: databaseName ! KeyNote.Error => { msg _ "Problems With KeyNote Database"; GO TO Die} ];
resultList _ KeyNote.FindDocumentsFromWords[db: handle, ropeList: resultList ];
KeyNote.CloseDatabase[handle];
IF resultList#NIL THEN {
resultList _ EliminateIrrelevantFiles[resultList, displayThreshhold * NARROW[resultList.first, REF KeyNote.ResultObject].overAllWeight];
OutputNames[cmd, resultList, displayOverAllWeight, displayTokensInEachFile];
};
EXITS	
Die => result _ $Failure;
Usage => RETURN[$Failure, "Usage: KeyWM {switch} TokenDatabaseName "];
};
KeyStop: Commander.CommandProc = {
argv: CommandTool.ArgumentVector _ CommandTool.Parse[cmd];
stopListCutOff: INT _ Convert.IntFromRope[argv[1]];
stopListFileName: ROPE _ argv[3];
fileNamesToMatch: ROPE _ argv[2];
wordList: KeyNote.ResultList _ KeyNote.BuildStopList[fileNamesToMatch: NEW[KeyNote.FileNamesToMatchObject _ [pattern: fileNamesToMatch]],  stopListCutOff: stopListCutOff];
stream: IO.STREAM _ FS.StreamOpen[fileName: stopListFileName, accessOptions: create ];
TRUSTED {
IO.PutBlock[self: stream, block: LOOPHOLE[Rope.InlineFlatten[ListToRope[wordList]], REF TEXT]];
IO.Close[self: stream]; };
EXITS	
Die => result _ $Failure;
Usage => RETURN[$Failure, "Usage: KeyStop stopListCutOff pattern stopListFileName"];
};
keyNoteRegistry: ROPE _ "keyNoteRegistry.txt";
KeyRegister: Commander.CommandProc = TRUSTED {
dataBaseInfo: ROPE _ Rope.Concat[ListToRope[LOOPHOLE[CommandTool.ParseToList[cmd].list, List.LORA]],"\n"];
stream: IO.STREAM _ FS.StreamOpen[fileName: keyNoteRegistry,  accessOptions: append ];
IO.PutBlock[self: stream, block: LOOPHOLE[Rope.InlineFlatten[dataBaseInfo], REF TEXT]];
IO.Flush[self: stream];
IO.Close[self: stream];
EXITS	
Die => result _ $Failure;
Usage => RETURN[$Failure, "KeyRegister keyNoteDatabase {Description of keyNoteDatabase}"];
};
KeyList: Commander.CommandProc = {
block: REF TEXT _ RefText.New[FS.FileInfo[keyNoteRegistry].bytes];
stream: IO.STREAM _ FS.StreamOpen[fileName: keyNoteRegistry,  accessOptions: read];
[] _ IO.GetBlock[self: stream, block: block];
IO.PutBlock[self: cmd.out, block: block];
IO.Close[self: stream]; 
};
KeyUnregister: Commander.CommandProc = {
argv: CommandTool.ArgumentVector _ CommandTool.Parse[cmd];
databaseToBeUnregistered: ROPE _ argv[1];
block: REF TEXT _ RefText.New[FS.FileInfo[keyNoteRegistry].bytes];
stream: IO.STREAM _ FS.StreamOpen[fileName: keyNoteRegistry,  accessOptions: write];
fileNames: ROPE;
startPos: INT; 
[] _ IO.GetBlock[self: stream, block: block];
fileNames _ RefText.TrustTextAsRope[block];
IF (startPos _ Rope.Find[fileNames, databaseToBeUnregistered]) # -1 THEN {
endPos: INT  _ Rope.Find[fileNames, "\n", startPos];
fileNames _ Rope.Concat[
Rope.Substr[base: fileNames, start: 0, len: startPos],
Rope.Substr[base: fileNames, start: endPos+2 ]
]
}
ELSE  {
msg _ "Database Not Registered";
GO TO Die};
IO.SetIndex[stream, 0];
TRUSTED { 
IO.PutBlock[self: stream, block: LOOPHOLE[Rope.InlineFlatten[fileNames], REF TEXT ]];
};
IO.SetLength[stream, Rope.InlineSize[fileNames]];
IO.Flush[self: stream];
IO.Close[self: stream]; 
EXITS	
Die => result _ $Failure;
Usage => RETURN[$Failure, "Usage: KeyUnregister TokenDatabaseName"];
};



ListToRope: PRIVATE PROC [listOfRope: List.LORA, spaceBetweenTokens: BOOLEAN _ TRUE] RETURNS [bigRope: ROPE] = {
ConvertListToRope: PROC [item: REF ANY, list: List.LORA] = {
bigRope _ IF spaceBetweenTokens THEN 
Rope.Cat[bigRope, " ",NARROW[item]]
ELSE
Rope.Concat[bigRope, NARROW[item]];
};
List.Map[list: listOfRope, proc: ConvertListToRope];
};

RopeToSymTab: PRIVATE PROC [bigRope: ROPE] RETURNS [table: SymTab.Ref] = {
positionAfterLastSpace: INT _ 0;
length: INT _ Rope.Length[base: bigRope];
stream: IO.STREAM _ IO.RIS[rope: bigRope];
table _  SymTab.Create[case: FALSE];
DO
token: ROPE _ IO.GetTokenRope[stream: stream!IO.EndOfStream => EXIT].token;
[] _ SymTab.Store[x: table, key: token, val: NIL];
ENDLOOP;
};

OutputNames: PRIVATE PROC [cmd: Commander.Handle, results: List.LORA, displayOverAllWeight: BOOLEAN _ TRUE,  displayTokensInEachFile: BOOLEAN _ TRUE]  = {
weightPrecision: Convert.RealPrecision = 4; 
ExtractAppropriateDataFromItem: PROC [file : REF ANY, fileList:List.LORA] = {
basicFileInfo: REF KeyNote.ResultObject _ NARROW[file];
ExtractAppropriateDataForEachTokenFromItem:  PROC [token : REF ANY, tokenList:List.LORA] = {
tokenPerFileInfo:  REF KeyNote.TokenInfoObject _ NARROW[token];
fileInfo _ Rope.Concat[ fileInfo, Rope.Cat[tokenPerFileInfo.token, " ", Convert.RopeFromInt[from: tokenPerFileInfo.frequency]," ", Convert.RopeFromReal[from: tokenPerFileInfo.weight, precision: weightPrecision]]];
fileInfo _  Rope.Cat[fileInfo, " "];
};
fileInfo _  Rope.Cat[fileInfo, KeyNotePseudoServer.SubstituteLongestPrefixWithPseudoServer[basicFileInfo.fileName], " "];
IF displayOverAllWeight THEN fileInfo _ Rope.Cat[fileInfo, Convert.RopeFromReal[from: basicFileInfo.overAllWeight, precision: weightPrecision], " "];
IF displayTokensInEachFile THEN List.Map[list: basicFileInfo.tokenInfoList, proc: ExtractAppropriateDataForEachTokenFromItem];
fileInfo _ Rope.Concat[fileInfo, "\n"];
};
fileInfo: ROPE;
List.Map[list: results, proc: ExtractAppropriateDataFromItem];
IO.PutRope[cmd.out, fileInfo];
};




Commander.Register[
key: "KeyBD",
proc: KeyBD,
doc: "KeyBD TokenDatabaseName pattern StopListName"];
Commander.Register[
key: "KeyWM",
proc: KeyWM,
doc: "KeyWM {switch} TokenDatabaseName  (token)+
switch = -t: suppress token display, -w: suppress aggregate weight display, -q threshold: display only files with weight greater than the best match * threshold  "];
Commander.Register[
key: "KeyStop",
proc: KeyStop,
doc: "KeyStop stopListCutOff pattern stopListFileName"];
Commander.Register[
key: "KeyRegister",
proc: KeyRegister,
doc: "KeyRegister keyNoteDatabase {Description of keyNoteDatabase}"];
Commander.Register[
key: "KeyUnregister",
proc: KeyUnregister,
doc: "KeyUnregister keyNoteDatabase"];
Commander.Register[
key: "KeyList",
proc: KeyList,
doc: "KeyList"];










}.


ΦFile: KeyNoteCommandToolImpl.mesa
Copyright Σ 1985, 1987 by Xerox Corporation.  All rights reserved.
Jack Kent February 24, 1988 10:59:34 am PST

Contents: Implementation of KeyNote
now get a handle on things
recognize server by [mumble or /mumble
now get a handle on things
chop off appropriate part of result List
register, list and unregister keyNote databases
maybe make sure that comment has no carriage returns
look for carriage return
private procedures


ΚX˜šΟnœ™!Jš
œB™BIcodeš
œ+™+—J™Jšœ™#J˜š
Οk	˜	Jšœžœ
˜J˜Jšœ
žœ!˜0Kšœžœ&˜7K˜K˜Kšžœ
˜Kšžœ
˜K˜K˜K˜J˜Jš
œ˜J˜J˜—J˜šœžœ
ž˜%Jšžœ)žœžœ?˜{J˜Jšžœžœžœ
˜J˜šœ˜ Kš
œ:˜:Kšœžœ˜Kšœžœ˜!Kšœžœ˜!š
œ!˜!šœ˜#šžœžœ
˜Kšœž˜—šžœ˜Kšœžœ
˜K˜—K˜—Kš
žœžœžœ
žœ
žœ˜JK•StartOfExpansionT[base: 
ROPE, start: 
INT _ 0, len: 
INT _ 2147483647, action: 
Rope.ActionType]šžœ1˜7Kš
œ˜—K™Kšœžœ
žœžœ(˜>Kšœžœ
žœžœ#˜CKš
œ˜Kšœžœ&˜-JšœMžœΑžœ
žœ˜Jš
œ˜Jš
œ˜J–'[self: 
STREAM, abort: 
BOOL _ FALSE]šžœ˜šžœ
˜Kš
œ˜Kšœ	žœF˜U—K˜—šœ˜ Kšœžœ
˜Kš
œ˜Kš
œ˜Kšœžœžœ
˜%Kšœžœžœ
˜(Kšœžœ˜Kš
œ?˜?šœžœ?žœžœ)˜“
š
œžœ	žœ
žœ
žœ˜@šžœžœžœ9žœ
˜bK–)[list: 
LIST OF REF ANY, ref: 
REF ANY]š
œ<˜<Kš
œ˜——Kš
œ;˜;K˜—šœžœžœžœ˜4šžœžœ
žœž˜*šžœž˜"Kšœ"žœ˜*Kšœ%žœ˜-˜
Kš
œ4˜4—K˜Kšžœ
˜—Kšžœ
˜—K˜—šžœžœ
žœž˜Kšœžœ˜Kšœ	žœžœžœ
žœ
žœ˜;šžœž˜KšœRžœ
žœ˜_˜K™&Kšœžœ
˜ šžœžœ
žœžœ
˜"Kšœ
žœ˜'—Kšžœ
˜Kšžœ
˜K˜—Kšžœžœ
˜——Kšžœ
˜šžœ3ž˜7Kš
œ/˜/šžœ˜Kšœ+žœ
žœ˜7——Kšœvžœ
žœ˜ƒ
K™Jš
œO˜OJš
œ˜šžœžœ
žœ˜JšœFžœžœ&˜ˆ
J™(Jš
œL˜LJ˜—šžœ
˜Kš
œ˜Kšœ	žœ7˜F—K˜—šœ˜"Kš
œ:˜:Kšœžœ ˜3Kšœžœ˜!Kšœžœ˜!JšœGžœa˜«
J–Κ[fileName: 
ROPE, accessOptions: 
FS.AccessOptions _ read, streamOptions: 
FS.StreamOptions _ (3)[TRUE, TRUE, TRUE], keep: 
CARDINAL _ 1B (1), createByteCount: 
FS.ByteCount _ 2560, streamBufferParms: 
FS.StreamBufferParms _ [vmPagesPerBuffer: 
8, nBuffers: 
2], extendFileProc: 
FS.ExtendFileProc, wantedCreatedTime: 
GMT _ nullGMT, remoteCheck: 
BOOL _ TRUE, wDir: 
ROPE _ NIL, checkFileType: 
BOOL _ FALSE, fileType: 
FS.FileType _ [0B (0)]]šœžœ
žœžœ@˜VJšžœ˜	J–F[self: 
STREAM, r: 
ROPE, start: 
INT _ 0, len: 
INT _ 2147483647]šžœžœ+žœ
žœ˜_J–'[self: 
STREAM, abort: 
BOOL _ FALSE]šžœ˜šžœ
˜Kš
œ˜Kšœ	žœE˜T—K˜—Kšœžœ˜.K™/šœžœ˜.K™4Kšœžœžœ)žœ	˜jJ–Κ[fileName: 
ROPE, accessOptions: 
FS.AccessOptions _ read, streamOptions: 
FS.StreamOptions _ (3)[TRUE, TRUE, TRUE], keep: 
CARDINAL _ 1B (1), createByteCount: 
FS.ByteCount _ 2560, streamBufferParms: 
FS.StreamBufferParms _ [vmPagesPerBuffer: 
8, nBuffers: 
2], extendFileProc: 
FS.ExtendFileProc, wantedCreatedTime: 
GMT _ nullGMT, remoteCheck: 
BOOL _ TRUE, wDir: 
ROPE _ NIL, checkFileType: 
BOOL _ FALSE, fileType: 
FS.FileType _ [0B (0)]]šœžœ
žœžœ@˜VJ–F[self: 
STREAM, r: 
ROPE, start: 
INT _ 0, len: 
INT _ 2147483647]šžœžœ#žœ
žœ˜WJšžœ˜J–'[self: 
STREAM, abort: 
BOOL _ FALSE]šžœ˜šžœ
˜Kš
œ˜Kšœ	žœK˜Z—K˜—šœ˜"Kšœžœ
žœžœ"˜BKšœžœ
žœžœ=˜SKšœžœ&˜-K–Y[self: 
STREAM, block: 
REF READONLY TEXT, startIndex: 
NAT _ 0, count: 
NAT _ 32767]šžœ'˜)J–'[self: 
STREAM, abort: 
BOOL _ FALSE]šžœ˜K˜—š
œ˜(Kš
œ:˜:Kšœžœ˜)Kšœžœ
žœžœ"˜BKšœžœ
žœžœ>˜TKšœžœ
˜Kšœ
žœ˜Kšœžœ&˜-Kš
œ+˜+šžœBžœ˜JK™Kšœžœ)˜4–([base: 
ROPE _ NIL, rest: 
ROPE _ NIL]š
œ˜K–9[base: 
ROPE, start: 
INT _ 0, len: 
INT _ 2147483647]š
œ6˜6K–9[base: 
ROPE, start: 
INT _ 0, len: 
INT _ 2147483647]š
œ.˜.K˜
K˜
—šžœ˜K–F[self: 
STREAM, r: 
ROPE, start: 
INT _ 0, len: 
INT _ 2147483647]š
œ ˜ Kšžœ
žœ˜——KšΠkzΟz
œ˜šžœ˜
Kšžœžœ žœ
žœ˜UK˜—Kšžœ/˜1Jšžœ˜J–'[self: 
STREAM, abort: 
BOOL _ FALSE]šžœ˜šžœ
˜Kš
œ˜Kšœ	žœ5˜D—K˜K˜—K˜K˜K™š
œžœ
žœžœžœžœžœžœ˜pš
œžœžœ
žœ
žœ˜<šœ
žœžœ
˜%Kšœžœ˜#Kš
ž˜Kšœžœ˜#—K˜—Kš
œ4˜4K˜K˜—š
œžœ
žœžœžœ˜JKšœžœ˜ K–[base: 
ROPE]šœžœ˜)K–)[rope: 
ROPE, oldStream: 
STREAM _ NIL]š	œžœ
žœžœ
Πbkœ˜*K–&[mod: 
NAT _ 17, case: 
BOOL _ TRUE]šœžœ˜$š
ž˜K–-[stream: 
STREAM, breakProc: 
IO.BreakProc]š	œžœžœžœžœ˜KK–1[x: 
SymTab.Ref, key: 
ROPE, val: 
SymTab.Val]šœ-žœ˜2—Kšžœ
˜K˜—J˜šœžœ
žœ'žœžœžœžœžœ˜š
Jš
œ,˜,š
œžœ	žœ
žœžœ˜MJšœžœžœ˜7š
*œžœ
žœ
žœžœ˜\Jšœžœžœ˜?Jš
œΥ
˜Υ
Jš
œ$˜$—J–B[from: 
INT, base: 
Convert.Base _ 10, showRadix: 
BOOL _ TRUE]˜Jš
œy˜yJ–B[from: 
INT, base: 
Convert.Base _ 10, showRadix: 
BOOL _ TRUE]šžœžœy˜•
Jšžœžœ_˜~Jš
œ'˜'Jš
œ˜—Jšœ
žœ
˜J–-[list: 
LIST OF REF ANY, proc: 
PROC [...]]š
œ>˜>Jšžœ˜J˜J˜—J˜J˜—J˜J™š
œ˜Kš
œ
˜
Kš
œ˜Kš
œ5˜5—š
œ˜Kš
œ
˜
Kš
œ˜Kšœ/Οu
˜0Kš
œ₯
˜₯
—š
œ˜Kš
œ˜Kš
œ˜Kš
œ8˜8—š
œ˜Kš
œ˜Kš
œ˜Kš
œE˜E—š
œ˜Kš
œ˜Kš
œ˜Kš
œ&˜&—š
œ˜Kš
œ˜Kš
œ˜Kš
œ˜—K˜˜K˜K˜K˜J˜K˜K˜K˜J˜—˜™J˜———…—$j8˜