File: WalnutUpdateImpl.mesa
Copyright © 1984 by Xerox Corporation. All rights reserved.
Contents:
reads the logStream, starting at startPos, & makes the appropriate database updates
Created by: Willie-Sue on April 27, 1983
Last edited by:
Willie-Sue on August 28, 1984 11:30:27 am PDT
DIRECTORY
DB USING [Value,
DestroyEntity, Eq, GetF, GetName, GetP, GetPList, Null, SetP,
I2V, V2B, V2E, V2I, V2S],
FS USING [Error, OpenFile,
Create, OpenOrCreate, PagesForBytes, StreamFromOpenFile, StreamOpen],
IO,
PrincOpsUtils USING [PagesForBytes],
Rope,
RopeIO USING [GetRope],
WalnutControlPrivate USING [InternalConfirm],
WalnutDB USING [Msg, MsgSet,
deletedMsgSet, mCategoryIs, mHasBeenReadIs, mHeadersPos, mMsgLengthIs,
msNumInSetIs, mPrefixPos, mSubjectIs,
AddMsgToMsgSet, DeclareMsg, DeclareMsgSet, DestroyMsgSet, MsgRecToMsg,
RemoveMsgFromMsgSet, SetMsgHasBeenRead],
WalnutDBLog USING [walnutInfoRelship, wStartExpungePos],
WalnutVoice USING [CheckForWalnuthatch, RemoveInterestEntry],
WalnutLog USING [minPrefixLength, MsgRec, MessageRecObject,
copyBuffer, msgIDRope,
ConstructMsgID, GenerateNewMsgID],
WalnutLogExtras USING [onlyOneTempLog, walnutTempLogName],
WalnutRetrieve USING [ParseMsgIntoFields],
WalnutStream USING [FindStartOfEntry, IsStart, MakeLogEntry, MsgRecFromStream,
TagAndValue],
WalnutWindow USING [walnutLogName, Report, ReportRope];
WalnutUpdateImpl: CEDAR PROGRAM
IMPORTS DB, FS, IO, PrincOpsUtils, Rope, RopeIO,
WalnutControlPrivate, WalnutDB, WalnutDBLog, WalnutLog, WalnutLogExtras,
WalnutRetrieve, WalnutStream, WalnutVoice, WalnutWindow
EXPORTS WalnutStream =
BEGIN OPEN DB, WalnutDB, WalnutLog, WalnutStream;
ROPE: TYPE = Rope.ROPE;
skipRope: ROPE← "Unexpected EOF; trying for next entry";
maxTailRewriteBytes: INT← 1250000;  -- try this out
BadCopyForExpunge: SIGNAL[numRead, numExpected: INT] = CODE;
********************************************************
UpdateFromStream: PUBLIC PROC [strm: IO.STREAM, startPos: INT] RETURNS[success: BOOL] =
BEGIN
doReport: BOOL← (startPos = 0);  -- report if scavenging from beginning
BadFormat: PROC[s: ROPE] =
{ IF doReport THEN
WalnutWindow.Report[IO.PutFR
["Bad log file format at %g: %g", IO.int[strm.GetIndex[]], IO.rope[s]]]
};
BEGIN ENABLE IO.EndOfStream => {BadFormat["Unexpected EOF"]; GOTO GiveUp};
entryLength, prefixLength, beginMsgPos: INT;
msg: Msg;
msgSet: MsgSet;
entryChar: CHAR;
count: INT← 0;
DomainUpdate: PROC RETURNS[BOOL] =
BEGIN
msName, tag: ROPE;
[tag, msName]← TagAndValue[strm, 2];
IF NOT tag.Equal["name"] THEN {BadFormat["expected name"]; RETURN[FALSE]};
WalnutWindow.ReportRope[Rope.FromChar[entryChar]];
SELECT entryChar FROM
'- =>
{msgSet← DeclareMsgSet[msName, OldOnly].msgSet;
IF msgSet = NIL THEN RETURN[TRUE];
[]← DestroyMsgSet[msgSet]
};
'+ => []← DeclareMsgSet[msName];
ENDCASE => ERROR;
RETURN[TRUE];
END;
RelationUpdate: PROC RETURNS[BOOL] =
BEGIN
mName, categoryName, tag: ROPE;
[tag, mName]← TagAndValue[strm, 2];
IF NOT tag.Equal["of"] THEN {BadFormat["of?"]; RETURN[FALSE]};
msg← DeclareMsg[mName, OldOnly].msg;
IF msg=NIL THEN {BadFormat[Rope.Cat[mName, " doesn\'t exist!"]]; RETURN[FALSE]};
[tag, categoryName]← TagAndValue[strm, 2];
IF NOT tag.Equal["is"] THEN {BadFormat["is?"]; RETURN[FALSE]};
WalnutWindow.ReportRope[Rope.FromChar[entryChar]];
SELECT entryChar FROM
'- =>
{ msgSet← DeclareMsgSet[categoryName, OldOnly].msgSet; -- check if msgSet exists
IF msgSet # NIL THEN []← RemoveMsgFromMsgSet[msg, msgSet]
};
'+ => -- create msgSet if it doesn't exist
[]← AddMsgToMsgSet[msg, DeclareMsgSet[categoryName].msgSet]
ENDCASE => ERROR;
RETURN[TRUE];
END;
ProcessHasBeenRead: PROC =
BEGIN
len: INT;
mName: ROPE← RopeIO.GetRope[strm, prefixLength-minPrefixLength ! IO.EndOfStream =>
{ WalnutWindow.Report["\nUnexpected EOF, ignoring HasBeenRead entry"]; GOTO eof}];
IF mName.Fetch[len← mName.Length[]-1] = '\n THEN mName← mName.Substr[0, len];
msg← DeclareMsg[mName, OldOnly].msg;
IF msg = NIL THEN RETURN; -- ignore
SetMsgHasBeenRead[msg];
EXITS eof => RETURN;
END;
strm.SetIndex[startPos];
DO
Read *start* from log
[beginMsgPos, prefixLength, entryLength, entryChar]← FindStartOfEntry[strm, doReport];
IF entryLength = -1 THEN RETURN[TRUE];
IF (entryLength=0) OR (entryLength=prefixLength) AND
NOT (entryChar = '←) THEN RETURN[TRUE];
Do delete, create, or message update to database
SELECT entryChar FROM
'← => ProcessHasBeenRead[]; -- mark message as read
'+, '- => -- add or remove relship
BEGIN
foo, domainOrRelation: ROPE;
ok: BOOL;
strm.SetIndex[beginMsgPos+prefixLength];  -- ignore prefix info
[domainOrRelation, foo] ← TagAndValue[strm, 2];
IF domainOrRelation.Equal["Domain"] THEN
{ IF (ok← foo.Equal["MsgSet"]) THEN ok← DomainUpdate[] -- Add or delete MsgSet
ELSE BadFormat[IO.PutFR["%g is not a valid domain", IO.rope[foo]]];
}
ELSE IF domainOrRelation.Equal["Relation"] THEN
{ IF (ok← foo.Equal["mCategory"]) THEN ok← RelationUpdate[] --AddTo or RemoveFrom
ELSE BadFormat["expected mCategory"]
}
ELSE -- domainOrRelation not "Domain" or "Relation"
{BadFormat["not domain or relation"]; ok← FALSE};
IF ~ok THEN strm.SetIndex[beginMsgPos+entryLength] -- next entry
ELSE IF strm.GetChar[]#'\n THEN BadFormat["Missing CR"];-- Skip over the trailing CR
END;
ENDCASE => -- regular message entry
BEGIN
msgRec: MsgRec←
WalnutStream.MsgRecFromStream[strm, prefixLength, entryLength-prefixLength];
IF msgRec = NIL THEN
WalnutWindow.Report[
IO.PutFR["\n Bad Entry at %g; skipping to next entry", IO.int[beginMsgPos]]]
ELSE
{ IF entryChar # '? THEN msgRec.hasBeenRead← TRUE;
[]← MsgRecToMsg[msgRec];
count← count + 1;
IF doReport THEN
{IF count MOD 10 = 0 THEN
WalnutWindow.ReportRope[IF count MOD 100 = 0 THEN "!" ELSE "~"]}
ELSE WalnutWindow.ReportRope["."];
};
strm.SetIndex[beginMsgPos+entryLength];
END;
ENDLOOP;
EXITS GiveUp => RETURN[TRUE];  -- try to press on with what we have
END;
END;
ExpungeFromStream: PUBLIC PROC[strm: IO.STREAM, doUpdates, tailRewrite: BOOL]
RETURNS[startExpungePos: INT, ok: BOOL, tempLog: IO.STREAM] =
Dumping is driven from the log file & preserves the bits that came from Grapevine
Dumps all the Msgs in the database to a tempLog, setting all
their body pointers to reference the NEW log, then copies tempLog to logStream
BEGIN OPEN IO;
newLen: INT;
BEGIN ENABLE IO.EndOfStream => GOTO badLogFile;
msg: Msg;
toBeDestroyedList: LIST OF Msg;
count: INT← 0;
expungeFilePos, startPos, prefixLength, entryLength: INT;
entryChar: CHAR;
numBad: INTEGER← 0;
DoCount: PROC =
{IF (count← count + 1) MOD 10 = 0 THEN
WalnutWindow.ReportRope[IF count MOD 100 = 0 THEN "!" ELSE "~"]
};
WriteBadLogEntry: PROC[startPos: INT] RETURNS[nextStartPos: INT] =
{ errorStream: STREAM← FS.StreamOpen["WalnutExpunge.errlog",
IF (numBad← numBad+1) = 1 THEN $create ELSE $append];
line: ROPE;
strm.SetIndex[startPos];
errorStream.PutRope["\n******************************************************"];
errorStream.PutRope[
IO.PutFR["\nBad entry from pos %g of log file, written at %g\n", int[startPos], time[]]];
errorStream.PutRope[strm.GetLineRope[]]; -- *start* line
errorStream.PutChar['\n];
DO
nextStartPos← strm.GetIndex[];
line← strm.GetLineRope[];
IF IsStart[line].startFound THEN {nextStartPos← strm.GetIndex[]-8; EXIT};
errorStream.PutRope[line];
errorStream.PutChar['\n];
ENDLOOP;
errorStream.SetLength[errorStream.GetIndex[]];
errorStream.Close[];
};
CopyEntryIfNecessary: PROC =
{ thisTag, thisValue: ROPE← NIL;
thisMsg: Msg;
doCopy: BOOL← TRUE;
prefixPos: INT;
IF entryChar = '← THEN  -- hasbeenread entry
{ -- strm.SetIndex[startPos+minPrefixLength]; where strm is pos'd
thisValue← RopeIO.GetRope[strm, prefixLength-minPrefixLength-1];
[]← strm.GetChar[ ! IO.EndOfStream => CONTINUE]
}
ELSE
{ strm.SetIndex[startPos+prefixLength];
[thisTag, thisValue]← TagAndValue[strm, 2];
IF thisTag.Equal["Relation"] AND thisValue.Equal["mCategory"] THEN
{ [thisTag, thisValue]← TagAndValue[strm, 2];
IF ~thisTag.Equal["of"] THEN thisValue← NIL;
};
};
IF thisValue # NIL THEN
{ thisMsg← DeclareMsg[thisValue, OldOnly].msg;
IF ~Null[thisMsg] THEN
IF (prefixPos← V2I[GetP[thisMsg, mPrefixPos]]) >= startExpungePos THEN
doCopy← FALSE;
};
IF doCopy THEN
{ numToDo: INT;
strm.SetIndex[startPos];
FOR numToDo← entryLength, numToDo-512 UNTIL numToDo<512 DO
[]← strm.GetBlock[copyBuffer];
tempLog.PutBlock[copyBuffer];
ENDLOOP;
IF numToDo # 0 THEN
{ numRead: INT← strm.GetBlock[copyBuffer, 0, numToDo];
IF numRead # numToDo THEN SIGNAL BadCopyForExpunge[numRead, numToDo];
tempLog.PutBlock[copyBuffer, 0, numToDo];
copyBuffer.length← 512;
};
};
};
logLength: INT← strm.GetLength[];
bytesInTail: INT;
logIsOnAlpine: BOOL← WalnutWindow.walnutLogName.Find[".alpine]", 0, FALSE] > 0;
startExpungePos←
IF tailRewrite THEN
V2I[GetF[WalnutDBLog.walnutInfoRelship, WalnutDBLog.wStartExpungePos]] ELSE 0;
if log file is on alpine AND tailRewrite AND startExpungePos#0 AND
(loglength-startExpungePos)>1.25mb, THEN do full expunge anyway (and inform user)
IF logIsOnAlpine AND tailRewrite AND startExpungePos#0 AND
(bytesInTail← logLength - startExpungePos) > maxTailRewriteBytes THEN
{ WalnutWindow.Report[IO.PutFR[
"Tail of log to be expunged was %g bytes - doing full expunge", IO.int[bytesInTail]]];
tailRewrite← FALSE;
startExpungePos← 0;
};
IF startExpungePos = 0 THEN {tailRewrite← FALSE; doUpdates← FALSE};
BEGIN ENABLE FS.Error => {WalnutWindow.Report[error.explanation]; GOTO quit};
pgs: INT← FS.PagesForBytes[IF tailRewrite THEN bytesInTail ELSE logLength/2];
of: FS.OpenFile←
IF WalnutLogExtras.onlyOneTempLog THEN
FS.OpenOrCreate[name: WalnutLogExtras.walnutTempLogName, keep: 1, pages: pgs]
ELSE FS.Create[name: WalnutLogExtras.walnutTempLogName, pages: pgs];
tempLog← FS.StreamFromOpenFile[of, $write];
tempLog.SetIndex[0];  -- start at beginning!!!
tempLog.SetLength[0];
EXITS
quit => NULL;
END;
IF tempLog = NIL THEN RETURN[startExpungePos, FALSE, NIL];
strm.SetIndex[startExpungePos];
DO   -- loop for dumping messages & making other log entries
msgID: ROPE← NIL;
fullText: ROPE← NIL;
headersPos, curPos: INT;
badEntry: BOOL← FALSE;
[startPos, prefixLength, entryLength, entryChar]← FindStartOfEntry[strm, TRUE];
IF entryLength = 0 THEN EXIT;
IF (entryChar # ' ) AND (entryChar # '?) THEN -- perhaps copy other entry to tempLog
{ IF startExpungePos # 0 THEN CopyEntryIfNecessary[];
strm.SetIndex[startPos+entryLength]; -- for good measure, even after CopyEntry
LOOP
};
headersPos← startPos+prefixLength;
UNTIL (curPos← strm.GetIndex[]) = headersPos DO
look for a msgID: field
tag, value: ROPE;
[tag, value]← TagAndValue[strm: strm, inc: 2];
IF Rope.Equal[tag, msgIDRope, FALSE] THEN msgID← value;
IF curPos > headersPos THEN -- bad entry we don't understand
{ nextStartPos: INT← WriteBadLogEntry[startPos];
cMsg: ROPE← " Confirm to push on, deny to stop the Expunge";
WalnutWindow.ReportRope[PutFR["\nBad message entry at log pos %g;", int[startPos]]];
WalnutWindow.Report[" entry written on WalnutExpunge.errlog"];
WalnutWindow.Report[cMsg];
IF (ok← WalnutControlPrivate.InternalConfirm[])
THEN {strm.SetIndex[nextStartPos]; badEntry← TRUE; EXIT};
RETURN[startExpungePos, FALSE, tempLog]};
ENDLOOP;
IF badEntry THEN LOOP;
IF msgID = NIL THEN
BEGIN  -- have to read fullText to fashion msgID
mr: MsgRec← NEW[MessageRecObject];
[]← WalnutRetrieve.ParseMsgIntoFields[mr, strm, entryLength-prefixLength !
IO.EndOfStream =>
{WalnutWindow.Report[skipRope]; strm.SetIndex[startPos+entryLength]; LOOP}];
mr.gvID← ConstructMsgID[mr];
mr.msgLength← entryLength-prefixLength;
need to make sure this is really a unique id for this message
some msgs from old mail files are not
IF ~CheckID[mr] THEN {strm.SetIndex[startPos+entryLength]; LOOP};
strm.SetIndex[startPos+prefixLength];  -- back up stream
msgID← mr.gvID;
END;
msg← DeclareMsg[msgID, OldOnly].msg;
IF (msg=NIL) OR Null[msg] THEN {strm.SetIndex[startPos+entryLength]; LOOP}
ELSE
BEGIN
curPrefixPos, thisPrefixLen, thisEntryLen: INT;
thisMsgSet: MsgSet;
nameOfCatsList: ROPE← NIL;
if is only in deletedMsgSet, don't put on tempLog
FOR mL: LIST OF DB.Value← GetPList[msg, mCategoryIs], mL.rest UNTIL mL=NIL DO
IF ~Eq[thisMsgSet← V2E[mL.first], deletedMsgSet] THEN
nameOfCatsList← Rope.Cat[nameOfCatsList, " ", DB.GetName[thisMsgSet]];
ENDLOOP;
IF nameOfCatsList.Length[] = 0 THEN
{ toBeDestroyedList← CONS[msg, toBeDestroyedList]; GOTO doneWithOne};
msgID← Rope.Cat[msgID, "\nCategories:", nameOfCatsList];
IF fullText = NIL THEN fullText←
RopeIO.GetRope[strm, entryLength-prefixLength ! IO.EndOfStream =>
{ WalnutWindow.Report[skipRope]; GOTO doneWithOne}];
DoCount[];
[expungeFilePos, thisPrefixLen, thisEntryLen]← MakeLogEntry[
tempLog,
IF V2B[GetP[msg, mHasBeenReadIs]] THEN message ELSE newMessage,
fullText, Rope.Cat[msgIDRope, ": ", msgID, "\n"]];
set header & body pos pointers to posn in NEW log if doing updates:
IF doUpdates THEN
{ thisPrefixPos: INT← startExpungePos + expungeFilePos - thisEntryLen;
curHeadersPos: INT← V2I[GetP[msg, mHeadersPos]];
thisHeadersPos: INT← thisPrefixPos+thisPrefixLen;
curPrefixPos← V2I[GetP[msg, mPrefixPos]];
IF (curPrefixPos # thisPrefixPos) THEN []← SetP[msg, mPrefixPos, I2V[thisPrefixPos]];
IF (curHeadersPos # thisHeadersPos) THEN
[]← SetP[msg, mHeadersPos, I2V[thisHeadersPos]];
};
EXITS
doneWithOne => strm.SetIndex[startPos+entryLength];
END;  -- put message on log file
ENDLOOP;  -- loop for dumping messages
WalnutWindow.Report[PutFR["\nDumped %g messages from Log File... ", int[count]]];
tempLog.SetLength[newLen← tempLog.GetIndex[]];
tempLog.Flush[];  -- make sure tempLog is in good shape
WalnutWindow.Report[PutFR[
"Old log length was %g bytes, (%g pages)", int[logLength],
 int[PrincOpsUtils.PagesForBytes[logLength]]]];
IF tailRewrite THEN
{ oldEnd: INT← strm.GetLength[]-startExpungePos;
fullLen: INT← startExpungePos+newLen;
WalnutWindow.Report[PutFR[
" Previous tail length was %g bytes, (%g pages)",
 int[oldEnd], int[PrincOpsUtils.PagesForBytes[oldEnd]]]];
WalnutWindow.Report[PutFR[
" New tail length is %g bytes (%g pages)", int[newLen],
 int[PrincOpsUtils.PagesForBytes[newLen]]]];
WalnutWindow.Report[PutFR[
"New log length is %g bytes, (%g pages)", int[fullLen],
 int[PrincOpsUtils.PagesForBytes[fullLen]]]];
}
ELSE
WalnutWindow.Report[PutFR
["New log length is %g bytes (%g messages, %g pages)", int[newLen], int[count],
int[PrincOpsUtils.PagesForBytes[newLen]]]];
IF doUpdates THEN
{ nuh: REF ANY;
msg: Msg;
id: ROPE;
WalnutWindow.ReportRope[" Destroying Msgs in Deleted MsgSet ..."];
may see a msg more than once in the log file, hence must check if already destroyed
count← 0;
nuh← WalnutVoice.CheckForWalnuthatch[];
FOR mL: LIST OF Msg← toBeDestroyedList, mL.rest UNTIL mL = NIL DO
IF ~Null[msg← mL.first] THEN
{ IF nuh#NIL THEN id← GetName[msg];
DestroyEntity[mL.first];  -- destroy the message first
IF nuh#NIL THEN WalnutVoice.RemoveInterestEntry[NIL, id];
DoCount[];
};
ENDLOOP;
WalnutWindow.Report[PutFR["\n%g Msgs destroyed",int[count]]];
BEGIN   -- fix up num in deleted (not necessarily zero)
num: INT← V2I[GetP[deletedMsgSet, msNumInSetIs]] - count;
[]← SetP[deletedMsgSet, msNumInSetIs, I2V[num]];
END;
};  -- end doUpdates
RETURN[startExpungePos, TRUE, tempLog];
EXITS
badLogFile =>
{ tempLog.SetLength[tempLog.GetIndex[]];
tempLog.Close[];
RETURN[startExpungePos, FALSE, NIL]
};
END;
END;
CheckID: PROC[mr: MsgRec] RETURNS[useThisMsg: BOOL] =
BEGIN
len: INT;
subj: ROPE;
msg: Msg;
DO
msg← DeclareMsg[mr.gvID, OldOnly].msg;
IF (msg=NIL) OR Null[msg] THEN RETURN[FALSE];
len← V2I[GetP[msg, mMsgLengthIs]];
IF len # mr.msgLength THEN { GenerateNewMsgID[mr]; LOOP};
subj← V2S[GetP[msg, mSubjectIs]];
IF ~Rope.Equal[subj, mr.subject] THEN { GenerateNewMsgID[mr]; LOOP};
RETURN[TRUE];
ENDLOOP;
END;
END.