-- File: WalnutDBLogImpl.mesa
-- Contents:
-- types and procedures for writing on the log file

-- Created by: Willie-Sue on July 16, 1982
-- Last edited by:
-- Willie-Sue Hoo-Hah on May 3, 1983 9:10 am
-- Rick Cattell on XXX
-- MBrown, December 15, 1982 11:42 am

DIRECTORY
 DateAndTime USING[Parse, Unintelligible],
 GVBasics USING[Timestamp],
 GVRetrieve USING [Handle],
 File USING [Capability],
 FileIO USING[CapabilityFromStream, commitAndReopenTransOnFlush, finishTransOnClose,
   Open, StreamFromCapability],
 IO,
 RopeIO USING [GetRope],
 Rope,
 System USING [GreenwichMeanTime],
 Time USING [Current],
 DB,
 UserCredentials USING [GetUserCredentials],
 ViewerTools USING [TiogaContentsRec],
 WalnutDB,
 WalnutDBAccess,
 WalnutDBLock,
 WalnutDBLog,
 WalnutDisplayerPrivate,
 WalnutRetrieve,
 WalnutSendMail,
 WalnutWindow;
 
WalnutDBLogImpl: CEDAR PROGRAM
IMPORTS DateAndTime, FileIO, IO, RopeIO, Rope, Time, UserCredentials,
  DB, WalnutDB, WalnutDBAccess, WalnutDBLock, WalnutDBLog,
  WalnutDisplayerPrivate, WalnutRetrieve, WalnutSendMail, WalnutWindow
EXPORTS WalnutDB, WalnutDBLog =
  
BEGIN OPEN DB, WalnutDB, WalnutDBAccess, WalnutDBLog, WalnutWindow;

ROPE: TYPE = Rope.ROPE;

msgIDRope: PUBLIC ROPE← "gvMsgID" ;
categoriesRope: PUBLIC ROPE← "Categories";

bufferSize: CARDINAL = 512;
copyBuffer: PUBLIC REF TEXT ← NEW[TEXT[bufferSize]];  -- one to share

-- ********************************************************
logStream: IO.STREAM← NIL;

lastMsgAddedToDB: INT; -- pos in log after last msg entered in database
lastMsgWrittenToLog: INT;  -- pos in log at end of last msg retrieved
skipRope: ROPE← "\nUnexpected EOF, skipping ahead.";

-- ********************************************************

DoInitializeLog: PUBLIC PROC RETURNS [curLength: INT] =
BEGIN
IF logStream # NIL THEN RETURN[logStream.GetLength[]];  -- already open

IF DB.TransactionOf[$Walnut] = NIL THEN OpenWTransaction[NIL, FALSE];
logStream← FileIO.Open[fileName: WalnutWindow.walnutLogName,
  accessOptions: write,
  closeOptions: FileIO.commitAndReopenTransOnFlush+FileIO.finishTransOnClose];
RETURN[lastMsgWrittenToLog← logStream.GetLength[]]
END;

DoCloseLogStream: PUBLIC PROC =
{ IF logStream#NIL THEN logStream.Close[]; logStream← NIL};

FinishExpunge: PUBLIC PROC =
-- called if wCopyInProgress is TRUE during startup
BEGIN
 startPos: INT← V2I[DB.GetF[walnutInfoRelship, wStartExpungePos]];
 tempLog: IO.Handle← FileIO.Open["Walnut.TempLog"];
 CopyTempLogToLog[tempLog, startPos];
 DB.MarkTransaction[DB.TransactionOf[$Walnut]];
END;

-- must know when the state of Walnut's segment is changing

OpenWTransaction: PUBLIC PROC[trans: DB.Transaction, noLog: BOOL] =
BEGIN
 fileNotFound, transOpen: BOOL← FALSE;
 IF walnut#NIL THEN ReportRope[" Opening Walnut transaction ..."];
 DB.OpenTransaction[
segment: $Walnut,
userName: WalnutSendMail.userRName,
password: UserCredentials.GetUserCredentials[].password,
useTrans: trans,
noLog: noLog ! DB.Error =>
IF code=FileNotFound THEN {fileNotFound← TRUE; CONTINUE}
ELSE IF code=TransactionAlreadyOpen THEN {transOpen← TRUE; CONTINUE}];

IF fileNotFound OR transOpen THEN
{ DB.CloseTransaction[DB.TransactionOf[$Walnut]];
IF fileNotFound THEN DB.DeclareSegment[walnutSegmentFile, $Walnut,,, NewOnly];
DB.OpenTransaction[
 segment: $Walnut,
 userName: WalnutSendMail.userRName,
 password: UserCredentials.GetUserCredentials[].password,
 useTrans: trans,
 noLog: noLog];
IF fileNotFound THEN
 DB.MarkTransaction[DB.TransactionOf[$Walnut]]; -- make sure segment initializated
};

 WalnutDBAccess.DoInitializeDBVars[];
 lastMsgAddedToDB← V2I[DB.GetF[walnutInfoRelship, wExpectedDBPos]];
 WalnutWindow.UpdateMsgSetButtonEntities[];
END;

CloseWTransaction: PUBLIC PROC =
BEGIN
 len: INT;
 ReportRope[" Closing Walnut transaction ..."];
 IF logStream#NIL THEN
  { len← logStream.GetLength[];
  logStream.Flush[];
  IF lastMsgAddedToDB >= lastMsgWrittenToLog THEN lastMsgAddedToDB← len;
  DB.SetF[walnutInfoRelship, wExpectedLength, I2V[len]];
  DB.SetF[walnutInfoRelship, wExpectedDBPos, I2V[lastMsgAddedToDB]];
  DB.CloseTransaction[DB.TransactionOf[$Walnut]];
  };
 WalnutWindow.SetWalnutUpdatesPending[FALSE];
 Report[" ... done"];
END;

AbortWTransaction: PUBLIC PROC =
BEGIN
 ReportRope[" Aborting Walnut transaction ..."];
 DB.AbortTransaction[TransactionOf[$Walnut]];
 WalnutDBAccess.DoInitializeDBVars[];
 lastMsgAddedToDB← V2I[GetF[walnutInfoRelship, wExpectedDBPos]];
 lastMsgWrittenToLog← logStream.GetLength[];
-- WalnutWindow.UpdateMsgSetButtonEntities[];
 Report[" ... done"];
END;

MarkWTransaction: PUBLIC PROC =
BEGIN
 len: INT← logStream.GetLength[];
 logCapability: File.Capability;
 IF WalnutWindow.logFileIsPilotFile THEN
   logCapability← FileIO.CapabilityFromStream[logStream];
-- close and reopen logFile to get create date set
 logStream.Close[];
 IF lastMsgAddedToDB >= lastMsgWrittenToLog THEN lastMsgAddedToDB← len;
 ReportRope[" Saving Walnut updates ..."];
 DB.SetF[walnutInfoRelship, wExpectedLength, I2V[len]];
 DB.SetF[walnutInfoRelship, wExpectedDBPos, I2V[lastMsgAddedToDB]];
 DB.MarkTransaction[DB.TransactionOf[$Walnut]];
 IF WalnutWindow.logFileIsPilotFile THEN
  logStream← FileIO.StreamFromCapability[
   capability: logCapability,
    accessOptions: write,
    closeOptions: FileIO.commitAndReopenTransOnFlush+FileIO.finishTransOnClose,
    fileName: WalnutWindow.walnutLogName]
  ELSE
  logStream← FileIO.Open[
   fileName: WalnutWindow.walnutLogName,
   accessOptions: write,
   closeOptions: FileIO.commitAndReopenTransOnFlush+FileIO.finishTransOnClose];

 WalnutWindow.SetWalnutUpdatesPending[FALSE];
 Report[" ... done"];
END;

LogLength: PUBLIC PROC[doFlush: BOOL] RETURNS[INT] =
{IF doFlush THEN logStream.Flush[]; RETURN[logStream.GetLength[]]};

DoArchiveMsgSet: PUBLIC PROC[msgSet: MsgSet, strm: IO.STREAM, doDelete: BOOL← FALSE]
 RETURNS[num: INT] =
-- writes a log style file for msgSet on strm
BEGIN
 rel, newRel: Relship;
 isDeletedDisplayed: BOOL← (FindMSViewer[deletedMsgSet] # NIL);
 rs: RelshipSet← RelationSubset[mCategory, LIST[[mCategoryIs, msgSet]]];
 restOfPrefix: ROPE← Rope.Cat["\nCategories: ", DB.GetName[msgSet], "\n"];
 num← 0;
 UNTIL DB.Null[rel← NextRelship[rs]] DO
  msg: Msg← WalnutDBAccess.GetFE[rel, mCategoryOf];
  startPos: INT← V2I[DB.GetP[msg, mHeadersPos]];
  length: INT← V2I[DB.GetP[msg, mMsgLengthIs]];
  fullText: ROPE← MsgRopeFromLog[startPos, length];
  prefix: ROPE← Rope.Cat[msgIDRope, ": ", DB.GetName[msg], restOfPrefix];
  []← MakeLogEntry[message, fullText, strm, prefix];
  num← num + 1;
  IF doDelete THEN
  { newRel← RemoveFrom[msg, msgSet, rel, TRUE, TRUE];
    IF isDeletedDisplayed AND (newRel # NIL) THEN
      WalnutDisplayerPrivate.AddMsgToMsgSetDisplayer[msg, deletedMsgSet, newRel];
   };
 ENDLOOP;
 ReleaseRelshipSet[rs];
END;

GVMessageToLog: PUBLIC PROC
[gvH: GVRetrieve.Handle, timeStamp: GVBasics.Timestamp, gvSender: RName]
 RETURNS[ok: BOOL] =
-- reads message items from Grapevine & makes a log entry for this message
{ prefix: ROPE←
Rope.Cat[msgIDRope, ": ", gvSender, " $ ", RopeFromTimestamp[timeStamp],
  "\nCategories: Active\n"];
[lastMsgWrittenToLog, ok]← GVLogEntry[gvH, logStream, prefix];
};

DoAddMessageToLog: PUBLIC PROC [entryText, prefix: ROPE] =
-- makes a message LogEntryType on log (used by old mail reader)
{ lastMsgWrittenToLog← MakeLogEntry[message, entryText, logStream, prefix]};

WriteMsgReadEntryToLog: PUBLIC PROC[msg: Msg] =
-- changes mHasBeenReadIs prop to TRUE & makes log entry
{ []← MakeLogEntry[hasbeenread, NIL, logStream, Rope.Concat[DB.GetName[msg], "\n"]];
[]← DB.SetP[msg, mHasBeenReadIs, B2V[TRUE]];
WalnutWindow.SetWalnutUpdatesPending[TRUE];
};

GetTiogaMsgFromLog: PUBLIC PROC[msg: Msg]
  RETURNS[contents: TiogaContents, startPos, length: INT] =
BEGIN
 startPos← V2I[DB.GetP[msg, mHeadersPos]];
 length← V2I[DB.GetP[msg, mMsgLengthIs]];
 contents← MsgFromLog[startPos, length];
END;

MsgFromLog: PUBLIC PROC[startPos, length: INT] RETURNS[contents: TiogaContents] =
BEGIN
 contents←
  WalnutSendMail.TiogaTextFromStrm[logStream, startPos-1, length+1];  -- hack for now

 IF contents # NIL THEN
  { IF contents.formatting=NIL THEN
   contents.contents← Rope.Substr[contents.contents, 1];
  RETURN;
  };
 contents← NEW[ViewerTools.TiogaContentsRec];
 contents.contents←
  IO.PutFR["[message body lost!!! (%g,%g)]\n", IO.int[startPos], IO.int[length]];
END;

MsgRopeFromLog: PUBLIC PROC[startPos, length: INT] RETURNS[ROPE] =
BEGIN ENABLE IO.EndOfStream => GOTO TooShort;

 logStream.SetIndex[startPos];
 RETURN[RopeIO.GetRope[logStream, length]];

 EXITS TooShort =>
RETURN[IO.PutFR["[message body lost!!! (%g,%g)]\n", IO.int[startPos], IO.int[length]]];
END;


-- ********************************************************
-- operations that read the log

ScavengeFromLog: PUBLIC PROC RETURNS[success: BOOL] =
-- Erase old segment and create new one from the log file
BEGIN
[]← InitializeLog[];  -- this does InitializeDBVars[]

IF success← ReadLogFile[0] THEN
{ Report[" WalnutScavenge done"];
lastMsgAddedToDB← lastMsgWrittenToLog← logStream.GetLength[];
DB.SetF[walnutInfoRelship, wExpectedLength, I2V[lastMsgWrittenToLog]];
DB.SetF[walnutInfoRelship, wExpectedDBPos, I2V[lastMsgAddedToDB]]}
ELSE Report["\nThere were problems doing the scavenge."];

logStream.Close[];
logStream← NIL;
END;

ProcessOldMessageFile: PUBLIC PROC[strm: IO.STREAM, defaultPrefix: ROPE]
  RETURNS [ok: BOOL] =
{ RETURN[OldMessageFile[strm, defaultPrefix]]};

GetMessagesFromLog: PUBLIC PROC[startOfNewMessages: INT]
  RETURNS[curLength, numNewMsgs: INT] =
BEGIN
anymore, msgWasNew: BOOL;
endPos: INT;
startPos: INT← startOfNewMessages;
msV: Viewer← NIL;
prevMsgSet: MsgSet← NIL;
msgRec: MsgRec;

numNewMsgs← 0;

 DO
  [endPos, anymore, msgRec, msgWasNew]← WalnutDBLock.GetOneMessageFromLog[startPos];
  IF ~anymore THEN EXIT;
  IF msgRec = NIL THEN {startPos← endPos; LOOP};
  IF msgWasNew THEN numNewMsgs← numNewMsgs + 1;
  FOR rL: LIST OF RelshipMsgSetPair← msgRec.relList, rL.rest UNTIL rL = NIL DO
  IF ~EqEntities[rL.first.msgSet, prevMsgSet] THEN
   msV← FindMSViewer[prevMsgSet← rL.first.msgSet];
  IF msV#NIL THEN
   WalnutDisplayerPrivate.AddParsedMsgToMSViewer[msgRec, msV, rL.first.rel];
  ENDLOOP;
  startPos← endPos;
ENDLOOP;

RETURN[endPos, numNewMsgs];
END;

OneMsgFromLog: PUBLIC PROC[startOfNextMessage: INT]
  RETURNS[endPos: INT, anymore: BOOL, msgRec: MsgRec, msgWasNew: BOOL] =
BEGIN
 existed: BOOL;
startPos, fullLength, prefixLength, prefixInfoLength: INT;
entryChar: CHAR;
msg: Msg;
categoriesList: LIST OF ROPE;
relList: LIST OF RelshipMsgSetPair← NIL;

CategoryRelshipList: PROC[msgSetName: ROPE] =
BEGIN
msgSet: MsgSet;
existed: BOOL;
rel: Relship;
avl: AttributeValueList;

[msgSet, existed]← WalnutDBAccess.DoDeclareMsgSet[msgSetName];

avl← LIST[[mCategoryOf, msg], [mCategoryIs, msgSet],
      [mCategoryDate, DB.GetP[msg, mDateCodeIs]]];
  rel← DeclareRelship[mCategory, avl, OldOnly];
  IF rel = NIL THEN
  { relList← CONS[[CreateRelship[mCategory, avl], msgSet], relList];
  ChangeNumInSet[msgSet, 1];
  };
  IF ~existed THEN AddToMsgSetButtons[msgSet, msgSetName];
  END;

  ELtoNL: PROC[el: LIST OF Value] RETURNS [nl: LIST OF RName] =
  BEGIN
  IF el=NIL THEN RETURN[NIL]
  ELSE RETURN[CONS[V2S[DB.GetName[V2E[el.first]]], ELtoNL[el.rest]]];
  END;

 logStream.SetIndex[startOfNextMessage];
-- Read *start* from log
 [startPos, prefixLength, fullLength, entryChar]← ReadStartOfMsg[logStream, TRUE];
 IF (fullLength = 0) OR (fullLength = -1) THEN
  { lastMsgAddedToDB← logStream.GetIndex[];
  RETURN[lastMsgAddedToDB, FALSE, NIL, FALSE];
  };

 prefixInfoLength← prefixLength - minPrefixLength;
-- ignore all log entries except those with entryChar = '? or SP
-- any other entries have been processed
 SELECT entryChar FROM
  '?, ' => -- msg entry
  { [msgRec, existed, categoriesList]←
    GetMsgFromStream[logStream, prefixLength, fullLength-prefixLength, entryChar];
   IF (msg← msgRec.msg) # NIL THEN
   { IF ~existed THEN  -- add to msgSets
   { IF categoriesList = NIL THEN CategoryRelshipList["Active"]
   ELSE
   FOR cL: LIST OF ROPE← categoriesList, cL.rest UNTIL cL = NIL DO
     CategoryRelshipList[cL.first] ENDLOOP;
}
ELSE -- see if any new MsgSets were on categoriesList
{ curCatList: LIST OF ROPE← ELtoNL[DB.GetPList[msg, mCategoryIs]];
found: BOOL← FALSE;
FOR newL: LIST OF ROPE← categoriesList, newL.rest UNTIL newL = NIL DO
FOR cL: LIST OF ROPE← curCatList, cL.rest UNTIL cL = NIL DO
IF Rope.Equal[newL.first, cL.first, FALSE] THEN
{ found← TRUE; EXIT}; ENDLOOP;

IF ~found THEN CategoryRelshipList[newL.first];
ENDLOOP;
};
};
  };
ENDCASE => NULL;
lastMsgAddedToDB← startPos + fullLength;
IF msgRec#NIL THEN msgRec.relList← relList;
RETURN[lastMsgAddedToDB, TRUE, msgRec, ~existed];

END;

-- ********************************************************

DoAddMsgToMsgSet: PUBLIC PROC[msg: Msg, msgSet: MsgSet] =
-- Adds msg to msgSet and makes entry in log for this update.
{ []← AddTo[msg, msgSet, TRUE]};

AddTo: PUBLIC PROC[msg: Msg, msgSet: MsgSet, doLogging: BOOL]
  RETURNS[rel: Relship, existed: BOOL] =
-- Adds msg to msgSet and IF doLogging THEN makes entry in log for this update.
BEGIN
msName: ROPE← DB.GetName[msgSet];
avl: AttributeValueList← LIST[[mCategoryOf, msg], [mCategoryIs, msgSet],
   [mCategoryDate, DB.GetP[msg, mDateCodeIs]]];
rel← DeclareRelship[mCategory, avl, OldOnly];
IF existed← (rel # NIL) THEN RETURN;  -- msg is already in this msgSet
IF doLogging THEN
[]← MakeLogEntry[
insertion,
Rope.Cat["Relation: mCategory\nof: ", DB.GetName[msg], "\nis: ", msName, "\n\n"],
logStream];
rel← CreateRelship[mCategory, avl];
ChangeNumInSet[msgSet, 1];
END;

DoRemoveMsgFromMsgSet: PUBLIC PROC
 [msg: Msg, msgSet: MsgSet, rel: Relship, checkForLast: BOOL] =
-- Removes msg from msgSet and makes entry in log for this update.
{ []← RemoveFrom[msg, msgSet, rel, checkForLast, TRUE]};

RemoveFrom: PUBLIC PROC
[msg: Msg, msgSet: MsgSet, rel: Relship, checkForLast, doLogging: BOOL]
 RETURNS[newRel: Relship] =
-- Removes msg from msgSet and IF doLogging THEN makes entry in log for this update.
-- IF checkForLast & was added to Deleted, newRel will be non-NIL
BEGIN
IF rel = NIL THEN
{ rs: RelshipSet← RelationSubset[mCategory, LIST[[mCategoryOf, msg]]];
  UNTIL DB.Null[rel← NextRelship[rs]] DO
  IF Eq[msgSet, WalnutDBAccess.GetFE[rel, mCategoryIs]] THEN EXIT;
  ENDLOOP;
  ReleaseRelshipSet[rs];
};
-- If anyone is displaying one of this msg's categories, notify them
IF doLogging THEN
{ []← MakeLogEntry[
 deletion,
 Rope.Cat["Relation: mCategory\nof: ", DB.GetName[msg], "\nis: ",
  DB.GetName[msgSet], "\n\n"],
 logStream];
};
DestroyRelship[rel];  -- MEraseP[msg, mCategoryIs, msgSet];
ChangeNumInSet[msgSet, -1];
WalnutWindow.SetWalnutUpdatesPending[TRUE];
IF checkForLast THEN
IF DB.GetPList[msg, mCategoryIs] = NIL THEN
  newRel← AddTo[msg, deletedMsgSet, doLogging].rel;
END;

ChangeNumInSet: PROC[msgSet: MsgSet, inc: INT] = INLINE
BEGIN
 num: INT← DB.V2I[DB.GetP[msgSet, msNumInSetIs]] + inc;
 []← DB.SetP[msgSet, msNumInSetIs, I2V[num]];
END;

DoDestroyMsgSet: PUBLIC PROC[msgSet: Msg] =
BEGIN
 rel: Relship;
 rs: RelshipSet← RelationSubset[mCategory, LIST[[mCategoryIs, msgSet]]];
 UNTIL DB.Null[rel← NextRelship[rs]] DO
  msg: Msg← WalnutDBAccess.GetFE[rel, mCategoryOf];
  []← RemoveFrom[msg, msgSet, rel, TRUE, TRUE];
 ENDLOOP;

 ReleaseRelshipSet[rs];
 MsgSetEntryToLog[DB.GetName[msgSet], deletion];
 DestroyEntity[msgSet];
END;

DoCreateMsgSet: PUBLIC PROC[name: ROPE] RETURNS[msgSet: MsgSet] =
{ MsgSetEntryToLog[name, insertion];
msgSet← DB.CreateEntity[MsgSetDomain, name];
WalnutWindow.SetWalnutUpdatesPending[TRUE];
};

MsgSetEntryToLog: PROC[name: ROPE, which: LogEntryType] =
-- makes a log entry for a newly created MsgSet
{ []← MakeLogEntry[
  which,
  Rope.Cat["Domain: MsgSet\nname: ", name, "\n\n"],
   logStream];
};

GetMsgFromStream: PUBLIC PROC
 [strm: IO.STREAM, prefixLength, msgLength: INT, entryChar: CHAR]
  RETURNS[msgRec: MsgRec, existed: BOOL, categoriesList: LIST OF ROPE] =
BEGIN ENABLE IO.EndOfStream =>
  {Report["\nUnexpected EOF, ignoring msg entry"]; GOTO badMessage};

 prefixPos: INT← strm.GetIndex[];
 headersPos: INT← prefixPos + prefixLength - minPrefixLength;
 msgID, categories: ROPE← NIL;
 outOfSynch: BOOL;
 msg: Msg;

 [msgID, categories, outOfSynch]← ReadPrefixInfo[strm, headersPos];

 IF outOfSynch THEN GOTO badMessage;
 IF categories # NIL THEN
  BEGIN OPEN IO;
  h: Handle← CreateInputStreamFromRope[categories];
  UNTIL h.EndOf[] DO
  categoriesList← CONS[h.GetToken[WhiteSpace], categoriesList] ENDLOOP;
  h.Close[];
  END;

 msgRec← NEW[MessageRecObject];
 []← WalnutRetrieve.ParseMsgIntoFields[msgRec, strm, msgLength];
 IF (msgRec.gvID← msgID) = NIL THEN
  []← ConstructMsgID[msgRec] ELSE ParseMsgID[msgRec];
 strm.SetIndex[headersPos+msgLength];  -- end of msg

 IF entryChar # '? THEN msgRec.hasBeenRead← TRUE;
 [msg, existed]← DoMsgRecToMsg[msgRec];
-- set body pointers to positions in file
 IF ~existed THEN
  { []← DB.SetP[msg, mPrefixPos, I2V[prefixPos]];
  []← DB.SetP[msg, mMsgLengthIs, I2V[msgLength]];
  []← DB.SetP[msg, mHeadersPos, I2V[headersPos]];
  msgRec.headersPos← headersPos;
  msgRec.msgLength← msgLength;
  };
  EXITS
  badMessage => RETURN[msgRec, TRUE, NIL];  -- pretend msg already existed
END;

ExpungeMsgs: PUBLIC PROC[tempLog: IO.STREAM, doUpdates, tailRewrite: BOOL]
  RETURNS[ok: BOOL] =
-- Dumping is driven from the log file & preserves the bits that came from Grapevine
-- Dumps all the Msgs in the database to a tempLog, setting all
-- their body pointers to reference the NEW log, then copies tempLog to logStream
BEGIN OPEN IO;
newLen: INT;
BEGIN ENABLE IO.EndOfStream => GOTO badLogFile;
msg: Msg;

toBeDestroyedList: LIST OF Msg;
count: INT← 0;
expungeFilePos, startPos, prefixLength, entryLength: INT;
startExpungePos: INT;
entryChar: CHAR;
numBad: INTEGER← 0;
DoCount: PROC =
{IF (count← count + 1) MOD 10 = 0 THEN
 ReportRope[IF count MOD 100 = 0 THEN "!" ELSE "~"]
};
WriteBadLogEntry: PROC[startPos: INT] RETURNS[nextStartPos: INT] =
 { errorStream: STREAM← FileIO.Open["WalnutExpunge.errlog", write];
  line: ROPE;
  IF (numBad← numBad+1) = 1 THEN errorStream.SetIndex[0]
  ELSE errorStream.SetIndex[errorStream.GetLength[]];
  logStream.SetIndex[startPos];
  errorStream.PutRope[
   IO.PutFR["\n Bad entry from pos %g of log file, written at %g\n", int[startPos], time[]]];
  errorStream.PutRope[logStream.GetSequence[]]; -- *start* line
  errorStream.PutRope["\n"];
  DO
  nextStartPos← logStream.GetIndex[];
  line← logStream.GetSequence[];
  IF line.Equal["*start*"] THEN {nextStartPos← logStream.GetIndex[]-8; EXIT};
  errorStream.PutRope[line];
  errorStream.PutRope["\n"];
  ENDLOOP;
  errorStream.SetLength[errorStream.GetIndex[]];
  errorStream.Close[];
};
 CopyEntryIfNecessary: PROC =
 { thisTag, thisValue: ROPE← NIL;
  thisMsg: Msg;
  doCopy: BOOL← TRUE;
  prefixPos: INT;
  IF entryChar = '← THEN  -- hasbeenread entry
   { -- logStream.SetIndex[startPos+minPrefixLength]; where logStream is pos'd
   thisValue← RopeIO.GetRope[logStream, prefixLength-minPrefixLength-1];
   []← logStream.GetChar[ ! IO.EndOfStream => CONTINUE]
   }
   ELSE
  { logStream.SetIndex[startPos+prefixLength];
   [thisTag, thisValue]← TagAndValue[logStream, 2];
   IF thisTag.Equal["Relation"] AND thisValue.Equal["mCategory"] THEN
   { [thisTag, thisValue]← TagAndValue[logStream, 2];
   IF ~thisTag.Equal["of"] THEN thisValue← NIL;
   };
   };
  IF thisValue # NIL THEN
   { thisMsg← DeclareEntity[MsgDomain, thisValue, OldOnly];
   IF ~DB.Null[thisMsg] THEN
   IF (prefixPos← V2I[DB.GetP[thisMsg, mPrefixPos]]) >= startExpungePos THEN
     doCopy← FALSE;
   };
  IF doCopy THEN
   { logStream.SetIndex[startPos];
tempLog.PutRope[RopeIO.GetRope[logStream, entryLength]];
   };
 };

tempLog.SetIndex[0];
startExpungePos←
 IF tailRewrite THEN V2I[DB.GetF[walnutInfoRelship, wStartExpungePos]] ELSE 0;
logStream.SetIndex[startExpungePos];

DO   -- loop for dumping messages & making other log entries
msgID: ROPE← NIL;
fullText: ROPE← NIL;
headersPos, curPos: INT;
badEntry: BOOL← FALSE;

[startPos, prefixLength, entryLength, entryChar]← ReadStartOfMsg[logStream, TRUE];
IF entryLength = 0 THEN EXIT;
IF (entryChar # ' ) AND (entryChar # '?) THEN -- prehaps copy other entry to tempLog
{ IF startExpungePos # 0 THEN CopyEntryIfNecessary[];
logStream.SetIndex[startPos+entryLength]; -- for good measure, even after GetRope
LOOP
};

headersPos← startPos+prefixLength;
UNTIL (curPos← logStream.GetIndex[]) = headersPos DO
-- look for a msgID: field
  tag, value: ROPE;

[tag, value]← TagAndValue[h: logStream, inc: 2];
IF Rope.Equal[tag, msgIDRope, FALSE] THEN msgID← value;
IF curPos > headersPos THEN -- bad entry we don't understand
 { nextStartPos: INT← WriteBadLogEntry[startPos];
  cMsg: ROPE← " Confirm to push on, deny to stop the Expunge";
  ReportRope[PutFR["\nBad message entry at log pos %g;", int[startPos]]];
  Report[" entry written on WalnutExpunge.errlog"];
  Report[cMsg];
  IF (ok← UserConfirmed[])
    THEN {logStream.SetIndex[nextStartPos]; badEntry← TRUE; EXIT};
  RETURN[FALSE]};
ENDLOOP;
 IF badEntry THEN LOOP;

IF msgID = NIL THEN
 BEGIN  -- have to read fullText to fashion msgID
mr: MsgRec← NEW[MessageRecObject];
[]← WalnutRetrieve.ParseMsgIntoFields[mr, logStream, entryLength-prefixLength !
 IO.EndOfStream => {Report[skipRope]; logStream.SetIndex[startPos+entryLength]; LOOP}];
msgID← ConstructMsgID[mr];
logStream.SetIndex[startPos+prefixLength];  -- back up stream
 END;

msg← DoNameToEntity[d: MsgDomain, name: msgID, oldOnly: TRUE];
IF (msg=NIL) OR DB.Null[msg] THEN {logStream.SetIndex[startPos+entryLength]; LOOP}
ELSE
BEGIN
curPrefixPos: INT;
thisMsgSet: MsgSet;
nameOfCatsList: ROPE← NIL;

-- if is only in deletedMsgSet, don't put on tempLog
  FOR mL: LIST OF DB.Value← DB.GetPList[msg, mCategoryIs], mL.rest UNTIL mL=NIL DO
  IF ~Eq[thisMsgSet← V2E[mL.first], deletedMsgSet] THEN
  nameOfCatsList← Rope.Cat[nameOfCatsList, " ", DB.GetName[thisMsgSet]];
  ENDLOOP;

  IF nameOfCatsList.Length[] = 0 THEN
  { toBeDestroyedList← CONS[msg, toBeDestroyedList]; GOTO doneWithOne};
  msgID← Rope.Cat[msgID, "\nCategories:", nameOfCatsList];

IF fullText = NIL THEN fullText←
RopeFromStream[logStream, entryLength-prefixLength ! IO.EndOfStream =>
{Report[skipRope]; GOTO doneWithOne}];

DoCount[];
expungeFilePos←
MakeLogEntry[IF V2B[DB.GetP[msg, mHasBeenReadIs]] THEN message ELSE newMessage,
   fullText, tempLog, Rope.Cat[msgIDRope, ": ", msgID, "\n"]];
-- set header & body pos pointers to posn in NEW log if doing updates:
 IF doUpdates THEN
  { thisPrefixPos: INT← startExpungePos + expungeFilePos - entryLength;
  curPrefixPos← V2I[DB.GetP[msg, mPrefixPos]];
  IF curPrefixPos # thisPrefixPos THEN
  { []← DB.SetP[msg, mHeadersPos, I2V[thisPrefixPos+prefixLength]];
   []← DB.SetP[msg, mPrefixPos, I2V[thisPrefixPos]];
  };
  };


 EXITS
  doneWithOne => logStream.SetIndex[startPos+entryLength];
END;  -- put message on log file
ENDLOOP;  -- loop for dumping messages

Report[PutFR["\nDumped %g messages from Log File... ", int[count]]];
tempLog.SetLength[newLen← tempLog.GetIndex[]];
tempLog.Flush[];  -- make sure tempLog is in good shape
Report[PutFR["Old log length was %g bytes", int[logStream.GetLength[]]]];
IF tailRewrite THEN
{ Report[PutFR
  ["Previous end of log was %g bytes", int[logStream.GetLength[]-startExpungePos]]];
Report[PutFR["New end of log file is %g bytes (%g messages)", int[newLen], int[count]]];
Report[PutFR["New log length is %g bytes", int[startExpungePos+newLen]]];
}
ELSE
{ Report[PutFR["Old log length was %g bytes", int[logStream.GetLength[]]]];
Report[PutFR["New log length is %g bytes (%g messages)", int[newLen], int[count]]]
};

IF doUpdates THEN
 { ReportRope[" Destroying Msgs in Deleted MsgSet ..."];
-- may see a msg more than once in the log file, hence must check if already destroyed
  count← 0;
  FOR mL: LIST OF Msg← toBeDestroyedList, mL.rest UNTIL mL = NIL DO
  IF ~DB.Null[mL.first] THEN
   { DB.DestroyEntity[mL.first];  -- destroy the message
   DoCount[];
   };
  ENDLOOP;
  Report[PutFR["\n%g Msgs destroyed",int[count]]];
 BEGIN   -- fix up num in deleted (should be 0 but who knows)
  num: INT← DB.V2I[DB.GetP[deletedMsgSet, msNumInSetIs]] - count;
  []← DB.SetP[deletedMsgSet, msNumInSetIs, I2V[num]];
 END;

Report["Copying tempLog to log file"];
DB.SetF[walnutInfoRelship, wCopyInProgress, B2V[TRUE]];
DB.SetF[walnutInfoRelship, wStartExpungePos, I2V[startExpungePos]];
DB.MarkTransaction[DB.TransactionOf[$Walnut]];

CopyTempLogToLog[tempLog, startExpungePos];

-- lastMsgAddedToDB← lastMsgWrittenToLog← logStream.GetIndex[];
-- DB.SetF[walnutInfoRelship, wCopyInProgress, B2V[FALSE]];
-- DB.SetF[walnutInfoRelship, wStartExpungePos, I2V[lastMsgAddedToDB]];
};  -- end doUpdates

RETURN[TRUE];

EXITS
badLogFile =>
{ tempLog.SetLength[tempLog.GetIndex[]];
tempLog.Close[];
RETURN[FALSE]
};
END;
END;

ConstructMsgID: PUBLIC PROC[mr: MsgRec] RETURNS[ROPE] =
-- uses gv address of Cabernet as default
-- if date is Unintelligible => use Time.Current
BEGIN
 ts: GVBasics.Timestamp← [net: 3, host: 14, time: 0];
 IF mr.gvSender = NIL THEN
  { IF mr.inMsgSender # NIL THEN mr.gvSender← mr.inMsgSender
  ELSE IF mr.from = NIL THEN mr.gvSender← "UnknownSender"
  ELSE mr.gvSender← mr.from};
 IF mr.date = NIL THEN mr.date← IO.PutFR[NIL, IO.time[]];  -- current time
 BEGIN
  ts.time← DateAndTime.Parse[mr.date ! DateAndTime.Unintelligible => GOTO badTime].dt;
  TRUSTED {mr.dateCode← LOOPHOLE[ts.time, System.GreenwichMeanTime]};
  EXITS
  badTime => TRUSTED {mr.dateCode← Time.Current[]};
 END;
 
-- mr.gvID← Rope.Cat[mr.gvSender, " $ ", ~~GVBasics.RopeFromTimestamp[ts]];

 mr.gvID← Rope.Cat[mr.gvSender, " $ ", RopeFromTimestamp[ts] ];
 RETURN[mr.gvID];
END;

ParseMsgID: PROC[mr: MsgRec] =
-- parses gvSender out of mr.gvID, compute dateCode
BEGIN
 local: ROPE← mr.gvID;
 mr.gvSender← local.Substr[0, local.Find[" $ "]];
 mr.dateCode← DateAndTime.Parse[mr.date ! DateAndTime.Unintelligible => GOTO badTime].dt;
EXITS
 badTime => TRUSTED {mr.dateCode← Time.Current[]};
END;

RopeFromTimestamp: PUBLIC PROC[ts: GVBasics.Timestamp] RETURNS [ROPE] =
BEGIN OPEN IO;
RETURN[PutFR["%g#%g@%g",
 int[ts.net], int[ts.host], time[LOOPHOLE[ts.time, System.GreenwichMeanTime]] ]];
END;

AddLogEntriesToDB: PUBLIC PROC [startPos: INT] RETURNS[success: BOOL] =
{ success← AddLogEntries[logStream, startPos];
IF success THEN lastMsgAddedToDB← logStream.GetIndex[];
};

CopyTempLogToLog: PROC[tempLog: IO.Handle, logStreamStartPos: INT] =
-- copy tempLog to logStream
BEGIN
bytes: NAT;
logStream.SetIndex[logStreamStartPos]; tempLog.SetIndex[0];
DO
IF (bytes← tempLog.GetBlock[copyBuffer]) = 0 THEN EXIT;
logStream.PutBlock[copyBuffer];
ENDLOOP;
lastMsgAddedToDB← lastMsgWrittenToLog← logStream.GetIndex[];
logStream.SetLength[lastMsgAddedToDB]; -- set the log length to current length
logStream.Flush[];

tempLog.Close[]; -- we don't need this guy any more, we've copied him to log.

DB.SetF[walnutInfoRelship, wCopyInProgress, B2V[FALSE]];
DB.SetF[walnutInfoRelship, wStartExpungePos, I2V[lastMsgAddedToDB]];
END;

END.

Change Log.

By Willie-Sue on February 14, 1983
-- added tailRewrite and doUpdates code to ExpungeMsgs