-- File: WalnutDBLogImpl.mesa
-- Contents:
  -- types and procedures for writing on the log file

-- Created by: Willie-Sue on July 16, 1982
-- Last edited by: 
  -- Willie-Sue Hoo-Hah on May 3, 1983 9:10 am
  -- Rick Cattell on XXX
--  MBrown, December 15, 1982 11:42 am

DIRECTORY
    DateAndTime USING[Parse, Unintelligible],
    GVBasics USING[Timestamp],
    GVRetrieve USING [Handle],
    File USING [Capability],
    FileIO USING[CapabilityFromStream, commitAndReopenTransOnFlush, finishTransOnClose,
            Open, StreamFromCapability],
    IO,
    RopeIO USING [GetRope],
    Rope,
    System USING [GreenwichMeanTime],
    Time USING [Current],
    DB,
    UserCredentials USING [GetUserCredentials],
    ViewerTools USING [TiogaContentsRec],
    WalnutDB,
    WalnutDBAccess,
    WalnutDBLock,
    WalnutDBLog,
    WalnutDisplayerPrivate,
    WalnutRetrieve,
    WalnutSendMail,
    WalnutWindow;
    
WalnutDBLogImpl: CEDAR PROGRAM
  IMPORTS DateAndTime, FileIO, IO, RopeIO, Rope, Time, UserCredentials,
          DB, WalnutDB, WalnutDBAccess, WalnutDBLock, WalnutDBLog,
          WalnutDisplayerPrivate, WalnutRetrieve, WalnutSendMail, WalnutWindow
  EXPORTS WalnutDB, WalnutDBLog =
        
BEGIN OPEN DB, WalnutDB, WalnutDBAccess, WalnutDBLog, WalnutWindow;

ROPE: TYPE = Rope.ROPE;

msgIDRope: PUBLIC ROPE_ "gvMsgID" ;
categoriesRope: PUBLIC ROPE_ "Categories";

bufferSize: CARDINAL = 512;
copyBuffer: PUBLIC REF TEXT _ NEW[TEXT[bufferSize]];        -- one to share

-- ********************************************************
logStream: IO.STREAM_ NIL;

lastMsgAddedToDB: INT; -- pos in log after last msg entered in database
lastMsgWrittenToLog: INT;        -- pos in log at end of last msg retrieved
skipRope: ROPE_ "\nUnexpected EOF, skipping ahead.";

-- ********************************************************

DoInitializeLog: PUBLIC PROC RETURNS [curLength: INT] =
  BEGIN
  IF logStream # NIL THEN RETURN[logStream.GetLength[]];        -- already open

  IF DB.TransactionOf[$Walnut] = NIL THEN OpenWTransaction[NIL, FALSE];
  logStream_ FileIO.Open[fileName: WalnutWindow.walnutLogName,
          accessOptions: write,
          closeOptions: FileIO.commitAndReopenTransOnFlush+FileIO.finishTransOnClose];
  RETURN[lastMsgWrittenToLog_ logStream.GetLength[]]
  END;

DoCloseLogStream: PUBLIC PROC =
  { IF logStream#NIL THEN logStream.Close[]; logStream_ NIL};

FinishExpunge: PUBLIC PROC =
--  called if wCopyInProgress is TRUE during startup
  BEGIN
    startPos: INT_ V2I[DB.GetF[walnutInfoRelship, wStartExpungePos]];
    tempLog: IO.Handle_ FileIO.Open["Walnut.TempLog"];
    CopyTempLogToLog[tempLog, startPos];
    DB.MarkTransaction[DB.TransactionOf[$Walnut]];
  END;

--  must know when the state of Walnut's segment is changing

OpenWTransaction: PUBLIC PROC[trans: DB.Transaction, noLog: BOOL] =
  BEGIN
    fileNotFound, transOpen: BOOL_ FALSE;
    IF walnut#NIL THEN ReportRope[" Opening Walnut transaction ..."];
    DB.OpenTransaction[
     segment: $Walnut,
     userName: WalnutSendMail.userRName,
     password: UserCredentials.GetUserCredentials[].password,
     useTrans: trans,
     noLog: noLog  ! DB.Error =>
       IF code=FileNotFound THEN {fileNotFound_ TRUE; CONTINUE}
        ELSE IF code=TransactionAlreadyOpen THEN {transOpen_ TRUE; CONTINUE}];

    IF fileNotFound OR transOpen THEN
     { DB.CloseTransaction[DB.TransactionOf[$Walnut]];
        IF fileNotFound THEN DB.DeclareSegment[walnutSegmentFile, $Walnut,,, NewOnly];
        DB.OpenTransaction[
              segment: $Walnut,
              userName: WalnutSendMail.userRName,
              password: UserCredentials.GetUserCredentials[].password,
              useTrans: trans,
              noLog: noLog];
        IF fileNotFound THEN
            DB.MarkTransaction[DB.TransactionOf[$Walnut]]; -- make sure segment initializated
     };

    WalnutDBAccess.DoInitializeDBVars[];
    lastMsgAddedToDB_ V2I[DB.GetF[walnutInfoRelship, wExpectedDBPos]];
    WalnutWindow.UpdateMsgSetButtonEntities[];
  END;

CloseWTransaction: PUBLIC PROC =
  BEGIN
    len: INT;
    ReportRope[" Closing Walnut transaction ..."];
    IF logStream#NIL THEN
     { len_ logStream.GetLength[];
        logStream.Flush[];
        IF lastMsgAddedToDB >= lastMsgWrittenToLog THEN lastMsgAddedToDB_ len;
        DB.SetF[walnutInfoRelship, wExpectedLength, I2V[len]];
        DB.SetF[walnutInfoRelship, wExpectedDBPos, I2V[lastMsgAddedToDB]];
        DB.CloseTransaction[DB.TransactionOf[$Walnut]];
     };
    WalnutWindow.SetWalnutUpdatesPending[FALSE];
    Report[" ... done"];
  END;

AbortWTransaction: PUBLIC PROC =
  BEGIN
    ReportRope[" Aborting Walnut transaction ..."];
    DB.AbortTransaction[TransactionOf[$Walnut]];
    WalnutDBAccess.DoInitializeDBVars[];
     lastMsgAddedToDB_ V2I[GetF[walnutInfoRelship, wExpectedDBPos]];
    lastMsgWrittenToLog_ logStream.GetLength[];
--    WalnutWindow.UpdateMsgSetButtonEntities[];
    Report[" ... done"];
  END;

MarkWTransaction: PUBLIC PROC =
  BEGIN 
    len: INT_ logStream.GetLength[];
    logCapability: File.Capability;
    IF WalnutWindow.logFileIsPilotFile THEN
            logCapability_ FileIO.CapabilityFromStream[logStream];
--  close and reopen logFile to get create date set
    logStream.Close[];
    IF lastMsgAddedToDB >= lastMsgWrittenToLog THEN lastMsgAddedToDB_ len;
    ReportRope[" Saving Walnut updates ..."];
    DB.SetF[walnutInfoRelship, wExpectedLength, I2V[len]];
    DB.SetF[walnutInfoRelship, wExpectedDBPos, I2V[lastMsgAddedToDB]];
    DB.MarkTransaction[DB.TransactionOf[$Walnut]];
    IF WalnutWindow.logFileIsPilotFile THEN 
     logStream_ FileIO.StreamFromCapability[
         capability: logCapability,
          accessOptions: write,
          closeOptions: FileIO.commitAndReopenTransOnFlush+FileIO.finishTransOnClose,
          fileName: WalnutWindow.walnutLogName]
     ELSE 
      logStream_ FileIO.Open[
         fileName: WalnutWindow.walnutLogName,
         accessOptions: write,
         closeOptions: FileIO.commitAndReopenTransOnFlush+FileIO.finishTransOnClose];

    WalnutWindow.SetWalnutUpdatesPending[FALSE];
    Report[" ... done"];
  END;

LogLength: PUBLIC PROC[doFlush: BOOL] RETURNS[INT] =
 {IF doFlush THEN logStream.Flush[]; RETURN[logStream.GetLength[]]};

DoArchiveMsgSet: PUBLIC PROC[msgSet: MsgSet, strm: IO.STREAM, doDelete: BOOL_ FALSE]
    RETURNS[num: INT] =
  --  writes a log style file for msgSet on strm
  BEGIN
    rel, newRel: Relship;
    isDeletedDisplayed: BOOL_ (FindMSViewer[deletedMsgSet] # NIL);
    rs: RelshipSet_ RelationSubset[mCategory, LIST[[mCategoryIs, msgSet]]];
    restOfPrefix: ROPE_ Rope.Cat["\nCategories: ", DB.GetName[msgSet], "\n"];
    num_ 0;
    UNTIL DB.Null[rel_ NextRelship[rs]] DO
      msg: Msg_ WalnutDBAccess.GetFE[rel, mCategoryOf];
      startPos: INT_ V2I[DB.GetP[msg, mHeadersPos]];
      length: INT_ V2I[DB.GetP[msg, mMsgLengthIs]];
      fullText: ROPE_ MsgRopeFromLog[startPos, length];
      prefix: ROPE_ Rope.Cat[msgIDRope, ": ", DB.GetName[msg], restOfPrefix];
      []_ MakeLogEntry[message, fullText, strm, prefix];
      num_ num + 1;
      IF doDelete THEN
       { newRel_ RemoveFrom[msg, msgSet, rel, TRUE, TRUE];
             IF isDeletedDisplayed AND (newRel # NIL) THEN
                     WalnutDisplayerPrivate.AddMsgToMsgSetDisplayer[msg, deletedMsgSet, newRel];
           };
    ENDLOOP;
    ReleaseRelshipSet[rs];
  END;

GVMessageToLog: PUBLIC PROC
  [gvH: GVRetrieve.Handle, timeStamp: GVBasics.Timestamp, gvSender: RName]
      RETURNS[ok: BOOL] =
--  reads message items from Grapevine & makes a log entry for this message
  { prefix: ROPE_
      Rope.Cat[msgIDRope, ": ", gvSender, " $ ", RopeFromTimestamp[timeStamp],
              "\nCategories: Active\n"];
    [lastMsgWrittenToLog, ok]_ GVLogEntry[gvH, logStream, prefix];
   };

DoAddMessageToLog: PUBLIC PROC [entryText, prefix: ROPE] =
--  makes a message LogEntryType on log (used by old mail reader)
  { lastMsgWrittenToLog_ MakeLogEntry[message, entryText, logStream, prefix]};

WriteMsgReadEntryToLog: PUBLIC PROC[msg: Msg] =
  -- changes mHasBeenReadIs prop to TRUE & makes log entry
 { []_ MakeLogEntry[hasbeenread, NIL, logStream, Rope.Concat[DB.GetName[msg], "\n"]];
   []_ DB.SetP[msg, mHasBeenReadIs, B2V[TRUE]];
   WalnutWindow.SetWalnutUpdatesPending[TRUE];
 };

GetTiogaMsgFromLog: PUBLIC PROC[msg: Msg]
        RETURNS[contents: TiogaContents, startPos, length: INT] =
  BEGIN
    startPos_ V2I[DB.GetP[msg, mHeadersPos]];
    length_ V2I[DB.GetP[msg, mMsgLengthIs]];
    contents_ MsgFromLog[startPos, length];
 END;

MsgFromLog: PUBLIC PROC[startPos, length: INT] RETURNS[contents: TiogaContents] =
  BEGIN
    contents_
        WalnutSendMail.TiogaTextFromStrm[logStream, startPos-1, length+1];     -- hack for now

    IF contents # NIL THEN
     { IF contents.formatting=NIL THEN
            contents.contents_ Rope.Substr[contents.contents, 1];
       RETURN;
     };
    contents_ NEW[ViewerTools.TiogaContentsRec];
    contents.contents_
             IO.PutFR["[message body lost!!! (%g,%g)]\n", IO.int[startPos], IO.int[length]];
  END;

MsgRopeFromLog: PUBLIC PROC[startPos, length: INT] RETURNS[ROPE] =
  BEGIN ENABLE IO.EndOfStream => GOTO TooShort;

    logStream.SetIndex[startPos];
    RETURN[RopeIO.GetRope[logStream, length]];

    EXITS TooShort =>
     RETURN[IO.PutFR["[message body lost!!! (%g,%g)]\n", IO.int[startPos], IO.int[length]]];
  END;


-- ********************************************************
--  operations that read the log

ScavengeFromLog: PUBLIC PROC RETURNS[success: BOOL] =
  -- Erase old segment and create new one from the log file
  BEGIN
  []_ InitializeLog[];        -- this does InitializeDBVars[]

  IF success_ ReadLogFile[0] THEN
   { Report[" WalnutScavenge done"];
      lastMsgAddedToDB_ lastMsgWrittenToLog_ logStream.GetLength[];
      DB.SetF[walnutInfoRelship, wExpectedLength, I2V[lastMsgWrittenToLog]];
      DB.SetF[walnutInfoRelship, wExpectedDBPos, I2V[lastMsgAddedToDB]]}
   ELSE Report["\nThere were problems doing the scavenge."];

   logStream.Close[];
   logStream_ NIL;
  END;

ProcessOldMessageFile: PUBLIC PROC[strm: IO.STREAM, defaultPrefix: ROPE]
        RETURNS [ok: BOOL] =
  { RETURN[OldMessageFile[strm, defaultPrefix]]};

GetMessagesFromLog: PUBLIC PROC[startOfNewMessages: INT]
        RETURNS[curLength, numNewMsgs: INT] =
  BEGIN
   anymore, msgWasNew: BOOL;
   endPos: INT;
   startPos: INT_ startOfNewMessages;
   msV: Viewer_ NIL;
   prevMsgSet: MsgSet_ NIL;
   msgRec: MsgRec;

   numNewMsgs_ 0;

    DO
     [endPos, anymore, msgRec, msgWasNew]_ WalnutDBLock.GetOneMessageFromLog[startPos];
     IF ~anymore THEN EXIT;
     IF msgRec = NIL THEN {startPos_ endPos; LOOP};
     IF msgWasNew THEN numNewMsgs_ numNewMsgs + 1;
      FOR rL: LIST OF RelshipMsgSetPair_ msgRec.relList, rL.rest UNTIL rL = NIL DO
        IF ~EqEntities[rL.first.msgSet, prevMsgSet] THEN
            msV_ FindMSViewer[prevMsgSet_ rL.first.msgSet];
        IF msV#NIL THEN
            WalnutDisplayerPrivate.AddParsedMsgToMSViewer[msgRec, msV, rL.first.rel];
      ENDLOOP;
     startPos_ endPos;
   ENDLOOP;

   RETURN[endPos, numNewMsgs];
  END;

OneMsgFromLog: PUBLIC PROC[startOfNextMessage: INT]
        RETURNS[endPos: INT, anymore: BOOL, msgRec: MsgRec, msgWasNew: BOOL] =
  BEGIN
    existed: BOOL;
   startPos, fullLength, prefixLength, prefixInfoLength: INT;
   entryChar: CHAR;
   msg: Msg;
   categoriesList: LIST OF ROPE;
   relList: LIST OF RelshipMsgSetPair_ NIL;

   CategoryRelshipList: PROC[msgSetName: ROPE] =
    BEGIN
    msgSet: MsgSet;
    existed: BOOL;
    rel: Relship;
    avl: AttributeValueList;

    [msgSet, existed]_ WalnutDBAccess.DoDeclareMsgSet[msgSetName];

    avl_ LIST[[mCategoryOf, msg], [mCategoryIs, msgSet],
                   [mCategoryDate, DB.GetP[msg, mDateCodeIs]]];
     rel_ DeclareRelship[mCategory, avl, OldOnly];
     IF rel = NIL THEN
      { relList_ CONS[[CreateRelship[mCategory, avl], msgSet], relList];
        ChangeNumInSet[msgSet, 1];
       };
     IF ~existed THEN AddToMsgSetButtons[msgSet, msgSetName];
     END;

     ELtoNL: PROC[el: LIST OF Value] RETURNS [nl: LIST OF RName]  =
      BEGIN
      IF el=NIL THEN RETURN[NIL]
       ELSE RETURN[CONS[V2S[DB.GetName[V2E[el.first]]], ELtoNL[el.rest]]];
      END;

    logStream.SetIndex[startOfNextMessage];
-- Read *start* from log
    [startPos, prefixLength, fullLength, entryChar]_ ReadStartOfMsg[logStream, TRUE];
    IF (fullLength = 0) OR (fullLength = -1) THEN
     { lastMsgAddedToDB_ logStream.GetIndex[];
        RETURN[lastMsgAddedToDB, FALSE, NIL, FALSE];
      };

    prefixInfoLength_ prefixLength - minPrefixLength;
--  ignore all log entries except those with entryChar = '? or SP
--  any other entries have been processed
    SELECT entryChar FROM
     '?, '  => -- msg entry
         { [msgRec, existed, categoriesList]_
                 GetMsgFromStream[logStream, prefixLength, fullLength-prefixLength, entryChar];
          IF (msg_ msgRec.msg) # NIL THEN
           { IF ~existed THEN        -- add to msgSets
             { IF categoriesList = NIL THEN CategoryRelshipList["Active"]
                ELSE
                 FOR cL: LIST OF ROPE_ categoriesList, cL.rest UNTIL cL = NIL DO
                  CategoryRelshipList[cL.first] ENDLOOP;
             }
            ELSE    -- see if any new MsgSets were on categoriesList
            { curCatList: LIST OF ROPE_ ELtoNL[DB.GetPList[msg, mCategoryIs]];
              found: BOOL_ FALSE;
              FOR newL: LIST OF ROPE_ categoriesList, newL.rest UNTIL newL = NIL DO
               FOR cL: LIST OF ROPE_ curCatList, cL.rest UNTIL cL = NIL DO
                IF Rope.Equal[newL.first, cL.first, FALSE] THEN
                { found_ TRUE; EXIT}; ENDLOOP;

               IF ~found THEN CategoryRelshipList[newL.first];
              ENDLOOP;
            };
          };
        };
    ENDCASE =>  NULL;
    lastMsgAddedToDB_ startPos + fullLength;
    IF msgRec#NIL THEN msgRec.relList_ relList;
    RETURN[lastMsgAddedToDB, TRUE, msgRec, ~existed];

  END;

-- ********************************************************

DoAddMsgToMsgSet: PUBLIC PROC[msg: Msg, msgSet: MsgSet] =
  -- Adds msg to msgSet and makes entry in log for this update.
  { []_ AddTo[msg, msgSet, TRUE]};

AddTo: PUBLIC PROC[msg: Msg, msgSet: MsgSet, doLogging: BOOL]
        RETURNS[rel: Relship, existed: BOOL] =
  -- Adds msg to msgSet and IF doLogging THEN makes entry in log for this update.
  BEGIN
  msName: ROPE_ DB.GetName[msgSet];
  avl: AttributeValueList_ LIST[[mCategoryOf, msg], [mCategoryIs, msgSet],
            [mCategoryDate, DB.GetP[msg, mDateCodeIs]]];
  rel_ DeclareRelship[mCategory, avl, OldOnly];
  IF existed_ (rel # NIL) THEN RETURN;        -- msg is already in this msgSet
  IF doLogging THEN
   []_ MakeLogEntry[
     insertion,
     Rope.Cat["Relation: mCategory\nof: ", DB.GetName[msg], "\nis: ", msName, "\n\n"],
     logStream];
  rel_ CreateRelship[mCategory, avl];
  ChangeNumInSet[msgSet, 1];
  END;

DoRemoveMsgFromMsgSet: PUBLIC PROC
    [msg: Msg, msgSet: MsgSet, rel: Relship, checkForLast:  BOOL] =
  -- Removes msg from msgSet and makes entry in log for this update.
   { []_ RemoveFrom[msg, msgSet, rel, checkForLast, TRUE]};

RemoveFrom: PUBLIC PROC
   [msg: Msg, msgSet: MsgSet, rel: Relship, checkForLast, doLogging: BOOL]
       RETURNS[newRel: Relship] =
  -- Removes msg from msgSet and IF doLogging THEN makes entry in log for this update.
  --  IF checkForLast & was added to Deleted, newRel will be non-NIL
  BEGIN
  IF rel = NIL THEN
   { rs: RelshipSet_ RelationSubset[mCategory, LIST[[mCategoryOf, msg]]];
     UNTIL DB.Null[rel_ NextRelship[rs]] DO
      IF Eq[msgSet, WalnutDBAccess.GetFE[rel, mCategoryIs]] THEN EXIT;
     ENDLOOP;
     ReleaseRelshipSet[rs];
   };
 -- If anyone is displaying one of this msg's categories, notify them
  IF doLogging THEN
   { []_ MakeLogEntry[
         deletion,
         Rope.Cat["Relation: mCategory\nof: ", DB.GetName[msg], "\nis: ",
             DB.GetName[msgSet], "\n\n"],
         logStream];
    };
  DestroyRelship[rel];        --  MEraseP[msg, mCategoryIs, msgSet];
  ChangeNumInSet[msgSet, -1];
  WalnutWindow.SetWalnutUpdatesPending[TRUE];
  IF checkForLast THEN 
   IF DB.GetPList[msg, mCategoryIs] = NIL THEN
           newRel_ AddTo[msg, deletedMsgSet, doLogging].rel;
  END; 

ChangeNumInSet: PROC[msgSet: MsgSet, inc: INT] = INLINE
  BEGIN
    num: INT_ DB.V2I[DB.GetP[msgSet, msNumInSetIs]] + inc;
    []_ DB.SetP[msgSet, msNumInSetIs, I2V[num]];
  END;

DoDestroyMsgSet: PUBLIC PROC[msgSet: Msg] =
  BEGIN
    rel: Relship;
    rs: RelshipSet_ RelationSubset[mCategory, LIST[[mCategoryIs, msgSet]]];
    UNTIL DB.Null[rel_ NextRelship[rs]] DO
      msg: Msg_ WalnutDBAccess.GetFE[rel, mCategoryOf];
      []_ RemoveFrom[msg, msgSet, rel, TRUE, TRUE];
    ENDLOOP;

    ReleaseRelshipSet[rs];
    MsgSetEntryToLog[DB.GetName[msgSet], deletion];
    DestroyEntity[msgSet];
  END;

DoCreateMsgSet: PUBLIC PROC[name: ROPE] RETURNS[msgSet: MsgSet] =
  { MsgSetEntryToLog[name, insertion];
     msgSet_ DB.CreateEntity[MsgSetDomain, name];
     WalnutWindow.SetWalnutUpdatesPending[TRUE];
   };

MsgSetEntryToLog: PROC[name: ROPE, which: LogEntryType] =
  -- makes a log entry for a newly created MsgSet
  {  []_ MakeLogEntry[
          which,
          Rope.Cat["Domain: MsgSet\nname: ",  name, "\n\n"],
             logStream];
  };

GetMsgFromStream: PUBLIC PROC
    [strm: IO.STREAM, prefixLength, msgLength: INT, entryChar: CHAR]
        RETURNS[msgRec: MsgRec, existed: BOOL, categoriesList: LIST OF ROPE] =
  BEGIN ENABLE IO.EndOfStream =>
     {Report["\nUnexpected EOF, ignoring msg entry"]; GOTO badMessage};

    prefixPos: INT_ strm.GetIndex[];
    headersPos: INT_ prefixPos + prefixLength - minPrefixLength;
    msgID, categories: ROPE_ NIL;
    outOfSynch: BOOL;
    msg: Msg;

    [msgID, categories, outOfSynch]_ ReadPrefixInfo[strm, headersPos];

    IF outOfSynch THEN GOTO badMessage;
    IF categories # NIL THEN
     BEGIN OPEN IO;
      h: Handle_ CreateInputStreamFromRope[categories];
      UNTIL h.EndOf[] DO
       categoriesList_ CONS[h.GetToken[WhiteSpace], categoriesList] ENDLOOP;
      h.Close[];
     END;

    msgRec_ NEW[MessageRecObject];
    []_ WalnutRetrieve.ParseMsgIntoFields[msgRec, strm, msgLength];
    IF (msgRec.gvID_ msgID) = NIL THEN
     []_ ConstructMsgID[msgRec] ELSE ParseMsgID[msgRec];
    strm.SetIndex[headersPos+msgLength];        -- end of msg

    IF entryChar # '? THEN msgRec.hasBeenRead_ TRUE;
    [msg, existed]_ DoMsgRecToMsg[msgRec];
-- set body pointers to positions in file 
    IF ~existed THEN
     { []_ DB.SetP[msg, mPrefixPos, I2V[prefixPos]];
       []_ DB.SetP[msg, mMsgLengthIs, I2V[msgLength]];
       []_ DB.SetP[msg, mHeadersPos, I2V[headersPos]];
       msgRec.headersPos_ headersPos;
       msgRec.msgLength_ msgLength;
     };
     EXITS
      badMessage => RETURN[msgRec, TRUE, NIL];        -- pretend msg already existed
  END;

ExpungeMsgs: PUBLIC PROC[tempLog: IO.STREAM, doUpdates, tailRewrite: BOOL]
        RETURNS[ok: BOOL] =
  -- Dumping is driven from the log file & preserves the bits that came from Grapevine
  -- Dumps all the Msgs in the database to a tempLog, setting all
  -- their body pointers to reference the NEW log, then copies tempLog to logStream
  BEGIN OPEN IO;
  newLen: INT;
  BEGIN ENABLE IO.EndOfStream => GOTO badLogFile;
  msg: Msg;

  toBeDestroyedList: LIST OF Msg;
  count: INT_ 0;
  expungeFilePos, startPos, prefixLength, entryLength: INT;
  startExpungePos: INT;
  entryChar: CHAR;
  numBad: INTEGER_ 0;
  DoCount: PROC =
   {IF (count_ count + 1) MOD 10 = 0 THEN
       ReportRope[IF count MOD 100 = 0 THEN "!" ELSE "~"]
    };
  WriteBadLogEntry: PROC[startPos: INT] RETURNS[nextStartPos: INT] =
    { errorStream: STREAM_ FileIO.Open["WalnutExpunge.errlog", write];
      line: ROPE;
      IF (numBad_ numBad+1) = 1 THEN errorStream.SetIndex[0]
       ELSE errorStream.SetIndex[errorStream.GetLength[]];
      logStream.SetIndex[startPos];
      errorStream.PutRope[
          IO.PutFR["\n  Bad entry from pos %g of log file, written at %g\n", int[startPos], time[]]];
      errorStream.PutRope[logStream.GetSequence[]];    -- *start* line
      errorStream.PutRope["\n"];
      DO
       nextStartPos_ logStream.GetIndex[];
       line_ logStream.GetSequence[];
       IF line.Equal["*start*"] THEN {nextStartPos_ logStream.GetIndex[]-8; EXIT};
       errorStream.PutRope[line];
       errorStream.PutRope["\n"];
      ENDLOOP;
      errorStream.SetLength[errorStream.GetIndex[]];
      errorStream.Close[];
   };
    CopyEntryIfNecessary: PROC =
    {  thisTag, thisValue: ROPE_ NIL;
       thisMsg: Msg;
        doCopy: BOOL_ TRUE;
        prefixPos: INT;
        IF entryChar = '_ THEN        -- hasbeenread entry
         { -- logStream.SetIndex[startPos+minPrefixLength];  where logStream is pos'd
           thisValue_ RopeIO.GetRope[logStream, prefixLength-minPrefixLength-1];
           []_ logStream.GetChar[ ! IO.EndOfStream => CONTINUE]
         }
         ELSE
        { logStream.SetIndex[startPos+prefixLength];
          [thisTag, thisValue]_ TagAndValue[logStream, 2];
           IF thisTag.Equal["Relation"]  AND thisValue.Equal["mCategory"] THEN
           { [thisTag, thisValue]_ TagAndValue[logStream, 2];
              IF ~thisTag.Equal["of"] THEN thisValue_ NIL;
           };
         };
        IF thisValue # NIL THEN
         { thisMsg_ DeclareEntity[MsgDomain, thisValue, OldOnly];
            IF ~DB.Null[thisMsg] THEN
            IF (prefixPos_ V2I[DB.GetP[thisMsg, mPrefixPos]]) >= startExpungePos THEN
                    doCopy_ FALSE;
         };
        IF doCopy THEN
         { logStream.SetIndex[startPos];
          tempLog.PutRope[RopeIO.GetRope[logStream, entryLength]];
         };
    };

  tempLog.SetIndex[0];
  startExpungePos_
      IF tailRewrite THEN V2I[DB.GetF[walnutInfoRelship, wStartExpungePos]] ELSE 0;
  logStream.SetIndex[startExpungePos];

   DO            -- loop for dumping messages & making other log entries
   msgID: ROPE_ NIL;
   fullText: ROPE_ NIL;
   headersPos, curPos: INT;
   badEntry: BOOL_ FALSE;
 
  [startPos, prefixLength, entryLength, entryChar]_ ReadStartOfMsg[logStream, TRUE];
  IF entryLength = 0 THEN EXIT;
  IF (entryChar # ' ) AND (entryChar # '?) THEN    -- prehaps copy other entry to tempLog
   { IF startExpungePos # 0 THEN CopyEntryIfNecessary[];
     logStream.SetIndex[startPos+entryLength];    -- for good measure, even after GetRope
     LOOP
    };

  headersPos_ startPos+prefixLength;
  UNTIL (curPos_ logStream.GetIndex[]) = headersPos DO
    -- look for a msgID: field
     tag, value: ROPE;

     [tag, value]_ TagAndValue[h: logStream, inc: 2];
     IF Rope.Equal[tag, msgIDRope, FALSE] THEN msgID_ value;
     IF curPos > headersPos THEN -- bad entry we don't understand
         { nextStartPos: INT_ WriteBadLogEntry[startPos];
           cMsg: ROPE_ " Confirm to push on, deny to stop the Expunge"; 
              ReportRope[PutFR["\nBad message entry at log pos %g;", int[startPos]]];
           Report[" entry written on WalnutExpunge.errlog"];
           Report[cMsg];
           IF (ok_ UserConfirmed[])
                   THEN {logStream.SetIndex[nextStartPos]; badEntry_ TRUE;  EXIT};
           RETURN[FALSE]};
   ENDLOOP;
    IF badEntry THEN LOOP;

   IF msgID = NIL THEN
    BEGIN        -- have to read fullText to fashion msgID
    mr: MsgRec_ NEW[MessageRecObject];
    []_ WalnutRetrieve.ParseMsgIntoFields[mr, logStream, entryLength-prefixLength !
        IO.EndOfStream => {Report[skipRope]; logStream.SetIndex[startPos+entryLength]; LOOP}];
    msgID_ ConstructMsgID[mr];
    logStream.SetIndex[startPos+prefixLength];        -- back up stream
    END;

   msg_ DoNameToEntity[d: MsgDomain, name: msgID, oldOnly: TRUE];
   IF (msg=NIL) OR DB.Null[msg] THEN {logStream.SetIndex[startPos+entryLength]; LOOP}
   ELSE
    BEGIN
     curPrefixPos: INT;
     thisMsgSet: MsgSet;
     nameOfCatsList: ROPE_ NIL;

-- if is only in deletedMsgSet, don't put on tempLog
     FOR mL: LIST OF DB.Value_ DB.GetPList[msg, mCategoryIs], mL.rest UNTIL mL=NIL DO
      IF ~Eq[thisMsgSet_ V2E[mL.first], deletedMsgSet] THEN
        nameOfCatsList_ Rope.Cat[nameOfCatsList, " ", DB.GetName[thisMsgSet]];
     ENDLOOP;

     IF nameOfCatsList.Length[] = 0 THEN
      { toBeDestroyedList_ CONS[msg, toBeDestroyedList]; GOTO doneWithOne};
     msgID_ Rope.Cat[msgID, "\nCategories:", nameOfCatsList];

     IF fullText = NIL THEN fullText_
       RopeFromStream[logStream, entryLength-prefixLength ! IO.EndOfStream =>
         {Report[skipRope]; GOTO doneWithOne}];

     DoCount[];
     expungeFilePos_
       MakeLogEntry[IF V2B[DB.GetP[msg, mHasBeenReadIs]] THEN message ELSE newMessage,
                   fullText, tempLog, Rope.Cat[msgIDRope, ": ", msgID, "\n"]];
   -- set header & body pos pointers to posn in NEW log if doing updates:
    IF doUpdates THEN
     { thisPrefixPos: INT_ startExpungePos + expungeFilePos - entryLength;
        curPrefixPos_ V2I[DB.GetP[msg, mPrefixPos]];
        IF curPrefixPos # thisPrefixPos THEN
        { []_ DB.SetP[msg, mHeadersPos, I2V[thisPrefixPos+prefixLength]];
          []_ DB.SetP[msg, mPrefixPos, I2V[thisPrefixPos]];
        };
     };


     EXITS
      doneWithOne => logStream.SetIndex[startPos+entryLength];
    END;        -- put message on log file
  ENDLOOP;        -- loop for dumping messages

  Report[PutFR["\nDumped %g messages from Log File... ", int[count]]];
  tempLog.SetLength[newLen_ tempLog.GetIndex[]];
  tempLog.Flush[];        -- make sure tempLog is in good shape
  Report[PutFR["Old log length was %g bytes", int[logStream.GetLength[]]]]; 
  IF tailRewrite THEN
   { Report[PutFR
           ["Previous end of log was %g bytes", int[logStream.GetLength[]-startExpungePos]]];
     Report[PutFR["New end of log file is %g bytes (%g messages)", int[newLen], int[count]]];
     Report[PutFR["New log length is %g bytes", int[startExpungePos+newLen]]];
   }
  ELSE
   { Report[PutFR["Old log length was %g bytes", int[logStream.GetLength[]]]];
     Report[PutFR["New log length is %g bytes (%g messages)", int[newLen], int[count]]]
    };

  IF doUpdates THEN
    { ReportRope[" Destroying Msgs in Deleted MsgSet ..."];
--  may see a msg more than once in the log file, hence must check if already destroyed
      count_ 0;
      FOR mL: LIST OF Msg_ toBeDestroyedList, mL.rest UNTIL mL = NIL DO
        IF ~DB.Null[mL.first] THEN
         { DB.DestroyEntity[mL.first];        -- destroy the message
            DoCount[];
          };
      ENDLOOP;
     Report[PutFR["\n%g Msgs destroyed",int[count]]];
    BEGIN            -- fix up num in deleted (should be 0 but who knows)
     num: INT_ DB.V2I[DB.GetP[deletedMsgSet, msNumInSetIs]] - count;
     []_ DB.SetP[deletedMsgSet, msNumInSetIs, I2V[num]];
    END;

  Report["Copying tempLog to log file"];
  DB.SetF[walnutInfoRelship, wCopyInProgress, B2V[TRUE]];
  DB.SetF[walnutInfoRelship, wStartExpungePos, I2V[startExpungePos]];
  DB.MarkTransaction[DB.TransactionOf[$Walnut]];

  CopyTempLogToLog[tempLog, startExpungePos];

--  lastMsgAddedToDB_ lastMsgWrittenToLog_ logStream.GetIndex[];
--  DB.SetF[walnutInfoRelship, wCopyInProgress, B2V[FALSE]];
--  DB.SetF[walnutInfoRelship, wStartExpungePos, I2V[lastMsgAddedToDB]];
  };        -- end doUpdates

  RETURN[TRUE];
  
  EXITS
    badLogFile =>
     { tempLog.SetLength[tempLog.GetIndex[]]; 
       tempLog.Close[];
       RETURN[FALSE]
     };
    END;
  END;

ConstructMsgID: PUBLIC PROC[mr: MsgRec] RETURNS[ROPE] =
-- uses gv address of Cabernet as default
-- if date is Unintelligible => use Time.Current
  BEGIN
     ts: GVBasics.Timestamp_ [net: 3, host: 14, time: 0];
    IF mr.gvSender = NIL THEN
     { IF mr.inMsgSender # NIL THEN mr.gvSender_ mr.inMsgSender
        ELSE IF mr.from = NIL THEN mr.gvSender_ "UnknownSender"
          ELSE mr.gvSender_ mr.from};
    IF mr.date = NIL THEN mr.date_ IO.PutFR[NIL, IO.time[]];        -- current time
    BEGIN
     ts.time_ DateAndTime.Parse[mr.date ! DateAndTime.Unintelligible => GOTO badTime].dt;
     TRUSTED {mr.dateCode_ LOOPHOLE[ts.time, System.GreenwichMeanTime]};
     EXITS
      badTime => TRUSTED {mr.dateCode_ Time.Current[]};
    END;
     
--    mr.gvID_ Rope.Cat[mr.gvSender, " $ ", ~~GVBasics.RopeFromTimestamp[ts]];

    mr.gvID_ Rope.Cat[mr.gvSender, " $ ", RopeFromTimestamp[ts] ];
    RETURN[mr.gvID];
  END;

ParseMsgID: PROC[mr: MsgRec] =
-- parses gvSender out of mr.gvID, compute dateCode
  BEGIN
    local: ROPE_ mr.gvID;
    mr.gvSender_ local.Substr[0, local.Find[" $ "]];
     mr.dateCode_ DateAndTime.Parse[mr.date ! DateAndTime.Unintelligible => GOTO badTime].dt;
  EXITS
    badTime => TRUSTED {mr.dateCode_ Time.Current[]};
  END;

RopeFromTimestamp: PUBLIC PROC[ts: GVBasics.Timestamp] RETURNS [ROPE] =
  BEGIN OPEN IO;
    RETURN[PutFR["%g#%g@%g",
        int[ts.net], int[ts.host], time[LOOPHOLE[ts.time, System.GreenwichMeanTime]] ]];
  END;

AddLogEntriesToDB: PUBLIC PROC [startPos: INT] RETURNS[success: BOOL] =
  { success_ AddLogEntries[logStream, startPos];
     IF success THEN lastMsgAddedToDB_ logStream.GetIndex[];
   };

CopyTempLogToLog: PROC[tempLog: IO.Handle, logStreamStartPos: INT] =
--  copy tempLog to logStream
  BEGIN
  bytes: NAT;
  logStream.SetIndex[logStreamStartPos]; tempLog.SetIndex[0];
  DO
    IF (bytes_ tempLog.GetBlock[copyBuffer]) = 0 THEN EXIT;
    logStream.PutBlock[copyBuffer];
  ENDLOOP;
  lastMsgAddedToDB_ lastMsgWrittenToLog_ logStream.GetIndex[];
  logStream.SetLength[lastMsgAddedToDB];  -- set the log length to current length
  logStream.Flush[];

  tempLog.Close[]; -- we don't need this guy any more, we've copied him to log.

  DB.SetF[walnutInfoRelship, wCopyInProgress, B2V[FALSE]];
  DB.SetF[walnutInfoRelship, wStartExpungePos, I2V[lastMsgAddedToDB]];
  END;

END.

Change Log.

By Willie-Sue on February 14, 1983
 --  added tailRewrite and doUpdates code to ExpungeMsgs