<<>>
    <<WalnutLogImpl.mesa>>
        <<Copyright Ó 1984, 1988, 1989, 1990, 1991, 1992 by Xerox Corporation.  All rights reserved.>>
        <<Willie-Sue, August 1, 1989 8:56:11 pm PDT>>
        <<Wert, August 31, 1984 9:49:21 pm PDT>>
        <<Donahue, January 22, 1986 8:04:31 am PST>>
        <<Doug Terry, October 16, 1990 9:31 am PDT>>
        <<Swinehar, April 26, 1991 9:00 am PDT>>
        <<Willie-s, April 27, 1992 1:51 pm PDT>>
        <<>>
        <<Walnut Log Operations Implementation>>

    DIRECTORY
        BasicTime USING [GMT, nullGMT, Now],
        FS USING [BytesForPages, Error, ErrorDesc, PagesForBytes],
        IO,
        MailUtils USING [GeneratePostmark],
        RefText USING [line, Length, ObtainScratch, ReleaseScratch, TrustTextAsRope],
        Rope,
        RuntimeError USING [BoundsFault],
        SendMailOps USING [IsThisTheCurrentUser, TiogaTextFromStrm, RopeFromStream],
        ViewerTools USING [TiogaContentsRec, TiogaContents],
        WalnutDefs USING [CheckReportProc, Error, LogInfo, WalnutOpsHandle],
        WalnutKernelDefs USING [LogEntry, LogEntryObject, MsgLogEntry, WhichTempLog],
        WalnutLog,
        WalnutMiscLog USING [TiogaControlItemType],
        WalnutRoot  -- using lots -- ,
        WalnutStream  -- using lots -- ;

    WalnutLogImpl: CEDAR PROGRAM

        IMPORTS
            BasicTime, FS, IO, MailUtils, RefText, Rope, RuntimeError, SendMailOps,
            WalnutDefs, WalnutRoot, WalnutStream

        EXPORTS
            WalnutDefs, WalnutLog, WalnutMiscLog

        = BEGIN
        <<>>
        <<Plan>>
            <<>>
            <<The log streams provide an abstraction for WalnutOps use in handling the log. WalnutOps procedures deal only with the notion of 
            writing an entry on the current log and reading the entry from a specified position. The log position for a write is 
            always at the end of the current log.>>
        <<>>
        <<Types>>

            GMT: TYPE = BasicTime.GMT;
            ROPE: TYPE = Rope.ROPE;
            STREAM: TYPE = IO.STREAM;
            LogEntry: TYPE = WalnutKernelDefs.LogEntry;
            MsgLogEntry: TYPE = WalnutKernelDefs.MsgLogEntry;
            TiogaContents: TYPE = ViewerTools.TiogaContents;

            LogInfo: TYPE = WalnutDefs.LogInfo;
            WalnutOpsHandle: TYPE = WalnutDefs.WalnutOpsHandle;

            LogHandle: TYPE = WalnutLog.LogHandle;
            LogHandleRec: PUBLIC TYPE = WalnutLog.LogHandleRec;
            RootHandle: TYPE = WalnutRoot.RootHandle;
            RootHandleRec: PUBLIC TYPE = WalnutRoot.RootHandleRec;
            InternalLogInfo: TYPE = WalnutRoot.InternalLogInfo;
            <<>>
            <<the all streams are under the same transaction;  WalnutLogImpl must get them WalnutRoot to open the files>>

        <<Variables for WalnutMiscLog>>
            <<>>
            msgIDRope: ROPE = "gvMsgID";
            categoriesRope: ROPE = "Categories";
            activeMsgSet: ROPE = "Active";
            newMsgSetList: LIST OF ROPE;

            generalBuffer: REF TEXT;
        <<>>
        <<Procedures>>
        <<>>
        <<Logging of operations that perform actions on the Walnut database>>

            ExpungeMsgs: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[at, next: INT] =
                { [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.expungeMsgs] };
            <<>>
            WriteExpungeLog: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[at, next: INT] = {
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.writeExpungeLog];
                };

            CreateMsgSet: PUBLIC PROC[opsH: WalnutOpsHandle, name: ROPE] RETURNS[at, next: INT] = {
                WalnutStream.logInfoRef.createMsgSet.msgSet Ź name;
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.createMsgSet]
                };

            EmptyMsgSet: PUBLIC PROC[opsH: WalnutOpsHandle, msgSet: ROPE]
                    RETURNS[at, next: INT] = {
                WalnutStream.logInfoRef.emptyMsgSet.msgSet Ź msgSet;
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.emptyMsgSet]
                };

            DestroyMsgSet: PUBLIC PROC[opsH: WalnutOpsHandle, msgSet: ROPE]
            RETURNS[at, next: INT] = {
                WalnutStream.logInfoRef.destroyMsgSet.msgSet Ź msgSet;
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.destroyMsgSet]
                };

            AddMsg: PUBLIC PROC[opsH: WalnutOpsHandle, msg, to: ROPE] RETURNS[at, next: INT] = {
                WalnutStream.logInfoRef.addMsg.msg Ź msg;
                WalnutStream.logInfoRef.addMsg.to Ź to;
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.addMsg]
                };

            RemoveMsg: PUBLIC PROC[opsH: WalnutOpsHandle, msg, from: ROPE]
                    RETURNS[at, next: INT] = {
                WalnutStream.logInfoRef.removeMsg.msg Ź msg;
                WalnutStream.logInfoRef.removeMsg.from Ź from;
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.removeMsg]
                };

            MoveMsg: PUBLIC PROC[opsH: WalnutOpsHandle, msg, from, to: ROPE]
                    RETURNS[at, next: INT] = {
                WalnutStream.logInfoRef.moveMsg.msg Ź msg;
                WalnutStream.logInfoRef.moveMsg.from Ź from;
                WalnutStream.logInfoRef.moveMsg.to Ź to;
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.moveMsg]
                };

            HasBeenRead: PUBLIC PROC[opsH: WalnutOpsHandle, msg: ROPE] RETURNS[at, next: INT] = {
                WalnutStream.logInfoRef.hasbeenRead.msg Ź msg;
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.hasbeenRead]
                };

            RecordNewMailInfo: PUBLIC PROC[opsH: WalnutOpsHandle, logLen: INT, when: GMT, server: ROPE, num: INT]
                    RETURNS[at, next: INT] = {
                WalnutStream.logInfoRef.recordNewMailInfo.logLen Ź logLen;
                WalnutStream.logInfoRef.recordNewMailInfo.when Ź when;
                WalnutStream.logInfoRef.recordNewMailInfo.server Ź server;
                WalnutStream.logInfoRef.recordNewMailInfo.num Ź num;
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.recordNewMailInfo]
                };

            StartCopyNewMail: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[at, next: INT] =
                { [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.startCopyNewMail] };

            AcceptNewMail: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[at, next: INT] =
                { [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.acceptNewMail] };

            StartReadArchiveFile: PUBLIC PROC[opsH: WalnutOpsHandle, file: ROPE, msgSet: ROPE] RETURNS[at, next: INT] = {
                WalnutStream.logInfoRef.startReadArchiveFile.file Ź file;
                WalnutStream.logInfoRef.startReadArchiveFile.msgSet Ź msgSet;
                [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.startReadArchiveFile]
                };

            EndReadArchiveFile: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[at, next: INT] =
                { [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.endReadArchiveFile] };

            StartCopyReadArchive: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[at, next: INT] =
                { [at, next] Ź WriteEntry[opsH, WalnutStream.logInfoRef.startCopyReadArchive] };
        <<>>
        <<Managing the current log>>

            InitLogStream: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[BOOL] = {
                opsH.logHandle Ź NEW[LogHandleRec Ź [] ];
                OpenLogStreams[opsH];
                RETURN[TRUE];
                };

            LogLength: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[length: INT] = {
                li: InternalLogInfo = opsH.rootHandle.currentLog;
                RETURN[ IF li.writeStream # NIL THEN li.writeStream.GetLength[] ELSE li.readStream.GetLength[] ];
                };

            OpenLogStreams: PUBLIC PROC[opsH: WalnutOpsHandle] = {
                IF opsH.rootHandle.currentLog = NIL OR opsH.rootHandle.currentLog.readStream = NIL THEN {
                    WalnutRoot.OpenLogStreams[opsH];
                    IF opsH.logHandle = NIL THEN opsH.logHandle Ź NEW[LogHandleRec Ź [] ];
                    IF opsH.logHandle.scanning THEN SetLogIndex[opsH, opsH.logHandle.entryPos, TRUE]; 
                    };
                };

            ReleaseWriteLock: PUBLIC PROC[opsH: WalnutOpsHandle] = {
                logH: LogHandle = opsH.logHandle;
                IF opsH.rootHandle.currentLog.writeStream = NIL THEN RETURN;
                WalnutRoot.ReleaseWriteLock[opsH];
                IF logH.scanning THEN SetLogIndex[opsH, logH.entryPos, TRUE];
                };

            ResetLog: PUBLIC PROC[opsH: WalnutOpsHandle, newLength: INT] = {
                strm: STREAM;
                IF opsH.logHandle = NIL THEN RETURN;
                SetLogIndex[opsH, newLength, TRUE];
                WalnutStream.SetHighWaterMark[strm Ź opsH.rootHandle.currentLog.writeStream,
                newLength, -1];
                strm.SetLength[newLength];
                strm.Flush[];
                <<April 26, 1991, DCS, assure invariant that log streams are positioned at end, except temporarily when explicitly repositioned.>>
                strm.SetIndex[newLength];
                opsH.logHandle.scanning Ź FALSE;
                };

            ForgetLogStreams: PUBLIC PROC[opsH: WalnutOpsHandle] = {
                logH: LogHandle = opsH.logHandle;
                IF logH = NIL THEN RETURN;
                logH.scanning Ź FALSE;
                logH.strm Ź NIL;
                logH.scanningLog Ź NIL;
                };

            ShutdownLog: PUBLIC PROC[opsH: WalnutOpsHandle] = {
                logH: LogHandle = opsH.logHandle;
                IF logH = NIL THEN RETURN;
                logH.scanning Ź FALSE;
                logH.strm Ź NIL;
                logH.scanningLog Ź NIL;
                WalnutRoot.CloseTransaction[opsH];
                };

        <<>>
        <<Write a message on the current log>>

            WriteMessage: PUBLIC PROC[opsH: WalnutOpsHandle, msg: ROPE, body: TiogaContents]
                    RETURNS[at: INT]= {
                wStrm: STREAM = opsH.rootHandle.currentLog.writeStream;
                wStrm.SetIndex[wStrm.GetLength[]];
                WalnutStream.logInfoRef.createMsg.msg Ź msg;
                IF ( body.formatting.Length[] # 0 ) AND ( body.contents.Fetch[0] = '\r OR body.contents.Fetch[0] = '\l ) THEN {
                    last: INT Ź body.contents.Length[] - 1;
                    IF body.contents.Fetch[last] = '\000 THEN    -- NUL for padding
                        { body.contents Ź Rope.Substr[body.contents, 1, last-1];
                            body.formatting Ź Rope.Concat["\000", body.formatting]
                        }
                    ELSE
                        IF body.contents.Fetch[0] = '\r OR body.contents.Fetch[0] = '\l THEN
                            body.contents Ź Rope.Substr[body.contents, 1]
                    };
                WalnutStream.logInfoRef.createMsg.textLen Ź body.contents.Length[];
                WalnutStream.logInfoRef.createMsg.formatLen Ź body.formatting.Length[];
                at Ź WalnutStream.WriteEntry[wStrm, WalnutStream.logInfoRef.createMsg];
                WalnutStream.WriteMsgBody[wStrm, body];
                WalnutStream.FlushStream[strm: wStrm, setCreateDate: TRUE];
                };
        <<>>
        <<Parsing a log (used by Archive and Replay/Scavenge>>

            SetPosition: PUBLIC PROC[opsH: WalnutOpsHandle, startPos: INT]
                    RETURNS[charsSkipped: INT] = {
                next: INT;
                logH: LogHandle = opsH.logHandle;
                SetLogIndex[opsH, startPos, TRUE];
                next Ź WalnutStream.FindNextEntry[logH.strm];
                logH.entryLengthHint Ź -1;
                IF next # -1 THEN logH.entryPos Ź next;
                RETURN[IF next = -1 THEN next ELSE next - startPos];
                };

            SetIndex: PUBLIC PROC[opsH: WalnutOpsHandle, pos: INT] = {
                logH: LogHandle = opsH.logHandle;
                SetLogIndex[opsH, pos, TRUE];
                logH.entryLengthHint Ź -1;
                logH.entryPos Ź pos;
                };

            NextAt: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[at: INT] =
                { RETURN[opsH.logHandle.entryPos] };

            NextEntry: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[le: LogEntry, at: INT] =
                { [le, at] Ź NextEntryInternal[opsH, FALSE] };

            NextEntryInternal: PROC[opsH: WalnutOpsHandle, quick: BOOL]
                    RETURNS[le: LogEntry, at: INT] = {
                logH: LogHandle = opsH.logHandle;
                length, startPos: INT;
                IF ~logH.scanning THEN ERROR WalnutDefs.Error[$log, $NotScanning];
                [le, length] Ź WalnutStream.ReadEntry[logH.strm, quick];
                startPos Ź logH.entryPos;
                logH.entryLengthHint Ź -1;
                IF length # -1 THEN logH.entryPos Ź startPos + length;    -- normal case
                RETURN[le, startPos];
                };

            QuickScan: PUBLIC PROC[opsH: WalnutOpsHandle] RETURNS[le: LogEntry, at: INT] =
                { [le, at] Ź NextEntryInternal[opsH: opsH, quick: TRUE] };

            ArchiveEntry: PUBLIC PROC[opsH: WalnutOpsHandle, to: STREAM] RETURNS[ok: BOOL]= {
                <<Copies the next entry of the current log to an archive stream.>>
                logH: LogHandle = opsH.logHandle;
                IF ~logH.scanning THEN ERROR WalnutDefs.Error[$log, $NotScanning];
                ok Ź FALSE;
                IF logH.entryLengthHint = -1 THEN
                    logH.entryLengthHint Ź
                        WalnutStream.PeekEntry[logH.strm, TRUE].length;
                IF logH.entryLengthHint = -1 THEN RETURN;
                WalnutStream.CopyBytes[
                        from: logH.strm, to: to, num: logH.entryLengthHint ];
                logH.entryPos Ź logH.strm.GetIndex[];
                logH.entryLengthHint Ź -1;        -- unknown
                ok Ź TRUE;
                };

            CopyBytesToArchive: PUBLIC PROC[
                    opsH: WalnutOpsHandle, to: STREAM, startPos, num: INT] = {
                SetLogIndex[opsH, startPos, FALSE];
                WalnutStream.CopyBytes[from: opsH.logHandle.strm, to: to, num: num];
                };
        <<>>
        <<Copying NewMail/ReadArchive logs>>

            PrepareToCopyTempLog: PUBLIC PROC[opsH: WalnutOpsHandle, which: WalnutKernelDefs.WhichTempLog, pagesAlreadyCopied: INT, reportProc: 
            WalnutDefs.CheckReportProc] RETURNS[BOOL] = {
                <<makes sure the writeStream is long enought to copy the which log onto>>
                RETURN[WalnutRoot.PrepareToCopyTempLog[opsH, which, pagesAlreadyCopied, reportProc] ];
                };

            CopyTempLog: PUBLIC PROC[opsH: WalnutOpsHandle, which: WalnutKernelDefs.WhichTempLog, startCopyPos, fromPos: INT, reportProc: 
            WalnutDefs.CheckReportProc ] = {
                <<copies the which templog, starting at fromPos, to the end of the writeStream; writes an EndCopy log entry, and sets the length of 
                which to 0; if the operation fails, WalnutDefs.Error is raised; NOT done inside carefullyApply, so that it can 
                catch transaction aborts>>
                exp: ROPE;
                endLE: LogEntry;
                abort: BOOL Ź TRUE;
                ok: BOOL Ź FALSE;
                copyStrm, wStrm: STREAM;
                bytesToCopy: INT;
                bytesPerCopy: INT = FS.BytesForPages[200];
                opsH.logHandle.scanning Ź FALSE;
                BEGIN ENABLE BEGIN
                    IO.Error => {
                        ed: FS.ErrorDesc;
                        <<ed Ź FS.ErrorFromStream[stream ! IO.Error => CONTINUE];>>
                        IF ed.code = $transAborted OR ed.code = $remoteCallFailed THEN
                            { exp Ź ed.explanation; GOTO exit };
                        REJECT;
                        };
                    FS.Error => {
                        IF error.code = $transAborted THEN GOTO exit;
                        IF error.code = $remoteCallFailed THEN {
                            exp Ź error.explanation;
                            GOTO exit;
                            };
                        REJECT;
                        };
                    END;

                    SELECT which FROM
                        newMail => {
                            WalnutStream.logInfoRef.endCopyNewMailInfo.startCopyPos Ź startCopyPos;
                            endLE Ź WalnutStream.logInfoRef.endCopyNewMailInfo;
                            };
                        readArchive => {
                            WalnutStream.logInfoRef.endCopyReadArchiveInfo.startCopyPos Ź startCopyPos;
                            endLE Ź WalnutStream.logInfoRef.endCopyReadArchiveInfo
                            };
                        ENDCASE => ERROR WalnutDefs.Error[$log, $UnknownLogFileType];

                    copyStrm Ź WalnutRoot.GetStreamsForCopy[opsH, which];
                    wStrm Ź opsH.rootHandle.currentLog.writeStream;
                    <<Play it safe!  DCS April 26, 1991>>
                    wStrm.SetIndex[wStrm.GetLength[]];

                    bytesToCopy Ź copyStrm.GetLength[] - fromPos;
                    IF bytesToCopy > 0 THEN {
                        first: BOOL Ź TRUE;
                        copyStrm.SetIndex[fromPos];
                        <<<<wStrm.SetIndex[wStrm.GetLength[]];>>  -- Moved outside conditional.>>
                        DO
                            thisCopy: INT Ź MIN[bytesPerCopy, bytesToCopy];
                            WalnutStream.CopyBytes[to: wStrm, from: copyStrm, num: thisCopy];
                            WalnutStream.FlushStream[wStrm, TRUE];
                            WalnutRoot.CommitAndContinue[opsH];
                            IF first THEN first Ź FALSE ELSE reportProc[opsH, "#"];
                            bytesToCopy Ź bytesToCopy - thisCopy;
                            IF bytesToCopy = 0 THEN EXIT;
                            ENDLOOP;
                        };
                    [] Ź WalnutStream.WriteEntry[wStrm, endLE];
                    WalnutStream.FlushStream[wStrm, TRUE];
                    WalnutRoot.CommitAndContinue[opsH];

                    EXITS
                        exit => {
                            who: ROPE Ź IF which = newMail THEN "NewMail" ELSE "ReadArchive";
                            IF ~abort THEN ERROR WalnutDefs.Error[$log, $AccessFailed, who];
                            WalnutRoot.AbortTempCopy[opsH, which];
                            WalnutRoot.CloseTransaction[opsH];
                            ERROR WalnutDefs.Error[
                                $log, IF exp = NIL THEN $TransactionAbort ELSE $RemoteCallFailed,
                                IO.PutFR1["Copying the %g log", [rope[who]]] ];
                            };
                    END;
                WalnutRoot.FinishCopy[opsH, which];
                };

            FinishTempLogCopy: PUBLIC PROC[opsH: WalnutOpsHandle, which: WalnutKernelDefs.WhichTempLog] =
                { WalnutRoot.FinishCopy[opsH, which] };

            CreateArchiveLog: PUBLIC PROC[
            opsH: WalnutOpsHandle, fileToRead: STREAM, msgSet: ROPE, reportProc: WalnutDefs.CheckReportProc]
            RETURNS[ok: BOOL] = {
                reason: ROPE;
                [ok, reason] Ź CreateReadArchiveLog[opsH, fileToRead, msgSet, reportProc];
                IF ok THEN [] Ź WriteEntry[opsH, WalnutStream.logInfoRef.endReadArchiveFile];
                WalnutRoot.ReturnReadArchiveStream[opsH];
                };
        <<>>
        <<Utilities>>

            GetTiogaContents: PUBLIC PROC[opsH: WalnutOpsHandle, textStart, textLen, formatLen: INT]
            RETURNS[contents: TiogaContents] = {
                logH: LogHandle = opsH.logHandle;

                IF formatLen # 0 THEN        -- tioga formatting nonsense
                    { textLen Ź textLen + 1; textStart Ź textStart - 1};
                SetLogIndex[opsH, textStart, FALSE];
                contents Ź NEW[ViewerTools.TiogaContentsRec];
                contents.contents Ź WalnutStream.ReadRope[logH.strm, textLen];
                contents.formatting Ź WalnutStream.ReadRope[logH.strm, formatLen];
                };

            GetRefTextFromLog: PUBLIC PROC[opsH: WalnutOpsHandle, startPos, length: INT, text: REF TEXT] = {
                natLen: NAT;
                bytesRead: NAT;
                logH: LogHandle = opsH.logHandle;
                CheckSize: PROC[len: INT] RETURNS[nat: NAT] = { nat Ź len };
                natLen Ź CheckSize[length ! RuntimeError.BoundsFault => GOTO noGood];

                SetLogIndex[opsH, startPos, FALSE];
                bytesRead Ź logH.strm.GetBlock[block: text, startIndex: 0, count: natLen];
                IF bytesRead < natLen THEN text.length Ź 0;
                EXITS
                    noGood =>
                        ERROR WalnutDefs.Error[$log, $BadLength,
                            IO.PutFR1[" Can't convert %g into a nat", [integer[length]] ]];
                };
        <<>>
        <<Local Utilities>>

            SetLogIndex: PROC[opsH: WalnutOpsHandle, pos: INT, setScan: BOOL] = {
                logH: LogHandle = opsH.logHandle;
                logH.strm Ź opsH.rootHandle.currentLog.readStream;
                IF ( logH.scanning Ź setScan ) THEN logH.scanningLog Ź opsH.rootHandle.currentLog;
                logH.strm.SetIndex[pos ! IO.Error => GOTO err];
                EXITS err =>
                    ERROR WalnutDefs.Error[$log, $BadLogIndex, IO.PutFR1["Pos beyond end of log %g", [integer[pos]]] ];
                };

            WriteEntry: PROC[opsH: WalnutOpsHandle, le: LogEntry] RETURNS[at, next: INT] = {
                wStrm: STREAM = opsH.rootHandle.currentLog.writeStream;
                at Ź WalnutStream.WriteEntry[wStrm, le];
                WalnutStream.FlushStream[wStrm, TRUE];
                next Ź wStrm.GetIndex[];
                };

        <<>>
        <<Writing and Reading NewMail/ReadArchive files>>

            GetNewMailLog: PUBLIC PROC[opsH: WalnutOpsHandle, lengthRequired, pagesWanted: INT]
                    RETURNS[STREAM] = {
                <<opens the NewMailLog (aborting any current stream that might be open); truncates it to the required length;  returns NIL if the 
                log is shorter than required, or if the file could not be opened >>
                BEGIN ENABLE IO.Error, FS.Error => {
                    AbortMailLog[opsH];
                    GOTO exit;
                    };

                    AbortMailLog[opsH];

                    IF NOT WalnutRoot.GetNewMailStream[opsH, lengthRequired, pagesWanted].ok THEN RETURN[NIL];

                    EXITS
                        exit => AbortMailLog[opsH];
                    END;
                RETURN[opsH.rootHandle.newMailLog.stream];
                };

            CloseNewMailLog: PUBLIC PROC[opsH: WalnutOpsHandle] = {
                mStrm: STREAM = opsH.rootHandle.newMailLog.stream;
                IF mStrm = NIL THEN RETURN;
                WalnutRoot.ReturnNewMailStream[opsH];
                };

            AbortMailLog: PROC[opsH: WalnutOpsHandle] = {
                mStrm: STREAM = opsH.rootHandle.newMailLog.stream;
                IF mStrm = NIL THEN RETURN;
                WalnutStream.AbortStream[mStrm ! IO.Error, FS.Error => CONTINUE];
                WalnutRoot.ReturnNewMailStream[opsH];
                };

            CreateReadArchiveLog: PUBLIC PROC[opsH: WalnutOpsHandle, fileToRead: STREAM, msgSet: ROPE, reportProc: WalnutDefs.CheckReportProc]
                    RETURNS[ok: BOOL, reason: ROPE] = {
                <<reads file and writes the ReadArchiveLogFile;  returns ok if all went well>>
                readArchiveStrm: STREAM;
                lastCommitPosInFileToRead: INT;
                fileToReadPages: INT Ź FS.PagesForBytes[fileToRead.GetLength[]];
                possibleArchiveLength: INT Ź MAX[20, fileToReadPages + (fileToReadPages/5)];    -- 120%
                ok Ź FALSE;
                reason Ź NIL;
                BEGIN ENABLE BEGIN
                    IO.Error => {
                        <<reason Ź FS.ErrorFromStream[fileToRead].explanation;>>
                        <<IF reason = NIL THEN reason Ź FS.ErrorFromStream[readArchiveStrm].explanation;>>
                        IF reason = NIL THEN reason Ź "IO error during createReadArchiveLog";
                        GOTO exit;
                        };
                    FS.Error => { reason Ź error.explanation; GOTO exit };
                    UNWIND => GOTO exit;
                    END;

                    IF (readArchiveStrm Ź WalnutRoot.GetReadArchiveStream[opsH, possibleArchiveLength]) = NIL THEN
                        RETURN[FALSE, "Can't get readArchiveStream"];

                    readArchiveStrm.SetIndex[0];
                    WalnutStream.SetHighWaterMark[strm: readArchiveStrm, hwmBytes: 0, numPages: -1];

                    WalnutRoot.CommitAndContinue[opsH];

                    [ok, lastCommitPosInFileToRead] Ź ArchiveReader[
                        opsH: opsH, archiveStream: readArchiveStrm,
                        fileToRead: fileToRead,
                        msgSet: msgSet, reportProc: reportProc, posToStartInFileToRead: 0];

                    IF ~ok THEN WalnutRoot.ReturnReadArchiveStream[opsH];

                    EXITS
                        exit =>
                            IF readArchiveStrm # NIL THEN {
                                WalnutRoot.ReturnReadArchiveStream[opsH];
                                };
                    END;
                };

            CloseReadArchiveLog: PUBLIC PROC[opsH: WalnutOpsHandle] = {
                WalnutRoot.ReturnReadArchiveStream[opsH];
                };


        <<********************************************************>>
        <<>>
        ArchiveEntryType: TYPE = {old, new};

            ArchiveReader: PUBLIC PROC[opsH: WalnutOpsHandle, archiveStream, fileToRead: STREAM, msgSet: ROPE,
            reportProc: WalnutDefs.CheckReportProc,
            posToStartInFileToRead: INT]
            RETURNS [ok: BOOL, lastCommitPosInFileToRead: INT] = {

            num: INT Ź 0;
            BadFormat: PROC[format: ROPE, v: IO.Value Ź [null[]] ] = {
                IF reportProc = NIL THEN RETURN;
                reportProc[opsH, "Bad log file format at %g: ", [integer[fileToRead.GetIndex[]]] ];
                reportProc[opsH, format, v];
                reportProc[opsH, "\n"];
                };

                BEGIN ENABLE BEGIN
                    ABORTED => { ok Ź FALSE; GOTO GiveUp };
                    FS.Error => {
                        reportProc[opsH, error.explanation];
                        ok Ź FALSE; GOTO GiveUp
                        };
                    IO.Error => {
                        IF reportProc # NIL THEN {
                            <<reason: ROPE Ź FS.ErrorFromStream[fileToRead].explanation;>>
                            reason: ROPE Ź NIL;
                            IF reason # NIL THEN
                                reason Ź IO.PutFR["IOError on file being read, reported as %g, at pos %g",
                                    [rope[reason]], [integer[fileToRead.GetIndex[]]] ]
                                <<ELSE reason Ź FS.ErrorFromStream[archiveStream].explanation>>;
                            IF reason # NIL THEN
                                reportProc[opsH, "\nIOError on tempLog being written, reported as %g, at pos%g",
                                    [rope[reason]], [integer[archiveStream.GetIndex[]]] ]
                                ELSE
                                    reportProc[opsH, "\nIOError at writePos %g", [integer[archiveStream.GetIndex[]]] ];
                            };
                        ok Ź FALSE; GOTO GiveUp
                        };
                    IO.EndOfStream => {BadFormat["Unexpected EOF"]; ok Ź FALSE; GOTO GiveUp};
                    END;

                entryLength, prefixLength, beginMsgPos: INT;
                entryChar: CHAR;
                bytesWritten: INT Ź 0;
                bytesToWriteBeforeCommit: INT = 100000;        -- try this
                lengthOfFileToRead: INT Ź fileToRead.GetLength[];
                startPosOnArchive: INT;

                newMsgSetList Ź NIL;        -- global for each file read
                ok Ź TRUE;
                generalBuffer Ź RefText.ObtainScratch[RefText.line];
                lastCommitPosInFileToRead Ź posToStartInFileToRead;
                fileToRead.SetIndex[posToStartInFileToRead];

                DO
                    <<Read *start* from log>>
                    [beginMsgPos, prefixLength, entryLength, entryChar] Ź FindStartOfEntry[fileToRead];
                    IF entryLength = -1 OR  (entryLength=0) OR (entryLength=prefixLength) AND NOT (entryChar = '_) THEN {
                        reportProc[opsH, "\t%g messages read from archive file\n", [integer[num]] ];
                        IF bytesWritten = 0 THEN RETURN[TRUE, lengthOfFileToRead];
                        IF Flush[opsH, archiveStream] THEN RETURN[TRUE, lengthOfFileToRead];
                        RETURN[FALSE, lastCommitPosInFileToRead];
                        };

                    SELECT entryChar FROM
                        '-, '+, '_ => {
                            BadFormat["Non-message entry (entrychar = %g)\n", [character[entryChar]] ];
                            fileToRead.SetIndex[beginMsgPos+entryLength]
                            };

                        ENDCASE => {         -- regular message entry
                            DO            -- fake DO so can use exit
                            msgID, msgEName, categories: ROPE;
                            catList: LIST OF ROPE;
                            outOfSynch, notInActive: BOOL;
                            headersPos: INT Ź beginMsgPos+prefixLength;
                            body: ViewerTools.TiogaContents;

                            archiveEntryType: ArchiveEntryType = DetermineEntryType[fileToRead];

                            IF archiveEntryType = old THEN {
                                [msgID, categories, outOfSynch] Ź ReadPrefixInfo[fileToRead, headersPos];
                                IF outOfSynch THEN {
                                    BadFormat["\nPrefix Info not found at %g\n", [integer[headersPos]] ];
                                    EXIT};
                                body Ź SendMailOps.TiogaTextFromStrm[        -- careful here
                                    fileToRead, headersPos, entryLength-prefixLength];
                                IF body.formatting.Length[] <= 2 THEN body.formatting Ź NIL;    --glitch

                                IF msgID.Length[] = 0 THEN {    -- need to construct an ID
                                    <<back up and parse from the strm>>
                                    from: ROPE;
                                    mle: WalnutKernelDefs.MsgLogEntry = WalnutStream.logInfoRef.createMsg;
                                    mle.textLen Ź entryLength - prefixLength;
                                    fileToRead.SetIndex[headersPos];
                                    WalnutStream.MsgEntryInfoFromStream[fileToRead, mle];
                                    from Ź IF SendMailOps.IsThisTheCurrentUser[mle.sender] THEN Rope.Concat["To: ", mle.to] ELSE mle.sender;
                                    IF from.Length[] = 0 THEN from Ź "UnknownFrom";
                                    IF mle.date = BasicTime.nullGMT THEN mle.date Ź BasicTime.Now[];

                                    msgEName Ź msgID Ź MailUtils.GeneratePostmark[mle.date, "SomeMachine" ];
                                    fileToRead.SetIndex[beginMsgPos+entryLength];
                                    };

                                IF body.formatting.Length[] = 0 THEN {
                                    pos: INT Ź 0;
                                    DO        -- replace \240's with spaces (used by laurel)
                                        pos Ź body.contents.Find["\240", pos];
                                        IF pos = -1 THEN EXIT;
                                        body.contents Ź body.contents.Replace[start: pos, len: 1, with: " "];
                                        ENDLOOP;
                                    };

                                }
                                ELSE {
                                    this: BOOL;
                                    [msgID, msgEName, categories, body, this] Ź
                                        ReadNewStyleArchiveFile[
                                            fileToRead, headersPos, entryLength - (headersPos-beginMsgPos) - 1];
                                    IF ~this THEN {
                                        BadFormat["\nCouldn't read entry at %g\n", [integer[beginMsgPos]] ];
                                        EXIT;
                                        };
                                    };

                        BEGIN ENABLE BEGIN
                            FS.Error =>
                                IF error.code = $transAborted THEN GOTO aborted ELSE REJECT;
                            IO.Error =>
                                IF WalnutStream.Aborted[archiveStream] THEN GOTO aborted ELSE REJECT;
                            END;

                            WalnutStream.logInfoRef.createMsg.msg Ź msgID;
                            WalnutStream.logInfoRef.createMsg.textLen Ź body.contents.Length[];
                            WalnutStream.logInfoRef.createMsg.formatLen Ź body.formatting.Length[];
                            startPosOnArchive Ź
                                WalnutStream.WriteEntry[archiveStream, WalnutStream.logInfoRef.createMsg];
                            WalnutStream.WriteMsgBody[archiveStream, body];

                            IF msgSet.Length[] # 0 THEN categories Ź NIL;
                            [catList, notInActive] Ź CheckCategories[archiveStream, msgSet, categories];

                            IF entryChar # '? THEN {
                                WalnutStream.logInfoRef.hasbeenRead.msg Ź msgEName;
                                [] Ź WalnutStream.WriteEntry[archiveStream, WalnutStream.logInfoRef.hasbeenRead];
                                };

                            <<-- ProcessCategories>>
                            BEGIN
                                first: BOOL Ź TRUE;
                                FOR mL: LIST OF ROPE Ź catList, mL.rest UNTIL mL=NIL DO
                                    IF first AND notInActive THEN {
                                        WalnutStream.logInfoRef.moveMsg.msg Ź msgEName;
                                        WalnutStream.logInfoRef.moveMsg.from Ź activeMsgSet;
                                        WalnutStream.logInfoRef.moveMsg.to Ź mL.first;
                                        [] Ź WalnutStream.WriteEntry[archiveStream, WalnutStream.logInfoRef.moveMsg]
                                        }
                                    ELSE {
                                        WalnutStream.logInfoRef.addMsg.msg Ź msgEName;
                                        WalnutStream.logInfoRef.addMsg.to Ź mL.first;
                                        [] Ź WalnutStream.WriteEntry[archiveStream, WalnutStream.logInfoRef.addMsg];
                                        };
                                    first Ź FALSE;
                                    ENDLOOP;
                            END;

                            IF (bytesWritten Ź bytesWritten + archiveStream.GetLength[]-startPosOnArchive) >= bytesToWriteBeforeCommit THEN {
                                WalnutStream.FlushStream[archiveStream];
                                WalnutRoot.CommitAndContinue[opsH];
                                lastCommitPosInFileToRead Ź beginMsgPos+entryLength;
                                bytesWritten Ź 0;
                                };

                            EXITS
                                aborted => {
                                    RefText.ReleaseScratch[generalBuffer];
                                    RETURN[FALSE, lastCommitPosInFileToRead];
                                    };
                            END;        -- of Enable FS.Error, IO.Error above - writing on archiveStream
                             
                            IF (num Ź num + 1) MOD 10 = 0 THEN {
                                IF num MOD 100 = 0 THEN reportProc[opsH, "(%g) ", [integer[num]] ]
                                 ELSE reportProc[opsH, "~"];
                                };
                            EXIT;
                            ENDLOOP;
                            fileToRead.SetIndex[beginMsgPos+entryLength];
                            };
                    ENDLOOP;

                        EXITS GiveUp => NULL;
                    END;

                RefText.ReleaseScratch[generalBuffer];
                reportProc[opsH, "\t%g messages read from archive file\n", [integer[num]] ];
                RETURN[ok, lastCommitPosInFileToRead];
                };

            Flush: PROC[opsH: WalnutOpsHandle, strm: STREAM] RETURNS[ok: BOOL] = {
                ENABLE FS.Error  => IF error.code = $transAborted THEN GOTO aborted ELSE REJECT;

                WalnutStream.FlushStream[strm];
                WalnutRoot.CommitAndContinue[opsH];
                RETURN[TRUE];
                EXITS
                    aborted => RETURN[FALSE]
                };

            CheckCategories: PROC[archiveStream: STREAM, msgSet, categories: ROPE]
                    RETURNS[catList: LIST OF ROPE, notInActive: BOOL] = {
                h: STREAM;
                CheckMsgSet: PROC[ms: ROPE] RETURNS[notActive: BOOL] = {
                    IF ms.Equal[activeMsgSet, FALSE] THEN RETURN[FALSE];

                    FOR mL: LIST OF ROPE Ź newMsgSetList, mL.rest UNTIL mL = NIL DO
                        IF mL.first.Equal[ms, FALSE] THEN RETURN[TRUE];
                        ENDLOOP;

                    WalnutStream.logInfoRef.createMsgSet.msgSet Ź ms;
                    [] Ź WalnutStream.WriteEntry[archiveStream, WalnutStream.logInfoRef.createMsgSet];
                    newMsgSetList Ź CONS[ms, newMsgSetList];        -- to be able to check next time
                    RETURN[TRUE];
                    };

                catList Ź NIL;
                notInActive Ź TRUE;
                IF categories = NIL THEN
                    IF msgSet.Length[] = 0 THEN RETURN[NIL, FALSE]
                        ELSE {
                            notInActive Ź CheckMsgSet[msgSet];
                            IF notInActive THEN RETURN[LIST[msgSet], notInActive]
                                ELSE RETURN[NIL, FALSE];
                            };

                IF msgSet.Length[] # 0 THEN {
                    notInActive Ź CheckMsgSet[msgSet];
                    IF notInActive THEN catList Ź LIST[msgSet];
                    };

                h Ź IO.RIS[categories];
                UNTIL h.EndOf[] DO
                    token: ROPE Ź h.GetTokenRope[breakProc: IO.IDProc].token;
                    IF token.Length[] # 0 THEN {
                        notActive: BOOL Ź CheckMsgSet[token];
                        notInActive Ź notInActive AND notActive;
                        IF notActive THEN catList Ź CONS[token, catList];
                        };
                    ENDLOOP;
                    h.Close[];
                };

            DetermineEntryType: PROC[strm: STREAM] RETURNS[aft: ArchiveEntryType] = {
                curPos: INT Ź strm.GetIndex[];
                aft Ź old;
                IF strm.GetChar[] = '@ THEN {        -- might be new or Hardy file
                    itemType: INT;
                    [] Ź strm.GetInt[];
                    IF (itemType Ź strm.GetInt[]) = WalnutMiscLog.TiogaControlItemType THEN
                        aft Ź new;
                    };
                strm.SetIndex[curPos];
                };

            ReadPrefixInfo: PROC[fileToRead: STREAM, headersPos: INT]
                    RETURNS[msgID: ROPE, categories: ROPE, outOfSynch: BOOL] = {
                curPos: INT;
                outOfSynch Ź FALSE;

                UNTIL (curPos Ź fileToRead.GetIndex[]) = headersPos DO
                    tag: ROPE;
                    IF curPos > headersPos THEN { outOfSynch Ź TRUE; RETURN };
                    tag Ź fileToRead.GetTokenRope[].token;
                    IF tag.Equal[msgIDRope, FALSE] THEN {
                        [] Ź fileToRead.GetChar[];    -- the :
                        [] Ź fileToRead.SkipWhitespace[];
                        msgID Ź WalnutStream.MsgNameFromIdOnFile[fileToRead];
                        LOOP;
                        };

                    IF tag.Equal[categoriesRope, FALSE] THEN {
                        [] Ź fileToRead.GetChar[];    -- the :
                        [] Ź fileToRead.SkipWhitespace[];
                        categories Ź fileToRead.GetLineRope[];
                        LOOP;
                        };
                    ENDLOOP;
                };

            ReadNewStyleArchiveFile: PROC[fileToRead: STREAM, headersPos, bodyLen: INT]
                    RETURNS[msgID, msgName, categories: ROPE, body: TiogaContents, ok: BOOL] = {
                len, type, pos, formatLen: INT;
                ok Ź FALSE;
                body Ź NEW[ViewerTools.TiogaContentsRec];

                IF fileToRead.GetChar[] # '@ THEN RETURN;
                len Ź fileToRead.GetInt[];
                type Ź fileToRead.GetInt[];
                formatLen Ź fileToRead.GetInt[];
                [] Ź fileToRead.GetChar[];        -- the \n
                IF type = WalnutMiscLog.TiogaControlItemType THEN {
                    msgIDPos: INT Ź fileToRead.GetIndex[];
                    msgID Ź WalnutStream.MsgNameFromIdOnFile[fileToRead];
                    fileToRead.SetIndex[msgIDPos];
                    msgName Ź WalnutStream.MsgNameFromIdOnFile[fileToRead];
                    categories Ź fileToRead.GetLineRope[];
                    IF formatLen # 0 THEN {
                        body.formatting Ź
                            SendMailOps.RopeFromStream[fileToRead, fileToRead.GetIndex[], formatLen];
                        [] Ź fileToRead.GetChar[];    -- cr
                        };
                    };
                [] Ź fileToRead.GetChar[];        -- the other @

                pos Ź fileToRead.GetIndex[];
                body.contents Ź SendMailOps.RopeFromStream[fileToRead, pos, bodyLen];
                [] Ź fileToRead.GetChar[ ! IO.EndOfStream => CONTINUE];        -- the trailing \n
                ok Ź TRUE;
                };

            FindStartOfEntry: PROC[fileToRead: STREAM]
                    RETURNS[startPos, prefixLength, entryLength: INT, entryChar: CHAR] = {
                ENABLE IO.EndOfStream => GOTO eoS;
                line: REF TEXT;
                relPos: INT Ź 0;
                startFound: BOOL;
                prefixLength Ź entryLength Ź 0;
                entryChar Ź '\000;        -- not a valid entryType char

                startPos Ź fileToRead.GetIndex[];
                IF fileToRead.EndOf[] THEN RETURN;
                line Ź fileToRead.GetLine[generalBuffer];
                [startFound, relPos] Ź IsStart[line];
                IF NOT startFound THEN
                    UNTIL startFound DO
                        IF fileToRead.EndOf[] THEN RETURN;
                        startPos Ź fileToRead.GetIndex[];
                        line Ź fileToRead.GetLine[generalBuffer];
                        [startFound, relPos] Ź IsStart[line];
                        ENDLOOP;

                    -- Read entry info line from log, e.g.: 00101 00029 US+
                    startPos Ź startPos + relPos;
                    entryLength Ź fileToRead.GetInt[];
                    prefixLength Ź fileToRead.GetInt[];
                    <<line _ fileToRead.GetLine[generalBuffer];>>
                    <<entryChar _ RefText.Fetch[line, RefText.Length[line]-1];>>
                    [] Ź fileToRead.GetChar[];        -- space
                    [] Ź fileToRead.GetChar[];        -- flag
                    [] Ź fileToRead.GetChar[];        -- flag
                    entryChar Ź fileToRead.GetChar[];        -- Laurel's march char
                    SELECT fileToRead.GetChar[] FROM
                        '\l, '\r => NULL;
                        ENDCASE => entryLength Ź -1;

                    EXITS
                        eoS => {entryLength Ź -1; RETURN};
                };

            starStartStar: ROPE = "*start*";
            IsStart: PROC[line: REF TEXT] RETURNS[startFound: BOOL, relPos: INT] = {
                relPos Ź 0;
                IF RefText.Length[line] = 0 THEN RETURN[FALSE, relPos];
                IF Rope.Equal[starStartStar, RefText.TrustTextAsRope[line]] THEN
                    RETURN[TRUE, relPos];
                IF (relPos Ź RefText.Length[line] - starStartStar.Length[]) <=0 THEN RETURN[FALSE, relPos];
                IF Rope.Find[s1: RefText.TrustTextAsRope[line], s2: starStartStar, pos1: relPos] = relPos THEN
                    RETURN[TRUE, relPos];

                RETURN[FALSE, relPos];
                };

        END.