
    <<WalnutUpdateImpl.mesa - reads the logStream, starting at startPos, & updates the database>>
        <<Copyright © 1984, 1985 by Xerox Corporation.  All rights reserved.>>
        <<Willie-Sue, March 20, 1985 8:52:03 am PST>>
        <<Russ Atkinson (RRA) March 21, 1985 0:50:05 am PST>>

    DIRECTORY
        DB USING [Value, DestroyEntity, Eq, GetF, GetP, GetPList, NameOf, Null, SetP, I2V, V2B, V2E, V2I, V2S],
        FS USING [Error, OpenFile, Create, OpenOrCreate, PagesForBytes, StreamFromOpenFile, StreamOpen],
        IO,
        PrincOpsUtils USING [PagesForBytes],
        Rope,
        WalnutControlPrivate USING [InternalConfirm],
        WalnutDB USING [Msg, MsgSet, deletedMsgSet, mCategoryIs, mHasBeenReadIs, mHeadersPos, mMsgLengthIs, msNumInSetIs, mPrefixPos, 
        mSubjectIs, AddMsgToMsgSet, DeclareMsg, DeclareMsgSet, DestroyMsgSet, MsgRecToMsg, RemoveMsgFromMsgSet, SetMsgHasBeenRead],
        WalnutDBLog USING [walnutInfoRelship, wStartExpungePos],
        WalnutVoice USING [CheckForWalnuthatch, RemoveInterestEntry],
        WalnutLog USING [minPrefixLength, MsgRec, MessageRecObject, copyBuffer, msgIDRope, ConstructMsgID, GenerateNewMsgID],
        WalnutLogExtras USING [onlyOneTempLog, walnutTempLogName],
        WalnutRetrieve USING [ParseMsgIntoFields],
        WalnutSendOps USING [RopeFromStream],
        WalnutStream USING [FindStartOfEntry, IsStart, MakeLogEntry, MsgRecFromStream, TagAndValue],
        WalnutWindow USING [walnutLogName, Report, ReportRope];

    WalnutUpdateImpl: CEDAR PROGRAM
        IMPORTS DB, FS, IO, PrincOpsUtils, Rope, WalnutControlPrivate, WalnutDB, WalnutDBLog, WalnutLog, WalnutLogExtras, WalnutRetrieve, 
        WalnutSendOps, WalnutStream, WalnutVoice, WalnutWindow

        EXPORTS WalnutStream =

        BEGIN OPEN DB, WalnutDB, WalnutLog, WalnutStream;

        ROPE: TYPE = Rope.ROPE;

    skipRope: ROPE_ "Unexpected EOF; trying for next entry";
    maxTailRewriteBytes: INT_ 1250000;        -- try this out

    BadCopyForExpunge: SIGNAL[numRead, numExpected: INT] = CODE;

    <<********************************************************>>
    UpdateFromStream: PUBLIC PROC [strm: IO.STREAM, startPos: INT] RETURNS[success: BOOL] =
        BEGIN
        doReport: BOOL_ (startPos = 0);        -- report if scavenging from beginning
        BadFormat: PROC[s: ROPE] =
        { IF doReport THEN
            WalnutWindow.Report[IO.PutFR
                ["Bad log file format at %g: %g", IO.int[strm.GetIndex[]], IO.rope[s]]]
        };

        BEGIN ENABLE IO.EndOfStream => {BadFormat["Unexpected EOF"]; GOTO GiveUp};
        entryLength, prefixLength, beginMsgPos: INT;
        msg: Msg;
        msgSet: MsgSet;
        entryChar: CHAR;
        count: INT_ 0;

        DomainUpdate: PROC RETURNS[BOOL] =
            BEGIN
                msName, tag: ROPE;
                [tag, msName]_ TagAndValue[strm, 2];
                IF NOT tag.Equal["name"] THEN {BadFormat["expected name"]; RETURN[FALSE]};
                WalnutWindow.ReportRope[Rope.FromChar[entryChar]];
                SELECT entryChar FROM
                '- =>
                    {msgSet_ DeclareMsgSet[msName, OldOnly].msgSet;
                        IF msgSet = NIL THEN RETURN[TRUE];
                        []_ DestroyMsgSet[msgSet]
                    };
                '+ => []_ DeclareMsgSet[msName];
                ENDCASE => ERROR;
                RETURN[TRUE];
        END;

        RelationUpdate: PROC RETURNS[BOOL] =
            BEGIN
            mName, categoryName, tag: ROPE;
            [tag, mName]_ TagAndValue[strm, 2];
            IF NOT tag.Equal["of"] THEN {BadFormat["of?"]; RETURN[FALSE]};
            msg_ DeclareMsg[mName, OldOnly].msg;
            IF msg=NIL THEN {BadFormat[Rope.Cat[mName, " doesn\'t exist!"]]; RETURN[FALSE]};
            [tag, categoryName]_ TagAndValue[strm, 2];
            IF NOT tag.Equal["is"] THEN {BadFormat["is?"]; RETURN[FALSE]}; 
            WalnutWindow.ReportRope[Rope.FromChar[entryChar]];
            SELECT entryChar FROM
                '- =>
                    { msgSet_ DeclareMsgSet[categoryName, OldOnly].msgSet;    -- check if msgSet exists
                        IF msgSet # NIL THEN []_ RemoveMsgFromMsgSet[msg, msgSet]
                    };
                '+ =>    --  create msgSet if it doesn't exist
                    []_ AddMsgToMsgSet[msg, DeclareMsgSet[categoryName].msgSet]
                ENDCASE => ERROR;
            RETURN[TRUE];
                END;

        ProcessHasBeenRead: PROC =
            BEGIN
                len: INT;
                mName: ROPE _ WalnutSendOps.RopeFromStream[strm, IO.GetIndex[strm], prefixLength-minPrefixLength
                    ! IO.EndOfStream => {
                        WalnutWindow.Report["\nUnexpected EOF, ignoring HasBeenRead entry"];
                        GOTO eof}
                    ];
                IF mName.Fetch[len_ mName.Length[]-1] = '\n THEN mName_ mName.Substr[0, len];
                msg_ DeclareMsg[mName, OldOnly].msg;
                IF msg = NIL THEN RETURN;    -- ignore
                    SetMsgHasBeenRead[msg];

                EXITS eof => RETURN;
        END;

        strm.SetIndex[startPos];
        DO
            <<Read *start* from log>>
            [beginMsgPos, prefixLength, entryLength, entryChar]_ FindStartOfEntry[strm, doReport];
            IF entryLength = -1 THEN RETURN[TRUE];
            IF (entryLength=0) OR (entryLength=prefixLength) AND 
                NOT (entryChar = '_) THEN RETURN[TRUE];
        <<Do delete, create, or message update to database >>
            SELECT entryChar FROM
                '_ => ProcessHasBeenRead[];    -- mark message as read

                '+, '- => -- add or remove relship
                    BEGIN
                        foo, domainOrRelation: ROPE;
                        ok: BOOL;
                        strm.SetIndex[beginMsgPos+prefixLength];        -- ignore prefix info
                        [domainOrRelation, foo] _ TagAndValue[strm, 2];

                        IF domainOrRelation.Equal["Domain"] THEN
                            { IF (ok_ foo.Equal["MsgSet"]) THEN ok_ DomainUpdate[]   -- Add or delete MsgSet
                                ELSE BadFormat[IO.PutFR["%g is not a valid domain", IO.rope[foo]]];
                                }
                            ELSE IF domainOrRelation.Equal["Relation"] THEN
                                { IF (ok_ foo.Equal["mCategory"]) THEN ok_ RelationUpdate[]    --AddTo or RemoveFrom
                                    ELSE BadFormat["expected mCategory"]
                                }
                                    ELSE -- domainOrRelation not "Domain" or "Relation"
                                {BadFormat["not domain or relation"]; ok_ FALSE};

                        IF ~ok THEN strm.SetIndex[beginMsgPos+entryLength]    -- next entry
                            ELSE IF strm.GetChar[]#'\n THEN BadFormat["Missing CR"];-- Skip over the trailing CR
                    END;

                ENDCASE => -- regular message entry
                    BEGIN
                        msgRec: MsgRec_
                            WalnutStream.MsgRecFromStream[strm, prefixLength, entryLength-prefixLength];
                            IF msgRec = NIL THEN
                                WalnutWindow.Report[
                                    IO.PutFR["\n Bad Entry at %g; skipping to next entry", IO.int[beginMsgPos]]]
                            ELSE 
                            { IF entryChar # '? THEN msgRec.hasBeenRead_ TRUE;
                                []_ MsgRecToMsg[msgRec];
                                count_ count + 1;
                                IF doReport THEN
                                    {IF count MOD 10 = 0 THEN
                                WalnutWindow.ReportRope[IF count MOD 100 = 0 THEN "!" ELSE "~"]}
                                ELSE WalnutWindow.ReportRope["."];
                                };
                        strm.SetIndex[beginMsgPos+entryLength];
                    END;
            ENDLOOP;

            EXITS GiveUp => RETURN[TRUE];        -- try to press on with what we have
            END;
        END;

    ExpungeFromStream: PUBLIC PROC[strm: IO.STREAM, doUpdates, tailRewrite: BOOL]
        RETURNS[startExpungePos: INT, ok: BOOL, tempLog: IO.STREAM] =
    <<Dumping is driven from the log file & preserves the bits that came from Grapevine>>
    <<Dumps all the Msgs in the database to a tempLog, setting all>>
    <<their body pointers to reference the NEW log, then copies tempLog to logStream>>
        BEGIN OPEN IO;
        newLen: INT;
        BEGIN ENABLE IO.EndOfStream => GOTO badLogFile;
        msg: Msg;

        toBeDestroyedList: LIST OF Msg;
        count: INT_ 0;
        expungeFilePos, startPos, prefixLength, entryLength: INT;
        entryChar: CHAR;
        numBad: INTEGER_ 0;
        DoCount: PROC =
            {IF (count_ count + 1) MOD 10 = 0 THEN
                WalnutWindow.ReportRope[IF count MOD 100 = 0 THEN "!" ELSE "~"]
                };
        WriteBadLogEntry: PROC[startPos: INT] RETURNS[nextStartPos: INT] =
            { errorStream: STREAM_ FS.StreamOpen["WalnutExpunge.errlog",
                IF (numBad_ numBad+1) = 1 THEN $create ELSE $append];
                line: ROPE;
                strm.SetIndex[startPos];
                errorStream.PutRope["\n******************************************************"];
                errorStream.PutRope[
                    IO.PutFR["\nBad entry from pos %g of log file, written at %g\n", int[startPos], time[]]];
                errorStream.PutRope[strm.GetLineRope[]];    -- *start* line
                errorStream.PutChar['\n];
                DO
                    nextStartPos_ strm.GetIndex[];
                    line_ strm.GetLineRope[];
                    IF IsStart[line].startFound THEN {nextStartPos_ strm.GetIndex[]-8; EXIT};
                    errorStream.PutRope[line];
                    errorStream.PutChar['\n];
                ENDLOOP;
                errorStream.SetLength[errorStream.GetIndex[]];
                errorStream.Close[];
            };
                CopyEntryIfNecessary: PROC =
                {  thisTag, thisValue: ROPE_ NIL;
                    thisMsg: Msg;
                        doCopy: BOOL_ TRUE;
                        prefixPos: INT;
                        IF entryChar = '_ THEN        -- hasbeenread entry
                            { -- strm.SetIndex[startPos+minPrefixLength];  where strm is pos'd
                                thisValue _ WalnutSendOps.RopeFromStream[strm, IO.GetIndex[strm], prefixLength-minPrefixLength-1];
                                []_ strm.GetChar[ ! IO.EndOfStream => CONTINUE]
                            }
                            ELSE
                        { strm.SetIndex[startPos+prefixLength];
                            [thisTag, thisValue]_ TagAndValue[strm, 2];
                                IF thisTag.Equal["Relation"]  AND thisValue.Equal["mCategory"] THEN
                                { [thisTag, thisValue]_ TagAndValue[strm, 2];
                                    IF ~thisTag.Equal["of"] THEN thisValue_ NIL;
                                };
                            };
                        IF thisValue # NIL THEN
                            { thisMsg_ DeclareMsg[thisValue, OldOnly].msg;
                                IF ~Null[thisMsg] THEN
                                IF (prefixPos_ V2I[GetP[thisMsg, mPrefixPos]]) >= startExpungePos THEN
                                    doCopy_ FALSE;
                            };
                        IF doCopy THEN
                            { numToDo: INT;
                                strm.SetIndex[startPos];
                    FOR numToDo_ entryLength, numToDo-512 UNTIL numToDo<512 DO
                        []_ strm.GetBlock[copyBuffer];
                        tempLog.PutBlock[copyBuffer];
                    ENDLOOP;
                    IF numToDo # 0 THEN
                        { numRead: INT_ strm.GetBlock[copyBuffer, 0, numToDo];
                            IF numRead # numToDo THEN SIGNAL BadCopyForExpunge[numRead, numToDo];
                            tempLog.PutBlock[copyBuffer, 0, numToDo];
                            copyBuffer.length_ 512;
                            };
                                };
                };

        logLength: INT_ strm.GetLength[];
        bytesInTail: INT;
        logIsOnAlpine: BOOL_ WalnutWindow.walnutLogName.Find[".alpine]", 0, FALSE] > 0;
        startExpungePos_
            IF tailRewrite THEN
                V2I[GetF[WalnutDBLog.walnutInfoRelship, WalnutDBLog.wStartExpungePos]] ELSE 0;

    <<if log file is on alpine AND tailRewrite AND startExpungePos#0 AND>>
    <<(loglength-startExpungePos)>1.25mb, THEN do full expunge anyway (and inform user)>>
        IF logIsOnAlpine AND tailRewrite AND startExpungePos#0 AND
            (bytesInTail_ logLength - startExpungePos) > maxTailRewriteBytes THEN
            { WalnutWindow.Report[IO.PutFR[
                "Tail of log to be expunged was %g bytes - doing full expunge", IO.int[bytesInTail]]];
                tailRewrite_ FALSE;
                startExpungePos_ 0;
                };

        IF startExpungePos = 0 THEN {tailRewrite_ FALSE; doUpdates_ FALSE};

        BEGIN ENABLE FS.Error => {WalnutWindow.Report[error.explanation]; GOTO quit};
        pgs: INT_ FS.PagesForBytes[IF tailRewrite THEN bytesInTail ELSE logLength/2];
        of: FS.OpenFile_
            IF WalnutLogExtras.onlyOneTempLog THEN
                FS.OpenOrCreate[name: WalnutLogExtras.walnutTempLogName, keep: 1, pages: pgs]
            ELSE FS.Create[name: WalnutLogExtras.walnutTempLogName, pages: pgs];
        tempLog_ FS.StreamFromOpenFile[of, $write];
        tempLog.SetIndex[0];        --  start at beginning!!!
        tempLog.SetLength[0];
        EXITS
            quit => NULL;
        END;
        IF tempLog = NIL THEN RETURN[startExpungePos, FALSE, NIL];

        strm.SetIndex[startExpungePos];

            DO            -- loop for dumping messages & making other log entries
            msgID: ROPE_ NIL;
            fullText: ROPE_ NIL;
            headersPos, curPos: INT;
            badEntry: BOOL_ FALSE;

        [startPos, prefixLength, entryLength, entryChar]_ FindStartOfEntry[strm, TRUE];
        IF entryLength = 0 THEN EXIT;
        IF (entryChar # ' ) AND (entryChar # '?) THEN    -- perhaps copy other entry to tempLog
            { IF startExpungePos # 0 THEN CopyEntryIfNecessary[];
                strm.SetIndex[startPos+entryLength];    -- for good measure, even after CopyEntry
                LOOP
                };

        headersPos_ startPos+prefixLength;
        UNTIL (curPos_ strm.GetIndex[]) = headersPos DO
            <<look for a msgID: field>>
                tag, value: ROPE;

                [tag, value]_ TagAndValue[strm: strm, inc: 2];
                IF Rope.Equal[tag, msgIDRope, FALSE] THEN msgID_ value;
                IF curPos > headersPos THEN -- bad entry we don't understand
                    { nextStartPos: INT_ WriteBadLogEntry[startPos];
                        cMsg: ROPE_ " Confirm to push on, deny to stop the Expunge"; 
                            WalnutWindow.ReportRope[PutFR["\nBad message entry at log pos %g;", int[startPos]]];
                        WalnutWindow.Report[" entry written on WalnutExpunge.errlog"];
                        WalnutWindow.Report[cMsg];
                        IF (ok_ WalnutControlPrivate.InternalConfirm[])
                            THEN {strm.SetIndex[nextStartPos]; badEntry_ TRUE;  EXIT};
                        RETURN[startExpungePos, FALSE, tempLog]};
            ENDLOOP;
                IF badEntry THEN LOOP;

            IF msgID = NIL THEN
                BEGIN        -- have to read fullText to fashion msgID
                mr: MsgRec_ NEW[MessageRecObject];
                []_ WalnutRetrieve.ParseMsgIntoFields[mr, strm, entryLength-prefixLength !
                    IO.EndOfStream =>
                        {WalnutWindow.Report[skipRope]; strm.SetIndex[startPos+entryLength]; LOOP}];
                mr.gvID_ ConstructMsgID[mr];
                mr.msgLength_ entryLength-prefixLength;
    <<need to make sure this is really a unique id for this message>>
    <<some msgs from old mail files are not>>
        IF ~CheckID[mr] THEN {strm.SetIndex[startPos+entryLength]; LOOP};
        strm.SetIndex[startPos+prefixLength];        -- back up stream
        msgID_ mr.gvID;
        END;

        msg_ DeclareMsg[msgID, OldOnly].msg;
        IF (msg=NIL) OR Null[msg] THEN {strm.SetIndex[startPos+entryLength]; LOOP}
        ELSE
            BEGIN
                curPrefixPos, thisPrefixLen, thisEntryLen: INT;
                thisMsgSet: MsgSet;
                nameOfCatsList: ROPE_ NIL;

    <<if is only in deletedMsgSet, don't put on tempLog>>
        FOR mL: LIST OF DB.Value_ GetPList[msg, mCategoryIs], mL.rest UNTIL mL=NIL DO
            IF ~Eq[thisMsgSet_ V2E[mL.first], deletedMsgSet] THEN
                nameOfCatsList_ Rope.Cat[nameOfCatsList, " ", DB.NameOf[thisMsgSet]];
        ENDLOOP;

        IF nameOfCatsList.Length[] = 0 THEN
            { toBeDestroyedList_ CONS[msg, toBeDestroyedList]; GOTO doneWithOne};
        msgID_ Rope.Cat[msgID, "\nCategories:", nameOfCatsList];

        IF fullText = NIL THEN
            fullText _ WalnutSendOps.RopeFromStream[strm, IO.GetIndex[strm], entryLength-prefixLength
                ! IO.EndOfStream => { WalnutWindow.Report[skipRope]; GOTO doneWithOne}
                ];

        DoCount[];
        [expungeFilePos, thisPrefixLen, thisEntryLen]_ MakeLogEntry[
            tempLog,
            IF V2B[GetP[msg, mHasBeenReadIs]] THEN message ELSE newMessage,
            fullText, Rope.Cat[msgIDRope, ": ", msgID, "\n"]];

        <<set header & body pos pointers to posn in NEW log if doing updates:>>
            IF doUpdates THEN
                { thisPrefixPos: INT_ startExpungePos + expungeFilePos - thisEntryLen;
                    curHeadersPos: INT_ V2I[GetP[msg, mHeadersPos]];
                    thisHeadersPos: INT_ thisPrefixPos+thisPrefixLen;
                    curPrefixPos_ V2I[GetP[msg, mPrefixPos]];
                        IF (curPrefixPos # thisPrefixPos) THEN []_ SetP[msg, mPrefixPos, I2V[thisPrefixPos]];
                        IF (curHeadersPos # thisHeadersPos) THEN
                            []_ SetP[msg, mHeadersPos, I2V[thisHeadersPos]];
                };


                EXITS
                    doneWithOne => strm.SetIndex[startPos+entryLength];
            END;        -- put message on log file
        ENDLOOP;        -- loop for dumping messages

        WalnutWindow.Report[PutFR["\nDumped %g messages from Log File... ", int[count]]];
        tempLog.SetLength[newLen_ tempLog.GetIndex[]];
        tempLog.Flush[];        -- make sure tempLog is in good shape
        WalnutWindow.Report[PutFR[
            "Old log length was %g bytes, (%g pages)", int[logLength],
                    int[PrincOpsUtils.PagesForBytes[logLength]]]]; 
        IF tailRewrite THEN
            { oldEnd: INT_ strm.GetLength[]-startExpungePos;
                fullLen: INT_ startExpungePos+newLen;
                WalnutWindow.Report[PutFR[
                    "  Previous tail length was %g bytes, (%g pages)",
                            int[oldEnd], int[PrincOpsUtils.PagesForBytes[oldEnd]]]];
                WalnutWindow.Report[PutFR[
                    "  New tail length is %g bytes (%g pages)", int[newLen],
                            int[PrincOpsUtils.PagesForBytes[newLen]]]];
                WalnutWindow.Report[PutFR[
                    "New log length is %g bytes, (%g pages)", int[fullLen],
                            int[PrincOpsUtils.PagesForBytes[fullLen]]]];
            }
        ELSE
            WalnutWindow.Report[PutFR
                ["New log length is %g bytes (%g messages, %g pages)", int[newLen], int[count],
                    int[PrincOpsUtils.PagesForBytes[newLen]]]];

        IF doUpdates THEN
            { nuh: REF ANY; 
                msg: Msg;
                id: ROPE;
                WalnutWindow.ReportRope[" Destroying Msgs in Deleted MsgSet ..."];

            <<may see a msg more than once in the log file, hence must check if already destroyed>>
                count_ 0;
                nuh_ WalnutVoice.CheckForWalnuthatch[];
                FOR mL: LIST OF Msg_ toBeDestroyedList, mL.rest UNTIL mL = NIL DO
                    IF ~Null[msg_ mL.first] THEN
                        { IF nuh#NIL THEN id_ NameOf[msg];
                            DestroyEntity[mL.first];        -- destroy the message first
                            IF nuh#NIL THEN WalnutVoice.RemoveInterestEntry[NIL, id];
                            DoCount[];
                        };
                ENDLOOP;
                WalnutWindow.Report[PutFR["\n%g Msgs destroyed",int[count]]];
                BEGIN            -- fix up num in deleted (not necessarily zero)
                    num: INT_ V2I[GetP[deletedMsgSet, msNumInSetIs]] - count;
                    []_ SetP[deletedMsgSet, msNumInSetIs, I2V[num]];
                END;
            };        -- end doUpdates

        RETURN[startExpungePos, TRUE, tempLog];

        EXITS
            badLogFile =>
                { tempLog.SetLength[tempLog.GetIndex[]]; 
                    tempLog.Close[];
                    RETURN[startExpungePos, FALSE, NIL]
                };
            END;
        END;

    CheckID: PROC[mr: MsgRec] RETURNS[useThisMsg: BOOL] =
        BEGIN
            len: INT;
            subj: ROPE;
            msg: Msg;
            DO
                msg_ DeclareMsg[mr.gvID, OldOnly].msg;
                IF (msg=NIL) OR Null[msg] THEN RETURN[FALSE];
                len_ V2I[GetP[msg, mMsgLengthIs]];
                IF len # mr.msgLength THEN { GenerateNewMsgID[mr]; LOOP};
                subj_ V2S[GetP[msg, mSubjectIs]];
                IF ~Rope.Equal[subj, mr.subject] THEN { GenerateNewMsgID[mr]; LOOP};
                RETURN[TRUE];
            ENDLOOP;
        END;

    END.