-- File: WalnutUpdateImpl.mesa
-- Contents:
-- reads the logStream, starting at startPos, & makes the appropriate database updates

-- Created by: Willie-Sue on April 27, 1983
-- Last edited by:
-- Willie-Sue on May 31, 1983 3:55 pm

DIRECTORY
 DB,
 FileIO,
 IO,
 Rope,
 RopeIO,
 Runtime,
 WalnutDB,
 WalnutDBLog,
 WalnutLog,
 WalnutRetrieve,
 WalnutStream,
 WalnutWindow;
 
WalnutUpdateImpl: CEDAR PROGRAM

IMPORTS DB, FileIO, IO, Rope, RopeIO,
 WalnutDB, WalnutDBLog, WalnutLog, WalnutRetrieve,
 WalnutStream, WalnutWindow

EXPORTS WalnutStream =

BEGIN OPEN DB, WalnutDB, WalnutDBLog, WalnutLog, WalnutStream, WalnutWindow;

skipRope: ROPE← "Unexpected EOF; trying for next entry";

-- ********************************************************
UpdateFromStream: PUBLIC PROC [strm: IO.STREAM, startPos: INT] RETURNS[success: BOOL] =
BEGIN
doReport: BOOL← (startPos = 0);  -- report if scavenging from beginning
BadFormat: PROC[s: ROPE] =
{ IF doReport THEN
  Report[IO.PutFR["Bad log file format at %g: %g", IO.int[strm.GetIndex[]], IO.rope[s]]]
};

BEGIN ENABLE IO.EndOfStream => {BadFormat["Unexpected EOF"]; GOTO GiveUp};
entryLength, prefixLength, beginMsgPos: INT;
msg: Msg;
msgSet: MsgSet;
entryChar: CHAR;

DomainUpdate: PROC RETURNS[BOOL] =
BEGIN
  msName, tag: ROPE;
  [tag, msName]← TagAndValue[strm, 2];
  IF NOT tag.Equal["name"] THEN {BadFormat["expected name"]; RETURN[FALSE]};

  ReportRope[Rope.FromChar[entryChar]];
  SELECT entryChar FROM
'- =>
{msgSet← DeclareMsgSet[msName, OldOnly].msgSet;
IF msgSet = NIL THEN RETURN[TRUE];
[]← DestroyMsgSet[msgSet]
};
'+ => []← DeclareMsgSet[msName];
ENDCASE => ERROR;
RETURN[TRUE];
END;

RelationUpdate: PROC RETURNS[BOOL] =
 BEGIN
mName, categoryName, tag: ROPE;
[tag, mName]← TagAndValue[strm, 2];
IF NOT tag.Equal["of"] THEN {BadFormat["of?"]; RETURN[FALSE]};
msg← DeclareMsg[mName, OldOnly].msg;
IF msg=NIL THEN {BadFormat[Rope.Cat[mName, " doesn\'t exist!"]]; RETURN[FALSE]};
[tag, categoryName]← TagAndValue[strm, 2];
IF NOT tag.Equal["is"] THEN {BadFormat["is?"]; RETURN[FALSE]};
ReportRope[Rope.FromChar[entryChar]];
SELECT entryChar FROM
'- =>
{ msgSet← DeclareMsgSet[categoryName, OldOnly].msgSet; -- check if msgSet exists
IF msgSet # NIL THEN []← RemoveMsgFromMsgSet[msg, msgSet]
};
'+ => -- create msgSet if it doesn't exist
 []← AddMsgToMsgSet[msg, DeclareMsgSet[categoryName].msgSet]
ENDCASE => ERROR;
RETURN[TRUE];
 END;

ProcessHasBeenRead: PROC =
 BEGIN
  len: INT;
  mName: ROPE← RopeIO.GetRope[strm, prefixLength-minPrefixLength ! IO.EndOfStream =>
  { Report["\nUnexpected EOF, ignoring HasBeenRead entry"]; GOTO eof}];
  IF mName.Fetch[len← mName.Length[]-1] = '\n THEN mName← mName.Substr[0, len];
  msg← DeclareMsg[mName, OldOnly].msg;
  IF msg = NIL THEN RETURN; -- ignore
  SetMsgHasBeenRead[msg];

  EXITS eof => RETURN;
END;

strm.SetIndex[startPos];
DO
-- Read *start* from log
[beginMsgPos, prefixLength, entryLength, entryChar]← FindStartOfEntry[strm, doReport];
IF entryLength = -1 THEN RETURN[TRUE];
IF (entryLength=0) OR (entryLength=prefixLength) AND
NOT (entryChar = '←) THEN RETURN[TRUE];
-- Do delete, create, or message update to database
SELECT entryChar FROM
'← => ProcessHasBeenRead[]; -- mark message as read

'+, '- => -- add or remove relship
BEGIN
foo, domainOrRelation: ROPE;
ok: BOOL;
strm.SetIndex[beginMsgPos+prefixLength];  -- ignore prefix info
[domainOrRelation, foo] ← TagAndValue[strm, 2];

IF domainOrRelation.Equal["Domain"] THEN
{ IF (ok← foo.Equal["MsgSet"]) THEN ok← DomainUpdate[] -- Add or delete MsgSet
ELSE BadFormat[IO.PutFR["%g is not a valid domain", IO.rope[foo]]];
}
ELSE IF domainOrRelation.Equal["Relation"] THEN
{ IF (ok← foo.Equal["mCategory"]) THEN ok← RelationUpdate[] --AddTo or RemoveFrom
 ELSE BadFormat["expected mCategory"]
}
   ELSE -- domainOrRelation not "Domain" or "Relation"
{BadFormat["not domain or relation"]; ok← FALSE};

IF ~ok THEN strm.SetIndex[beginMsgPos+entryLength] -- next entry
ELSE IF strm.GetChar[]#'\n THEN BadFormat["Missing CR"];-- Skip over the trailing CR
END;

ENDCASE => -- regular message entry
BEGIN
msgRec: MsgRec←
 WalnutStream.MsgRecFromStream[strm, prefixLength, entryLength-prefixLength];
IF entryChar # '? THEN msgRec.hasBeenRead← TRUE;
[]← MsgRecToMsg[msgRec];
ReportRope["."];
strm.SetIndex[beginMsgPos+entryLength];
END;
ENDLOOP;

EXITS GiveUp => RETURN[TRUE];  -- try to press on with what we have
END;
END;

ExpungeFromStream: PUBLIC PROC[strm, tempLog: IO.STREAM, doUpdates, tailRewrite: BOOL]
  RETURNS[startExpungePos: INT, ok: BOOL] =
-- Dumping is driven from the log file & preserves the bits that came from Grapevine
-- Dumps all the Msgs in the database to a tempLog, setting all
-- their body pointers to reference the NEW log, then copies tempLog to logStream
BEGIN OPEN IO;
newLen: INT;
BEGIN ENABLE IO.EndOfStream => GOTO badLogFile;
msg: Msg;

toBeDestroyedList: LIST OF Msg;
count: INT← 0;
expungeFilePos, startPos, prefixLength, entryLength: INT;
entryChar: CHAR;
numBad: INTEGER← 0;
DoCount: PROC =
{IF (count← count + 1) MOD 10 = 0 THEN
 ReportRope[IF count MOD 100 = 0 THEN "!" ELSE "~"]
};
WriteBadLogEntry: PROC[startPos: INT] RETURNS[nextStartPos: INT] =
 { errorStream: STREAM← FileIO.Open["WalnutExpunge.errlog", write];
  line: ROPE;
  IF (numBad← numBad+1) = 1 THEN errorStream.SetIndex[0]
  ELSE errorStream.SetIndex[errorStream.GetLength[]];
  strm.SetIndex[startPos];
  errorStream.PutRope[
   IO.PutFR["\n Bad entry from pos %g of log file, written at %g\n", int[startPos], time[]]];
  errorStream.PutRope[strm.GetSequence[]]; -- *start* line
  errorStream.PutChar[strm.GetChar[]];  -- CR
  DO
  nextStartPos← strm.GetIndex[];
  line← strm.GetSequence[];
  IF line.Equal["*start*"] THEN {nextStartPos← strm.GetIndex[]-8; EXIT};
  errorStream.PutRope[line];
  errorStream.PutChar[strm.GetChar[]];
  ENDLOOP;
  errorStream.SetLength[errorStream.GetIndex[]];
  errorStream.Close[];
};
 CopyEntryIfNecessary: PROC =
 { thisTag, thisValue: ROPE← NIL;
  thisMsg: Msg;
  doCopy: BOOL← TRUE;
  prefixPos: INT;
  IF entryChar = '← THEN  -- hasbeenread entry
   { -- strm.SetIndex[startPos+minPrefixLength]; where strm is pos'd
   thisValue← RopeIO.GetRope[strm, prefixLength-minPrefixLength-1];
   []← strm.GetChar[ ! IO.EndOfStream => CONTINUE]
   }
   ELSE
  { strm.SetIndex[startPos+prefixLength];
   [thisTag, thisValue]← TagAndValue[strm, 2];
   IF thisTag.Equal["Relation"] AND thisValue.Equal["mCategory"] THEN
   { [thisTag, thisValue]← TagAndValue[strm, 2];
   IF ~thisTag.Equal["of"] THEN thisValue← NIL;
   };
   };
  IF thisValue # NIL THEN
   { thisMsg← DeclareMsg[thisValue, OldOnly].msg;
   IF ~Null[thisMsg] THEN
   IF (prefixPos← V2I[GetP[thisMsg, mPrefixPos]]) >= startExpungePos THEN
     doCopy← FALSE;
   };
  IF doCopy THEN
   { numToDo: INT;
   strm.SetIndex[startPos];
FOR numToDo← entryLength, numToDo-512 UNTIL numToDo<512 DO
[]← tempLog.GetBlock[copyBuffer];
strm.PutBlock[copyBuffer];
ENDLOOP;
IF numToDo # 0 THEN
{ copyBuffer.length← numToDo;
[]← tempLog.GetBlock[copyBuffer];
strm.PutBlock[copyBuffer];
copyBuffer.length← 512;
};
   };
 };

tempLog.SetIndex[0];
startExpungePos←
IF tailRewrite THEN V2I[GetF[walnutInfoRelship, wStartExpungePos]] ELSE 0;
strm.SetIndex[startExpungePos];

DO   -- loop for dumping messages & making other log entries
msgID: ROPE← NIL;
fullText: ROPE← NIL;
headersPos, curPos: INT;
badEntry: BOOL← FALSE;

[startPos, prefixLength, entryLength, entryChar]← FindStartOfEntry[strm, TRUE];
IF entryLength = 0 THEN EXIT;
IF (entryChar # ' ) AND (entryChar # '?) THEN -- perhaps copy other entry to tempLog
{ IF startExpungePos # 0 THEN CopyEntryIfNecessary[]
 ELSE strm.SetIndex[startPos+entryLength];
LOOP
};

headersPos← startPos+prefixLength;
UNTIL (curPos← strm.GetIndex[]) = headersPos DO
-- look for a msgID: field
  tag, value: ROPE;

[tag, value]← TagAndValue[strm: strm, inc: 2];
IF Rope.Equal[tag, msgIDRope, FALSE] THEN msgID← value;
IF curPos > headersPos THEN -- bad entry we don't understand
 { nextStartPos: INT← WriteBadLogEntry[startPos];
  cMsg: ROPE← " Confirm to push on, deny to stop the Expunge";
  ReportRope[PutFR["\nBad message entry at log pos %g;", int[startPos]]];
  Report[" entry written on WalnutExpunge.errlog"];
  Report[cMsg];
  IF (ok← UserConfirmed[])
    THEN {strm.SetIndex[nextStartPos]; badEntry← TRUE; EXIT};
  RETURN[startExpungePos, FALSE]};
ENDLOOP;
 IF badEntry THEN LOOP;

IF msgID = NIL THEN
 BEGIN  -- have to read fullText to fashion msgID
mr: MsgRec← NEW[MessageRecObject];
[]← WalnutRetrieve.ParseMsgIntoFields[mr, strm, entryLength-prefixLength !
 IO.EndOfStream => {Report[skipRope]; strm.SetIndex[startPos+entryLength]; LOOP}];
msgID← ConstructMsgID[mr];
strm.SetIndex[startPos+prefixLength];  -- back up stream
 END;

msg← DeclareMsg[msgID, OldOnly].msg;
IF (msg=NIL) OR Null[msg] THEN {strm.SetIndex[startPos+entryLength]; LOOP}
ELSE
BEGIN
curPrefixPos: INT;
thisMsgSet: MsgSet;
nameOfCatsList: ROPE← NIL;

-- if is only in deletedMsgSet, don't put on tempLog
  FOR mL: LIST OF DB.Value← GetPList[msg, mCategoryIs], mL.rest UNTIL mL=NIL DO
  IF ~Eq[thisMsgSet← V2E[mL.first], deletedMsgSet] THEN
  nameOfCatsList← Rope.Cat[nameOfCatsList, " ", DB.GetName[thisMsgSet]];
  ENDLOOP;

  IF nameOfCatsList.Length[] = 0 THEN
  { toBeDestroyedList← CONS[msg, toBeDestroyedList]; GOTO doneWithOne};
  msgID← Rope.Cat[msgID, "\nCategories:", nameOfCatsList];

IF fullText = NIL THEN fullText←
RopeIO.GetRope[strm, entryLength-prefixLength ! IO.EndOfStream =>
{Report[skipRope]; GOTO doneWithOne}];

DoCount[];
expungeFilePos← MakeLogEntry[
   tempLog,
 IF V2B[GetP[msg, mHasBeenReadIs]] THEN message ELSE newMessage,
 fullText, Rope.Cat[msgIDRope, ": ", msgID, "\n"]];

-- set header & body pos pointers to posn in NEW log if doing updates:
 IF doUpdates THEN
  { thisPrefixPos: INT← startExpungePos + expungeFilePos - entryLength;
  curPrefixPos← V2I[GetP[msg, mPrefixPos]];
  IF curPrefixPos # thisPrefixPos THEN
  { []← SetP[msg, mHeadersPos, I2V[thisPrefixPos+prefixLength]];
   []← SetP[msg, mPrefixPos, I2V[thisPrefixPos]];
  };
  };


 EXITS
  doneWithOne => strm.SetIndex[startPos+entryLength];
END;  -- put message on log file
ENDLOOP;  -- loop for dumping messages

Report[PutFR["\nDumped %g messages from Log File... ", int[count]]];
tempLog.SetLength[newLen← tempLog.GetIndex[]];
tempLog.Flush[];  -- make sure tempLog is in good shape
Report[PutFR["Old log length was %g bytes", int[strm.GetLength[]]]];
IF tailRewrite THEN
{ Report[PutFR
  ["Previous end of log was %g bytes", int[strm.GetLength[]-startExpungePos]]];
Report[PutFR["New end of log file is %g bytes (%g messages)", int[newLen], int[count]]];
Report[PutFR["New log length is %g bytes", int[startExpungePos+newLen]]];
}
ELSE
{ Report[PutFR["Old log length was %g bytes", int[strm.GetLength[]]]];
Report[PutFR["New log length is %g bytes (%g messages)", int[newLen], int[count]]]
};

IF doUpdates THEN
 { ReportRope[" Destroying Msgs in Deleted MsgSet ..."];
-- may see a msg more than once in the log file, hence must check if already destroyed
  count← 0;
  FOR mL: LIST OF Msg← toBeDestroyedList, mL.rest UNTIL mL = NIL DO
  IF ~Null[mL.first] THEN
   { DestroyEntity[mL.first];  -- destroy the message
   DoCount[];
   };
  ENDLOOP;
  Report[PutFR["\n%g Msgs destroyed",int[count]]];
 BEGIN   -- fix up num in deleted (should be 0 but who knows)
  num: INT← V2I[GetP[deletedMsgSet, msNumInSetIs]] - count;
  []← SetP[deletedMsgSet, msNumInSetIs, I2V[num]];
 END;
};  -- end doUpdates

RETURN[startExpungePos, TRUE];

EXITS
badLogFile =>
{ tempLog.SetLength[tempLog.GetIndex[]];
tempLog.Close[];
RETURN[startExpungePos, FALSE]
};
END;
END;

END.