<> <> <> <<>> <> <<>> DIRECTORY Basics USING [UnsafeBlock], BasicTime USING [GMT, Now, nullGMT, Period, Update], Camelot USING [btidT, tidT], IO USING [EndOfStream, GetChar, GetLength, GetRope, STREAM, UnsafeGetBlock], PBasics USING [charsPerWord], Process USING [Detach, MsecToTicks, Pause, SetTimeout, Ticks], RedBlackTree USING [Compare, Create, Delete, GetKey, Insert, InsertNode, Lookup, Node, Size, Table, UserData], RefText USING [ObtainScratch, ReleaseScratch], Rope USING [Concat, Compare, Fetch, FromRefText, IsEmpty, ROPE], SafeStorage USING [CantEstablishFinalization, EnableFinalization, EstablishFinalization, FinalizationQueue, FQNext, NewFQ], YggDID USING [DID], YggDIDPrivate USING [DIDRep], YggDIDMap USING [GetComponentFiles, OpenDocumentFromDID], YggEnvironment USING [AccessRights, LockMode, LockOption, nullDID, nullTransID, TransID], YggFileStream USING [StreamFromComponentFilesAndTid], YggFixedNames USING [AutoIndices, ChildContainers, Children, Contents, Inlinks, Outlinks, ParentContainers, Parents], YggInternal USING [Document, FileHandle], YggLock USING [Failed, MakeLockID, Set], YggMonitoringLog USING [LockConflictInfo, notice], YggRep USING [AccurateGMT, AccurateGMTRep, AccurateGMTRepByteSize, Attribute, AttributePreambleByte, AttributeValue, Bits, BitsFromBits, BitsRep, date, float, did, int, metaAttributeMod, noValue, rope, shortRope, TypedPrimitiveElement, unknown, uninterpretedBytes, VDoc, VDocRep], YggTransaction USING [EqualTrans, GetParent, IsNullTrans, NotePossibleDocumentUpdate], YggVolatileObjectCache; YggVolatilizeImpl: CEDAR MONITOR IMPORTS BasicTime, IO, Process, RedBlackTree, RefText, Rope, SafeStorage, YggDIDMap, YggFileStream, YggFixedNames, YggLock, YggMonitoringLog, YggRep, YggTransaction EXPORTS YggDID, YggRep, YggVolatileObjectCache ~ BEGIN ROPE: TYPE ~ Rope.ROPE; scratchBuffSize: INT = 128; didCache: RedBlackTree.Table; desiredSizeOfDIDCache: INT _ 500; -- move this up after debuggng cacheEntry: TYPE ~ REF cacheEntryRep; cacheEntryRep: TYPE ~ RECORD [ users: INT _ 0, timeOfLastVolatize: BasicTime.GMT _ BasicTime.nullGMT, vDoc: YggRep.VDoc _ NIL ]; myCondition: CONDITION; latchCondition: CONDITION; aTickCondition: CONDITION; DID: PUBLIC TYPE ~ REF DIDRep; DIDRep: PUBLIC TYPE ~ YggDIDPrivate.DIDRep; Now: BasicTime.GMT _ BasicTime.Now[]; EmptyRope: ROPE _ ""; ListOfFinalizedVDocs: LIST OF YggRep.VDoc _ NIL; ListOfFinalizedVDocsLast: LIST OF YggRep.VDoc _ NIL; DoubleVDoc: TYPE = RECORD [ aOne: YggRep.VDoc, andATwo: YggRep.VDoc ]; VDocsAllocList: LIST OF DoubleVDoc _ NIL; MyQueue: SafeStorage.FinalizationQueue = SafeStorage.NewFQ[length: 200]; <> VolatizeFromDID: PUBLIC PROC [transID: YggEnvironment.TransID, did: YggDID.DID, access: YggEnvironment.AccessRights, lock: YggEnvironment.LockOption, metaAttributesOnly: BOOL] RETURNS [vDoc: YggRep.VDoc _ NIL] ~ { <> parseAttributes: PROC [attrStream: IO.STREAM] RETURNS [attributeList: LIST OF YggRep.Attribute _ NIL, contentsAttribute: YggRep.TypedPrimitiveElement _ [YggRep.unknown, NIL], outlinksAttribute: LIST OF YggRep.AttributeValue _ NIL, inlinksAttribute: LIST OF YggRep.AttributeValue _ NIL, metaAttributes: LIST OF YggRep.Attribute _ NIL] = { readInt: PROC RETURNS [int: INT] = { <> charsRead: INT _ 0; rint: REF INT; rint _ NEW[INT]; TRUSTED {charsRead _ IO.UnsafeGetBlock[attrStream, [LOOPHOLE[rint], 0, BYTES[INT]]];}; IF charsRead # BYTES[INT] THEN ERROR; RETURN[rint^]; }; readString: PROC [extraChars: INT] RETURNS [string: ROPE _ NIL]= { <> nullFound: BOOL _ FALSE; lenInScratch: INT _ 0; numberToFetch: INT _ PBasics.charsPerWord-extraChars; fourChars: REF PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR; scratch: REF TEXT = RefText.ObtainScratch[scratchBuffSize]; fourChars _ NEW[PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR]; UNTIL nullFound DO charsRead: INT _ 0; TRUSTED {charsRead _ IO.UnsafeGetBlock[attrStream, [LOOPHOLE[fourChars], 0 , numberToFetch]];}; IF charsRead # numberToFetch THEN ERROR; FOR charNo: INT IN [0..numberToFetch) DO IF fourChars[charNo] = 0C THEN {nullFound _ TRUE; EXIT}; scratch[lenInScratch] _ fourChars[charNo]; lenInScratch _ lenInScratch + 1; IF lenInScratch = scratchBuffSize THEN { scratch.length _ lenInScratch; string _ Rope.Concat[string, Rope.FromRefText[s: scratch, start: 0, len: lenInScratch]]; lenInScratch _ 0; }; ENDLOOP; numberToFetch _ PBasics.charsPerWord; ENDLOOP; IF lenInScratch # 0 THEN { scratch.length _ lenInScratch; string _ Rope.Concat[string, Rope.FromRefText[s: scratch, start: 0, len: lenInScratch]]; }; RefText.ReleaseScratch[scratch]; }; doneWithAttributes: BOOL _ FALSE; attributeListLast: LIST OF YggRep.Attribute _ NIL; metaAttributeListLast: LIST OF YggRep.Attribute _ NIL; contentsDocType _ readInt[]; DO -- for each attribute attributeName: ROPE _ NIL; -- attribute name attributeValue: LIST OF YggRep.AttributeValue _ NIL; -- attribute name attributeValueLast: LIST OF YggRep.AttributeValue _ NIL; -- attribute name numberOfAttributeValues: INT _ 1; preampleByte: CHAR; preample: YggRep.AttributePreambleByte; fieldDocType: CARD; TRUSTED {preampleByte _ IO.GetChar[attrStream ! IO.EndOfStream => { doneWithAttributes _ TRUE; CONTINUE; }; ];}; -- @preample points to preample[0] IF doneWithAttributes THEN EXIT; preample _ LOOPHOLE[preampleByte]; fieldDocType _ SELECT preample.typeCode FROM integer => YggRep.int, ropeLarge => YggRep.rope, ropeShort => YggRep.shortRope, float => YggRep.float, date => YggRep.date, did => YggRep.did, uninterpretedBytes => YggRep.uninterpretedBytes, ENDCASE => YggRep.unknown; attributeName _ readString[1]; -- read attribute name IF ~preample.singletonAttribute THEN { numberOfAttributeValues _ readInt[]; }; FOR attVal: INT IN [0..numberOfAttributeValues) DO -- for each attribute value (a field) fieldName: ROPE _ NIL; tpeCount: INT _ 1; valueSet: LIST OF YggRep.TypedPrimitiveElement _ NIL; -- set of values for field valueSetLast: LIST OF YggRep.TypedPrimitiveElement _ NIL; -- set of values for field IF ~preample.noFieldNames THEN { fieldName _ readString[0]; -- read field name }; IF ~preample.singletonField THEN { tpeCount _ readInt[]; -- read field name }; FOR tpeNo: INT IN [0..tpeCount) DO -- for each typed primitive element in the field's value set fieldValue: YggRep.Bits; IF preample.typeCode = separate THEN { fieldDocType _ readInt[]; }; SELECT fieldDocType FROM YggRep.unknown => EXIT; YggRep.int => { i: INT32; ri: REF INT32; i _ readInt[]; ri _ NEW[INT32 _ i]; fieldValue _ ri; }; YggRep.shortRope => { rRope: ROPE; rRope _ readString[0]; fieldValue _ rRope; }; YggRep.rope => { len: INT; lenToRead: INT; charsRead: INT; rRope: ROPE; scratch: REF TEXT; len _ readInt[]; lenToRead _ PBasics.charsPerWord * ((len - 1 + PBasics.charsPerWord)/PBasics.charsPerWord); scratch _ RefText.ObtainScratch[lenToRead]; TRUSTED {charsRead _ IO.UnsafeGetBlock[attrStream, [LOOPHOLE[scratch, LONG POINTER]+UNITS[TEXT[0]], 0 , lenToRead]];}; IF charsRead # lenToRead THEN ERROR; scratch.length _ len; rRope _ Rope.FromRefText[s: scratch, start: 0, len: len]; fieldValue _ rRope; }; YggRep.float => { rReal: REF REAL32; rReal _ NEW[REAL32]; TRUSTED { charsRead: INT; ptReal: LONG POINTER TO REAL32; ptReal _ LOOPHOLE[rReal]; charsRead _ IO.UnsafeGetBlock[attrStream, [LOOPHOLE[ptReal], 0, BYTES[REAL32]]]; IF charsRead # BYTES[REAL32] THEN ERROR; }; fieldValue _ rReal; }; YggRep.date => { rAccurateGMT: YggRep.AccurateGMT; rAccurateGMT _ NEW[YggRep.AccurateGMTRep]; TRUSTED { charsRead: INT; ptAGMT: LONG POINTER TO YggRep.AccurateGMTRep; ptAGMT _ LOOPHOLE[rAccurateGMT]; charsRead _ IO.UnsafeGetBlock[attrStream, [LOOPHOLE[ptAGMT], 0, YggRep.AccurateGMTRepByteSize]]; IF charsRead # YggRep.AccurateGMTRepByteSize THEN ERROR; }; fieldValue _ rAccurateGMT; }; YggRep.did => { rDID: DID; rDID _ NEW[DIDRep]; TRUSTED { charsRead: INT; ptDID: LONG POINTER TO DIDRep; ptDID _ LOOPHOLE[rDID]; charsRead _ IO.UnsafeGetBlock[attrStream, [LOOPHOLE[ptDID], 0, BYTES[DIDRep]]]; IF charsRead # BYTES[DIDRep] THEN ERROR; }; fieldValue _ rDID; }; ENDCASE => { charsRead: INT; junk: REF PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR; len: INT; lenToRead: INT; nullsToRead: INT; rBits: REF YggRep.BitsRep; len _ readInt[]; lenToRead _ PBasics.charsPerWord * ((len - 1 + PBasics.charsPerWord)/PBasics.charsPerWord); rBits _ NEW[YggRep.BitsRep[len]]; rBits.validBytes _ len; TRUSTED { ptBits: LONG POINTER TO YggRep.BitsRep; ptBits _ LOOPHOLE[rBits]; charsRead _ IO.UnsafeGetBlock[attrStream, [LOOPHOLE[ptBits, LONG POINTER] + UNITS[YggRep.BitsRep[0]], 0, len]]; IF charsRead # len THEN ERROR; }; nullsToRead _ lenToRead - len; IF nullsToRead > 0 THEN TRUSTED { junk _ NEW[PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR]; charsRead _ IO.UnsafeGetBlock[attrStream, [LOOPHOLE[junk], 0 , nullsToRead]]; IF charsRead # nullsToRead THEN ERROR; }; fieldValue _ rBits; }; IF valueSet = NIL THEN valueSetLast _ valueSet _ CONS[[fieldDocType, fieldValue], NIL] ELSE valueSetLast _ (valueSetLast.rest _ CONS[[fieldDocType, fieldValue], NIL]); IF preample.singletonField THEN EXIT; ENDLOOP; -- for each typed primitive element in the field's value set IF attributeValue = NIL THEN attributeValueLast _ attributeValue _ CONS[[fieldName, valueSet], NIL] ELSE attributeValueLast _ (attributeValueLast.rest _ CONS[[fieldName, valueSet], NIL]); ENDLOOP; -- for each attribute value SELECT TRUE FROM Rope.Compare[YggFixedNames.Contents, attributeName, FALSE] = equal => { <> IF attributeValue = NIL THEN ERROR; IF attributeValue.first.valueSet = NIL THEN ERROR; IF attributeValue.rest # NIL THEN ERROR; IF attributeValue.first.valueSet = NIL THEN ERROR; IF attributeValue.first.valueSet.rest # NIL THEN ERROR; contentsAttribute _ attributeValue.first.valueSet.first; }; Rope.Compare[YggFixedNames.Outlinks, attributeName, FALSE] = equal => { <> FOR linkList: LIST OF YggRep.AttributeValue _ attributeValue, linkList.rest UNTIL linkList = NIL DO FOR vS: LIST OF YggRep.TypedPrimitiveElement _ linkList.first.valueSet, vS.rest UNTIL vS = NIL DO IF vS.first.docType # YggRep.did THEN ERROR; ENDLOOP; ENDLOOP; outlinksAttribute _ attributeValue; }; Rope.Compare[YggFixedNames.Inlinks, attributeName, FALSE] = equal => { <> FOR linkList: LIST OF YggRep.AttributeValue _ attributeValue, linkList.rest UNTIL linkList = NIL DO FOR vS: LIST OF YggRep.TypedPrimitiveElement _ linkList.first.valueSet, vS.rest UNTIL vS = NIL DO IF vS.first.docType # YggRep.did THEN ERROR; ENDLOOP; ENDLOOP; inlinksAttribute _ attributeValue; }; ENDCASE => { metaAttr: BOOL _ FALSE; IF ~Rope.IsEmpty[attributeName] AND Rope.Fetch[attributeName, 0] = '$ THEN { -- maybe a meta attribute SELECT TRUE FROM Rope.Compare[YggFixedNames.Parents, attributeName, FALSE] = equal => metaAttr _ TRUE; Rope.Compare[YggFixedNames.Children, attributeName, FALSE] = equal => metaAttr _ TRUE; Rope.Compare[YggFixedNames.ParentContainers, attributeName, FALSE] = equal => metaAttr _ TRUE; Rope.Compare[YggFixedNames.ChildContainers, attributeName, FALSE] = equal => metaAttr _ TRUE; Rope.Compare[YggFixedNames.AutoIndices, attributeName, FALSE] = equal => metaAttr _ TRUE; ENDCASE; }; IF metaAttr THEN { IF metaAttributes = NIL THEN metaAttributeListLast _ metaAttributes _ CONS[[attributeName, preample.ordered, attributeValue], NIL] ELSE metaAttributeListLast _ (metaAttributeListLast.rest _ CONS[[attributeName, preample.ordered, attributeValue], NIL]); } ELSE { IF attributeList = NIL THEN attributeListLast _ attributeList _ CONS[[attributeName, preample.ordered, attributeValue], NIL] ELSE attributeListLast _ (attributeListLast.rest _ CONS[[attributeName, preample.ordered, attributeValue], NIL]); }; }; ENDLOOP; -- for each attribute }; contents, attributes, meta: IO.STREAM _ NIL; contentsDocType: INT _ -1; doc: YggInternal.Document _ NIL; componentFiles: LIST OF YggInternal.FileHandle; effectiveTransID: YggEnvironment.TransID; vDocs: LIST OF DoubleVDoc _ NIL; IF ~metaAttributesOnly THEN { [] _ YggLock.Set[ trans: transID, lock: YggLock.MakeLockID[did], mode: lock.mode, wait: lock.ifConflict=wait ! YggLock.Failed => { -- Log the error and let the error propagate. logProc: PROC [YggMonitoringLog.LockConflictInfo]; IF (logProc _ YggMonitoringLog.notice.lockConflict) # NIL THEN logProc[[ what: why, -- (conflict or timeout) where: "YggVolatilizeImpl.VolatizeFromDID", transID: transID, mode: lock.mode, specifics: entireFile[""], message: "" ]] } ]; }; IF (vDoc _ LookupDIDInCache[did, transID, lock.mode]) # NIL THEN RETURN; IF lock.mode = read THEN { effectiveTransID _ YggEnvironment.nullTransID; IF (vDoc _ LookupDIDInCache[did, effectiveTransID, lock.mode]) # NIL THEN RETURN; } ELSE effectiveTransID _ transID; ReserveDIDInCache[did, effectiveTransID]; doc _ YggDIDMap.OpenDocumentFromDID[did, effectiveTransID]; componentFiles _ YggDIDMap.GetComponentFiles[doc]; contents _ YggFileStream.StreamFromComponentFilesAndTid[componentFiles: componentFiles, fileUse: $contents, tid: effectiveTransID, readOnly: TRUE]; <> <> <<};>> attributes _ YggFileStream.StreamFromComponentFilesAndTid[componentFiles: componentFiles, fileUse: $attributes, tid: effectiveTransID, readOnly: TRUE]; <> <> <<};>> meta _ YggFileStream.StreamFromComponentFilesAndTid[componentFiles: componentFiles, fileUse: $meta, tid: effectiveTransID]; [vDoc, vDocs] _ NewVDoc[]; vDoc.did _ did; vDoc.tid _ effectiveTransID; IF attributes # NIL THEN { attributeList: LIST OF YggRep.Attribute _ NIL; metaAttributes: LIST OF YggRep.Attribute _ NIL; contentsAttribute: YggRep.TypedPrimitiveElement _ [YggRep.unknown, NIL]; outlinksAttribute: LIST OF YggRep.AttributeValue _ NIL; inlinksAttribute: LIST OF YggRep.AttributeValue _ NIL; [attributeList, contentsAttribute, outlinksAttribute, inlinksAttribute, metaAttributes] _ parseAttributes[attributes]; IF contentsAttribute # [YggRep.unknown, NIL] THEN { IF contents # NIL THEN { IF IO.GetLength[contents] # 0 THEN ERROR; }; vDoc.contents _ contentsAttribute; }; vDoc.attributes _ attributeList; vDoc.metaAttributes _ metaAttributes; vDoc.outlinks _ outlinksAttribute; vDoc.inlinks _ inlinksAttribute; }; IF meta # NIL THEN { attributeList: LIST OF YggRep.Attribute _ NIL; metaAttributes: LIST OF YggRep.Attribute _ NIL; outlinksAttribute: LIST OF YggRep.AttributeValue _ NIL; inlinksAttribute: LIST OF YggRep.AttributeValue _ NIL; contentsAttribute: YggRep.TypedPrimitiveElement _ [YggRep.unknown, NIL]; [attributeList, contentsAttribute, outlinksAttribute, inlinksAttribute, metaAttributes] _ parseAttributes[meta]; IF contentsAttribute # [YggRep.unknown, NIL] THEN ERROR; vDoc.metaAttributes _ metaAttributes; IF outlinksAttribute # NIL THEN vDoc.outlinks _ outlinksAttribute; IF inlinksAttribute # NIL THEN vDoc.inlinks _ inlinksAttribute; }; IF contents # NIL THEN { -- contents file exists IF IO.GetLength[contents] # 0 THEN { -- contents file has something in it IF contentsDocType = INT[YggRep.rope] THEN { newRope: ROPE _ NIL; newRope _ IO.GetRope[contents, INT.LAST, FALSE]; vDoc.contents _ [contentsDocType, newRope]; } ELSE { bits: REF YggRep.BitsRep; size: INT; bytesLeft: INT; nextByteToRead: INT _ 0; IF contentsDocType = -1 THEN ERROR; vDoc.contents.docType _ contentsDocType; size _ bytesLeft _ IO.GetLength[contents]; <> vDoc.contents.bits _ bits _ NEW[YggRep.BitsRep[size]]; bits.validBytes _ size; WHILE bytesLeft > 0 DO nBytesRead: INT; unsafeBlock: Basics.UnsafeBlock; TRUSTED { firstByte: LONG POINTER; firstByte _ LOOPHOLE[vDoc.contents.bits, LONG POINTER] + UNITS[YggRep.BitsRep[0]]; unsafeBlock _ [LOOPHOLE[firstByte], nextByteToRead, bytesLeft]; nBytesRead _ IO.UnsafeGetBlock[self: contents, block: unsafeBlock]; }; bytesLeft _ bytesLeft - nBytesRead; nextByteToRead _ nextByteToRead + nBytesRead; ENDLOOP; } } ELSE { -- contents file is empty IF vDoc.contents.bits = NIL THEN { -- didn't find contents in the attributes, so it is a null uninterpreted bytes or rope object IF contentsDocType = INT[YggRep.rope] THEN vDoc.contents _ [YggRep.rope, EmptyRope] ELSE { newRep: REF YggRep.BitsRep; newRep _ NEW[YggRep.BitsRep[10]]; newRep.validBytes _ 0; vDoc.contents _ [contentsDocType, newRep]; }; }; }; }; IF lock.mode # read THEN YggTransaction.NotePossibleDocumentUpdate[transID, vDoc]; <> CacheDID[did, effectiveTransID, vDoc]; SafeStorage.EnableFinalization[vDoc]; }; <<>> <> <<>> <> LatchVDoc: PUBLIC ENTRY PROC [vDoc: YggRep.VDoc, wait: BOOL _ TRUE] RETURNS [latched: BOOL _ TRUE] ~ { <> ENABLE UNWIND => {}; IF vDoc = NIL THEN RETURN[FALSE]; WHILE vDoc.latched DO IF ~wait THEN RETURN[FALSE]; WAIT latchCondition; ENDLOOP; vDoc.latched _ TRUE; }; <<>> <<>> UnlatchVDoc: PUBLIC ENTRY PROC [vDoc: YggRep.VDoc] RETURNS [latched: BOOL _ FALSE] ~ { <> ENABLE UNWIND => {}; IF vDoc.latched THEN {vDoc.latched _ FALSE; RETURN [TRUE]}; BROADCAST latchCondition; }; <<>> scratchVDocUnderMonitor: YggRep.VDoc _ NEW[YggRep.VDocRep]; scratchCacheEntryUnderMonitor: cacheEntry _ NEW[cacheEntryRep]; LookupDIDInCache: PUBLIC ENTRY PROC [did: DID, transID: YggEnvironment.TransID, mode: YggEnvironment.LockMode] RETURNS [vDoc: YggRep.VDoc _ NIL] ~ { <> <> <> ENABLE UNWIND => {}; val: REF; tid: Camelot.tidT _ transID; DO transFound: BOOL _ FALSE; scratchVDocUnderMonitor.did _ did; -- do this every time due to the WAIT below scratchVDocUnderMonitor.tid _ tid; val _ RedBlackTree.Lookup[self: didCache, lookupKey: scratchVDocUnderMonitor]; IF val # NIL THEN { entry: cacheEntry; last: LIST OF YggRep.Attribute; vDoc: YggRep.VDoc _ NIL; vDocs: LIST OF DoubleVDoc _ NIL; entry _ NARROW[val]; entry.timeOfLastVolatize _ Now; IF entry.users # 0 THEN {WAIT myCondition; LOOP;}; entry.vDoc.reissued _ TRUE; IF mode = read THEN RETURN[entry.vDoc]; IF YggTransaction.EqualTrans[tid, transID] THEN { IF mode # read THEN YggTransaction.NotePossibleDocumentUpdate[transID, entry.vDoc]; RETURN[entry.vDoc]; }; [vDoc, vDocs] _ NewVDocInternal[]; vDoc.did _ did; vDoc.tid _ transID; vDoc.contents _ [YggRep.unknown, NIL]; <> FOR aL: LIST OF YggRep.Attribute _ entry.vDoc.attributes, aL.rest UNTIL aL = NIL DO IF vDoc.attributes = NIL THEN last _ vDoc.attributes _ LIST[aL.first] ELSE {last.rest _ LIST[aL.first]; last _ last.rest; }; ENDLOOP; <> FOR aL: LIST OF YggRep.Attribute _ entry.vDoc.metaAttributes, aL.rest UNTIL aL = NIL DO IF vDoc.metaAttributes = NIL THEN last _ vDoc.metaAttributes _ LIST[aL.first] ELSE {last.rest _ LIST[aL.first]; last _ last.rest; }; ENDLOOP; vDoc.contents.docType _ entry.vDoc.contents.docType; SELECT vDoc.contents.docType FROM YggRep.unknown => ERROR; YggRep.int => vDoc.contents.bits _ NEW[INT _ NARROW[entry.vDoc.contents.bits, REF INT]^]; YggRep.shortRope, YggRep.rope, YggRep.did => vDoc.contents.bits _ entry.vDoc.contents.bits; YggRep.float => vDoc.contents.bits _ NEW[REAL32 _ NARROW[entry.vDoc.contents.bits, REF REAL32]^]; YggRep.date => vDoc.contents.bits _ NEW[YggRep.AccurateGMTRep _ NARROW[entry.vDoc.contents.bits, REF YggRep.AccurateGMTRep]^]; YggRep.noValue => vDoc.contents.bits _ NIL; ENDCASE => vDoc.contents.bits _ YggRep.BitsFromBits[entry.vDoc.contents.bits]; scratchVDocUnderMonitor.tid _ transID; RedBlackTree.Insert[self: didCache, insertKey: scratchVDocUnderMonitor, dataToInsert: NEW[cacheEntryRep _ [0, Now, vDoc]]]; IF mode # read THEN YggTransaction.NotePossibleDocumentUpdate[transID, vDoc]; SafeStorage.EnableFinalization[vDoc]; RETURN[vDoc]; }; IF YggTransaction.IsNullTrans[tid] THEN EXIT; [transFound, tid] _ YggTransaction.GetParent[tid]; IF ~transFound THEN ERROR; ENDLOOP; }; <<>> <<>> ReserveDIDInCache: PUBLIC ENTRY PROC [did: DID, transID: YggEnvironment.TransID] ~ { <> ENABLE UNWIND => {}; val: REF; scratchVDocUnderMonitor.did _ did; scratchVDocUnderMonitor.tid _ transID; val _ RedBlackTree.Lookup[self: didCache, lookupKey: scratchVDocUnderMonitor]; IF val # NIL THEN { entry: cacheEntry; entry _ NARROW[val]; WHILE entry.users # 0 DO WAIT myCondition; ENDLOOP; entry.users _ 1; } ELSE { vDoc: YggRep.VDoc; vDocs: LIST OF DoubleVDoc; [vDoc, vDocs] _ NewVDocInternal[]; vDoc.did _ did; vDoc.tid _ transID; RedBlackTree.Insert[self: didCache, insertKey: scratchVDocUnderMonitor, dataToInsert: NEW[cacheEntryRep _ [1, Now, vDoc]]]; <> }; }; <<>> <<>> UnreserveDIDInCache: PUBLIC ENTRY PROC [did: DID, transID: YggEnvironment.TransID] ~ { <> ENABLE UNWIND => {}; val: REF; scratchVDocUnderMonitor.did _ did; scratchVDocUnderMonitor.tid _ transID; val _ RedBlackTree.Lookup[self: didCache, lookupKey: scratchVDocUnderMonitor]; IF val # NIL THEN { entry: cacheEntry; entry _ NARROW[val]; entry.users _ entry.users - 1; entry.timeOfLastVolatize _ Now; BROADCAST myCondition; } ELSE ERROR; }; <<>> <<>> CacheDID: PUBLIC ENTRY PROC [did: DID, transID: YggEnvironment.TransID, vDoc: YggRep.VDoc] ~ { <> ENABLE UNWIND => {}; val: REF; scratchVDocUnderMonitor.did _ did; scratchVDocUnderMonitor.tid _ transID; val _ RedBlackTree.Lookup[self: didCache, lookupKey: scratchVDocUnderMonitor]; IF val # NIL THEN { entry: cacheEntry; entry _ NARROW[val]; VDocsAllocList _ CONS[[entry.vDoc, entry.vDoc], VDocsAllocList]; <> entry.vDoc _ vDoc; entry.users _ 0; entry.timeOfLastVolatize _ Now; BROADCAST myCondition; } ELSE { RedBlackTree.Insert[self: didCache, insertKey: vDoc, dataToInsert: NEW[cacheEntryRep _ [0, Now, vDoc]]]; }; }; <<>> <<>> InvalidateDID: PUBLIC ENTRY PROC [did: DID, transID: YggEnvironment.TransID] ~ { <> ENABLE UNWIND => {}; val: REF; scratchVDocUnderMonitor.did _ did; scratchVDocUnderMonitor.tid _ transID; val _ RedBlackTree.Lookup[self: didCache, lookupKey: scratchVDocUnderMonitor]; IF val # NIL THEN { entry: cacheEntry; entry _ NARROW[val]; WHILE entry.users # 0 DO WAIT myCondition; ENDLOOP; scratchVDocUnderMonitor.did _ did; -- WAIT may have changed did/tid scratchVDocUnderMonitor.tid _ transID; RedBlackTreeDelete[deleteKey: scratchVDocUnderMonitor]; }; }; <<>> PromoteToParent: PUBLIC ENTRY PROC [did: YggDID.DID, transID: YggEnvironment.TransID] ~ { ENABLE UNWIND => {}; val: REF; parentVal: REF; transFound: BOOL _ FALSE; parentTid: YggEnvironment.TransID; scratchVDocUnderMonitor.did _ did; scratchVDocUnderMonitor.tid _ transID; val _ RedBlackTree.Lookup[self: didCache, lookupKey: scratchVDocUnderMonitor]; IF val # NIL THEN { entry: cacheEntry; entry _ NARROW[val]; [transFound, parentTid] _ YggTransaction.GetParent[transID]; IF ~transFound THEN ERROR; scratchVDocUnderMonitor.tid _ parentTid; parentVal _ RedBlackTree.Lookup[self: didCache, lookupKey: scratchVDocUnderMonitor]; IF parentVal # NIL THEN { deletedNode: RedBlackTree.Node _ NIL; parentEntry: cacheEntry; parentEntry _ NARROW[parentVal]; scratchCacheEntryUnderMonitor.vDoc _ parentEntry.vDoc; -- remember old vDoc scratchVDocUnderMonitor.tid _ transID; deletedNode _ RedBlackTree.Delete[self: didCache, deleteKey: scratchVDocUnderMonitor]; -- delete child IF deletedNode = NIL THEN ERROR; IF entry.vDoc.destroyed THEN { scratchVDocUnderMonitor.tid _ parentTid; parentVal _ RedBlackTree.Delete[self: didCache, deleteKey: scratchVDocUnderMonitor]; -- delete parent VDocsAllocList _ CONS[[entry.vDoc, entry.vDoc], VDocsAllocList]; } ELSE { parentEntry.vDoc _ entry.vDoc; parentEntry.vDoc.tid _ parentTid; }; VDocsAllocList _ CONS[[scratchCacheEntryUnderMonitor.vDoc, scratchCacheEntryUnderMonitor.vDoc], VDocsAllocList]; <> <> <> <> <> <> <> <> <> <> <> < {>> <> <> <> << ];>> <> <<};>> <<} ELSE ERROR;>> IF entry.vDoc.metaAttributesChanged # NIL THEN { IF parentEntry.vDoc.metaAttributesChanged = NIL THEN parentEntry.vDoc.metaAttributesChanged _ entry.vDoc.metaAttributesChanged ELSE { last: LIST OF YggRep.metaAttributeMod _ NIL; FOR lomam: LIST OF YggRep.metaAttributeMod _ parentEntry.vDoc.metaAttributesChanged, lomam.rest UNTIL lomam = NIL DO last _ lomam; ENDLOOP; last.rest _ entry.vDoc.metaAttributesChanged; }; }; } ELSE { -- no parent node: RedBlackTree.Node; parentEntry: cacheEntry; scratchVDocUnderMonitor.tid _ transID; node _ RedBlackTree.Delete[self: didCache, deleteKey: scratchVDocUnderMonitor]; IF node = NIL THEN ERROR; parentEntry _ NARROW[node.data]; parentEntry.vDoc.tid _ parentTid; scratchVDocUnderMonitor.tid _ parentTid; RedBlackTree.InsertNode[self: didCache, nodeToInsert: node, insertKey: scratchVDocUnderMonitor]; }; }; }; SetSizeOfDIDCache: PUBLIC ENTRY PROC [size: INT] = { <> ENABLE UNWIND => {}; desiredSizeOfDIDCache _ size; }; <<>> <> TickerProcess: PROC = { ticksToWait: Process.Ticks; ticksToWait _ Process.MsecToTicks[1123]; DO Process.Pause[ticksToWait]; Now _ BasicTime.Now[]; ENDLOOP; }; DIDCacheTrimProcess: PROC = { ticksToWait: Process.Ticks; lastVDoc: YggRep.VDoc _ NIL; threshold: BasicTime.GMT _ BasicTime.Now[]; ticksToWait _ Process.MsecToTicks[293]; DO innerTrim: ENTRY PROC = { ENABLE UNWIND => {}; minGMT: BasicTime.GMT _ threshold; minPeriod: INT _ LAST[INT]; loopCount: INT _ 0; size: INT; size _ RedBlackTree.Size[didCache]; FOR vdl: LIST OF YggRep.VDoc _ ListOfFinalizedVDocs, vdl.rest UNTIL vdl = NIL DO IF loopCount >= 10 THEN {loopCount _ 0; WAIT aTickCondition}; loopCount _ loopCount + 1; IF vdl.first.reissued THEN { vdl.first.reissued _ FALSE; SafeStorage.EnableFinalization[vdl.first]; } ELSE { data: RedBlackTree.UserData; ce: cacheEntry; data _ RedBlackTree.Lookup[didCache, vdl.first]; IF data # NIL THEN { ce _ NARROW[data]; IF ce.users = 0 THEN { period: INT; IF (period _ BasicTime.Period[from: threshold, to: ce.timeOfLastVolatize]) <= 0 THEN { [] _ RedBlackTree.Delete[self: didCache, deleteKey: vdl.first]; size _ size -1; } ELSE { IF period < minPeriod THEN { minGMT _ ce.timeOfLastVolatize; minPeriod _ period; }; }; }; }; }; ListOfFinalizedVDocs _ vdl.rest; IF size <= desiredSizeOfDIDCache THEN EXIT; ENDLOOP; IF size >= desiredSizeOfDIDCache THEN threshold _ BasicTime.Update[minGMT, 2]; }; Process.Pause[ticksToWait]; IF RedBlackTree.Size[didCache] > desiredSizeOfDIDCache THEN innerTrim[]; ENDLOOP; }; <> RedBlackTreeDelete: INTERNAL PROC [deleteKey: REF] ~ { deletedNode: RedBlackTree.Node _ NIL; deletedNode _ RedBlackTree.Delete[self: didCache, deleteKey: deleteKey]; IF deletedNode # NIL THEN { entry: cacheEntry; entry _ NARROW[deletedNode.data]; VDocsAllocList _ CONS[[entry.vDoc, entry.vDoc], VDocsAllocList]; <> <> <> <> <> <> < {>> <> <> <> << ];>> <> <<};>> }; }; GetKeyProc: RedBlackTree.GetKey = { <> entry: cacheEntry _ NARROW[data]; RETURN[ entry.vDoc ]; }; CompareProc: RedBlackTree.Compare = { <> entryData: cacheEntry _ NARROW[data]; keyData: YggRep.VDoc _ NARROW[k]; SELECT keyData.did.didHigh FROM > entryData.vDoc.did.didHigh => RETURN [greater]; < entryData.vDoc.did.didHigh => RETURN [less]; ENDCASE => { SELECT keyData.did.didLow FROM > entryData.vDoc.did.didLow => RETURN [greater]; < entryData.vDoc.did.didLow => RETURN [less]; ENDCASE => { SELECT keyData.tid.top.nodeId.value FROM > entryData.vDoc.tid.top.nodeId.value => RETURN [greater]; < entryData.vDoc.tid.top.nodeId.value => RETURN [less]; ENDCASE => { SELECT keyData.tid.top.highTicker FROM > entryData.vDoc.tid.top.highTicker => RETURN [greater]; < entryData.vDoc.tid.top.highTicker => RETURN [less]; ENDCASE => { SELECT keyData.tid.top.lowTicker FROM > entryData.vDoc.tid.top.lowTicker => RETURN [greater]; < entryData.vDoc.tid.top.lowTicker => RETURN [less]; ENDCASE => RETURN [equal]; }; }; }; }; }; <> <> <<>> NewVDocInternal: INTERNAL PROC RETURNS [vDoc: YggRep.VDoc _ NIL, vDocs: LIST OF DoubleVDoc _ NIL] = { IF VDocsAllocList = NIL THEN { vDoc _ NEW[YggRep.VDocRep]; RETURN; } ELSE { vDocs _ VDocsAllocList; vDoc _ VDocsAllocList.first.aOne; VDocsAllocList _ VDocsAllocList.rest; vDocs.rest _ NIL; vDoc.reissued _ TRUE; -- Better safe than in 815 vDoc.latched _ FALSE; vDoc.fromDID _ YggEnvironment.nullDID; vDoc.toDID _ YggEnvironment.nullDID; vDoc.linkType _ NIL; vDoc.linkChanged _ FALSE; vDoc.destroyed _ FALSE; vDoc.contents _ [YggRep.unknown, NIL]; vDoc.contentsChanged _ FALSE; vDoc.outlinks _ NIL; vDoc.outlinksChanged _ NIL; vDoc.inlinks _ NIL; vDoc.inlinksChanged _ NIL; vDoc.attributes _ NIL; vDoc.namesOfAttributesChanged _ NIL; vDoc.attributesChanged _ NIL; vDoc.metaAttributes _ NIL; vDoc.metaAttributesChanged _ NIL; }; }; NewVDoc: PROC RETURNS [vDoc: YggRep.VDoc _ NIL, vDocs: LIST OF DoubleVDoc _ NIL] = { beCareful: ENTRY PROC = { ENABLE UNWIND => {}; [vDoc, vDocs] _ NewVDocInternal[]; }; IF VDocsAllocList = NIL THEN RETURN [vDoc: NEW[YggRep.VDocRep]]; beCareful[]; }; <> VDocFinalizationProcess: PROC = { DO innerVDocFinalizationProcess: ENTRY PROC [] = { IF vDoc.reissued THEN { vDoc.reissued _ FALSE; SafeStorage.EnableFinalization[vDoc]; } ELSE { IF ListOfFinalizedVDocs = NIL THEN ListOfFinalizedVDocsLast _ ListOfFinalizedVDocs _ LIST[vDoc] ELSE ListOfFinalizedVDocsLast _ (ListOfFinalizedVDocsLast.rest _ LIST[vDoc]); }; }; vDoc: YggRep.VDoc _ NIL; vDoc _ NARROW[SafeStorage.FQNext[MyQueue]]; innerVDocFinalizationProcess[]; vDoc _ NIL; ENDLOOP; }; <> didCache _ RedBlackTree.Create[getKey: GetKeyProc, compare: CompareProc]; TRUSTED { mcp: LONG POINTER TO CONDITION _ @myCondition; Process.Detach[FORK DIDCacheTrimProcess]; Process.Detach[FORK TickerProcess]; Process.SetTimeout[condition: mcp, ticks: Process.MsecToTicks[157]]; mcp _ @latchCondition; Process.SetTimeout[condition: mcp, ticks: Process.MsecToTicks[250]]; mcp _ @aTickCondition; Process.SetTimeout[condition: mcp, ticks: 1]; SafeStorage.EstablishFinalization[type: CODE[YggRep.VDocRep], npr: 1, fq: MyQueue ! SafeStorage.CantEstablishFinalization => CONTINUE;]; Process.Detach[FORK VDocFinalizationProcess[] ]; }; <<>> <<>> END.