YggDIDMapImpl.mesa
Copyright Ó 1988, 1989 by Xerox Corporation. All rights reserved.
Last edited by
Bob Hagmann July 6, 1989 12:32:24 pm PDT
DIRECTORY
Atom USING [GetPName, MakeAtom],
Basics USING [BITAND, charsPerWord, Comparison],
BasicTime USING [Now, nullGMT],
ExtendibleHash USING [DeleteEntry, Entry, ExtendibleHashTable, New, Open, PageNumber, ReadEntry, ReferencePage, ReleasePage, UpdateEntry],
Camelot USING [btidT, DSLogNewValue, DSPinObject, -- ErSuccess, -- optrT, -- TABegin, TAEnd, -- tidT],
Mach USING [kernReturnT, -- KernSuccess, -- pointerT],
Process USING [Detach, InitializeCondition, MsecToTicks, Pause, Ticks],
PBasics USING [BITAND, ByteBlt, bytesPerWord, charsPerWord, Move],
RedBlackTree USING [Compare, Create, Delete, GetKey, Insert, Lookup, Table, UserData],
RefText USING [New, TrustTextAsRope],
Rope USING [Equal, Fetch, Length, NewText, ROPE, Substr, Text],
SafeStorage USING [CantEstablishFinalization, EnableFinalization, EstablishFinalization, FinalizationQueue, FQNext, IsFinalizationEnabled, NewFQ],
YggBuffMan USING [ReadPages, ReleaseState, ReleaseVMPageSet, VMPageSet, WritePages],
YggDID USING [CompareDIDs, DID, EqualDIDs, HashDID, ValidateDID],
YggDIDMap,
YggDIDMapPrivate USING [DocItem, DocPart, DocPartPtr, Document, DocumentRep, Entry, EntryPtr, notCompressed, PartType, RunItem, SegmentItem, SegmentRep, TextRep, TextRP, Word32OverheadPerEntry],
YggDIDPrivate USING [DIDForRootDIDs, DIDRep],
YggEnvironment USING [dsPort, nullDID, nullTransID, Outcome, -- taPort, -- TransID],
YggFile USING [BytesForPages, FileHandle, GetMaxDIDAddress, Info, Open, OpenDIDMapFile, PageCount, SetSize],
YggFixedNames USING [IndexPrefix, IndexPrefixSize],
YggInternal USING [Document, FileHandle],
YggNav USING [GetProperty, SetContents, SetProperty],
YggRep USING [Attribute, did, TypedPrimitiveElement],
YggTransaction USING [CreateTrans, Finish];
YggDIDMapImpl: CEDAR MONITOR
IMPORTS Atom, Basics, BasicTime, Camelot, ExtendibleHash, PBasics, Process, RedBlackTree, RefText, Rope, SafeStorage, YggBuffMan, YggDID, YggDIDPrivate, YggEnvironment, YggFile, YggFixedNames, YggNav, YggTransaction
EXPORTS YggInternal, YggDIDMap, YggDID =
BEGIN
Data types and variables
Document: TYPE = REF DocumentRep;
DocumentRep: PUBLIC TYPE = YggDIDMapPrivate.DocumentRep;
DID: PUBLIC TYPE ~ REF DIDRep;
DIDRep: PUBLIC TYPE ~ YggDIDPrivate.DIDRep;
HashPageSizeInWords: INT = 1024;
MaxWordsPerEntry: INT = HashPageSizeInWords/3 - 5;
FilePagesPerBtreePage: INT = 1;
Hash64Table: ARRAY [0..64) OF CARD;
Lock64Table: ARRAY [0..64) OF Lock64Item;
Lock64Item: TYPE = RECORD[
interlock: BOOLFALSE,
tid: Camelot.tidT
];
TheDIDMap: ExtendibleHash.ExtendibleHashTable;
FileForDIDMap: YggFile.FileHandle;
CurrentVMPageSetsForDIDMap: LIST OF YggBuffMan.VMPageSet ← NIL;
NextDID: DIDRep;
LastDIDAllocateable: DIDRep;
MaxDIDOptr: Camelot.optrT;
AddressForMaxDID: Mach.pointerT;
RecoverableMaxDID: LONG POINTER TO DIDRep;
LastDidLow: CARD ← 1000000000;
NumberOfDIDsToGrap: CARD ← 200;
ThresholdToGrabDIDs: CARD ← 20;
KeyObject: TYPE = RECORD [did: YggDID.DID, extensionCount: INT32];
EntryDataBuffer: TYPE = RECORD[SEQUENCE COMPUTED CARD16 OF EntryWord];
EntryWord: TYPE = MACHINE DEPENDENT RECORD[
SELECT OVERLAID * FROM
card => [card: CARD32],
int => [int: INT32],
pair => [hi, lo: CARD16],
bytes => [hh, hl, lh, ll: BYTE],
bits => [bits: PACKED ARRAY [0..32) OF BOOL],
ENDCASE
];
updateAction: TYPE = {create, delete, newValue};
updates: TYPE = RECORD[
did: YggDID.DID,
tid: Camelot.tidT,
action: updateAction,
doc: Document
];
TIDObj: TYPE = REF TIDObjRep;
TIDObjRep: TYPE = RECORD [ btid: Camelot.btidT];
scratchTIDObj: TIDObj ← NEW[TIDObjRep];
DocTrans: TYPE = REF DocTransRep;
DocTransRep: TYPE = RECORD [
btidForDoc: TIDObj,
docs: LIST OF Document ← NIL,
bucketsToUnlatch: LIST OF CARD ← NIL
];
transactionToModifiedDocumentMap: RedBlackTree.Table;
finalizationQ: SafeStorage.FinalizationQueue;
ZeroIllegal: --CALLING-- ERROR = CODE;
secondLookupHashDocument: HashDocument ← NIL; -- wierdness to protect object from finalization.
WORDSInWords: CARD ← 0;
Exported procedures for documents
MarkInterestInDID: PUBLIC PROC [did: YggDID.DID] ~ {
Notes continued interest in this did.
[] ← LookupInCache[did, FALSE];
};
OpenDocumentFromDID: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT, destroyedOK: BOOL] RETURNS [doc: YggInternal.Document ← NIL] = {
Finds or creates a Document for the supplied did and tid.
tryCount: CARD ← 0;
DO
tryAgain: BOOLFALSE;
IF (doc ← LookupInCache[did, destroyedOK]) = NIL THEN {
found: BOOLFALSE;
gottaBeInCache: BOOLFALSE;
doc ← NEW[DocumentRep ← [did: did, componentFiles: NIL, interlock: TRUE, next: NIL]];
[found, gottaBeInCache] ← LookupInMap[did, tid, doc];
IF gottaBeInCache THEN {
doc ← LookupInCache[did, destroyedOK];
IF doc = NIL THEN ERROR;
}
ELSE {
IF ~found THEN RETURN [NIL];
Insert[doc ! HashPkgDuplicateKey => {
tryAgain ← TRUE;
CONTINUE;
};
];
Document is now inserted into the cache. Interlock is set. Open up the componentFiles.
IF ~tryAgain THEN openFilesFromParsedDocList[doc, did, tid];
};
}
ELSE { -- found in cache
waitForInterlockDone: ENTRY PROC ~ {
ENABLE UNWIND => {};
WHILE doc.interlock DO WAIT aShortTime ENDLOOP;
};
IF doc.interlock THEN waitForInterlockDone[];
};
IF ~tryAgain THEN {
releaseDoc[doc];
EXIT;
};
tryCount ← tryCount + 1;
IF tryCount > 25 THEN ERROR;
ENDLOOP;
};
GetDID: PUBLIC PROC [doc: Document] RETURNS [did: YggDID.DID] ~ {
errors defined in this interface: none.
RETURN[doc.did];
};
GetLinkInfo: PUBLIC PROC [doc: Document] RETURNS [fromDID, toDID: YggDID.DID, linkType: Rope.ROPE] ~ {
Get the link info for the Document.
RETURN[doc.fromDID, doc.toDID, doc.linkType];
};
SwapLinkInfo: PUBLIC PROC [doc: Document, newFromDID, newToDID: YggDID.DID, newLinkType: Rope.ROPE] RETURNS [oldFromDID, oldToDID: YggDID.DID, oldLinkType: Rope.ROPE] ~ {
Swap the link info for the document, and report what it used to be.
oldFromDID ← doc.fromDID;
doc.fromDID ← newFromDID;
oldToDID ← doc.toDID;
doc.toDID ← newToDID;
oldLinkType ← doc.linkType;
doc.linkType ← newLinkType;
};
GetComponentFiles: PUBLIC PROC [doc: Document] RETURNS [componentFiles: LIST OF YggInternal.FileHandle] ~ {
no errors possible, but may return NIL.
RETURN[doc.componentFiles];
};
AddComponentFile: PUBLIC ENTRY PROC [doc: Document, componentFile: YggInternal.FileHandle, tid: Camelot.tidT] ~ {
ENABLE UNWIND => {};
lastCF: LIST OF YggInternal.FileHandle ← NIL;
FOR files: LIST OF YggInternal.FileHandle ← doc.componentFiles, files.rest UNTIL files = NIL DO
lastCF ← files;
ENDLOOP;
IF lastCF = NIL THEN doc.componentFiles ← CONS[componentFile, NIL]
ELSE lastCF.rest ← CONS[componentFile, NIL];
};
RemoveComponentFile: PUBLIC PROC [doc: Document, componentFile: YggInternal.FileHandle, tid: Camelot.tidT] ~ {
lastCF: LIST OF YggInternal.FileHandle ← NIL;
FOR files: LIST OF YggInternal.FileHandle ← doc.componentFiles, files.rest UNTIL files = NIL DO
IF files.first = componentFile THEN {
IF lastCF = NIL THEN doc.componentFiles ← files.rest
ELSE lastCF.rest ← files.rest;
EXIT;
};
lastCF ← files;
ENDLOOP;
};
GetName: PUBLIC PROC [doc: Document] RETURNS [Rope.ROPE] ~ {
no errors possible, but may return NIL.
RETURN[doc.name];
};
CreateNewDID: PUBLIC PROC [tid: Camelot.tidT] RETURNS [did: YggDID.DID ] ~ {
doc: YggInternal.Document ← NIL;
did ← GetNextDID[];
doc ← NEW[DocumentRep ← [did: did, modifiedDIDMap: TRUE, componentFiles: NIL, interlock: FALSE, next: NIL]];
Insert[doc ! HashPkgDuplicateKey => ERROR];
noteUseInTransaction[doc, tid];
};
CreateExplicitDID: PUBLIC PROC [tid: Camelot.tidT, did: YggDID.DID] RETURNS [ok: BOOLTRUE ] ~ {
doc: YggInternal.Document ← NIL;
IF YggDID.ValidateDID[did] THEN RETURN [FALSE];
doc ← NEW[DocumentRep ← [did: did, modifiedDIDMap: TRUE, componentFiles: NIL, interlock: FALSE, next: NIL]];
Insert[doc ! HashPkgDuplicateKey => {ok ← FALSE; CONTINUE}];
IF ok THEN noteUseInTransaction[doc, tid];
};
DestroyDID: PUBLIC PROC [tid: Camelot.tidT, did: YggDID.DID] ~ {
doc: YggInternal.Document ← NIL;
doc ← OpenDocumentFromDID[did: did, tid: tid, destroyedOK: FALSE];
doc.destroyed ← TRUE;
};
GetRootDIDs: PUBLIC PROC RETURNS [dids: LIST OF YggDID.DID ← NIL] ~ {
trans: YggEnvironment.TransID;
propertyExists: BOOLFALSE;
property: LIST OF YggRep.Attribute ← NIL;
trans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];
[propertyExists: propertyExists, property: property] ← YggNav.GetProperty[trans: YggEnvironment.nullTransID, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs"];
IF propertyExists THEN {
lastDIDs: LIST OF YggDID.DIDNIL;
IF property.rest # NIL THEN ERROR;
IF property.first.value.rest # NIL THEN ERROR;
FOR lotpe: LIST OF YggRep.TypedPrimitiveElement ← property.first.value.first.valueSet, lotpe.rest UNTIL lotpe = NIL DO
didInList: DID;
IF lotpe.first.docType # YggRep.did THEN ERROR;
didInList ← NARROW[lotpe.first.bits];
IF dids = NIL THEN {dids ← LIST[didInList]; lastDIDs ← dids}
ELSE {lastDIDs.rest ← LIST[didInList]; lastDIDs ← lastDIDs.rest;};
ENDLOOP;
};
[] ← YggTransaction.Finish[transID: trans, requestedOutcome: commit];
};
RootDIDLatch: BOOLFALSE;
LatchRootDID: ENTRY PROC ~ {
ENABLE UNWIND => {};
WHILE RootDIDLatch DO WAIT aShortTime ENDLOOP;
RootDIDLatch ← TRUE;
};
UnlatchRootDID: ENTRY PROC ~ {
RootDIDLatch ← FALSE;
};
AddRootDID: PUBLIC PROC [did: YggDID.DID] RETURNS [ok: BOOLFALSE] ~ {
trans: YggEnvironment.TransID;
propertyExists: BOOLFALSE;
property: LIST OF YggRep.Attribute ← NIL;
alreadyRoot: BOOLFALSE;
newProperty: YggRep.Attribute;
LatchRootDID[];
trans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];
[propertyExists: propertyExists, property: property] ← YggNav.GetProperty[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs"];
newProperty ← [attributeName: "$rootDIDs", ordered: TRUE, value: LIST[[fieldName: "", valueSet: LIST [[docType: YggRep.did, bits: did]]]]];
IF propertyExists THEN {
lastDIDs: LIST OF YggRep.TypedPrimitiveElement ← NIL;
IF property.rest # NIL THEN ERROR;
IF property.first.value.rest # NIL THEN ERROR;
FOR lotpe: LIST OF YggRep.TypedPrimitiveElement ← property.first.value.first.valueSet, lotpe.rest UNTIL lotpe = NIL DO
didInList: DID;
IF lotpe.first.docType # YggRep.did THEN ERROR;
didInList ← NARROW[lotpe.first.bits];
IF YggDID.EqualDIDs[did, didInList] THEN {alreadyRoot ← TRUE; EXIT};
lastDIDs ← lotpe;
ENDLOOP;
IF ~alreadyRoot THEN lastDIDs.rest ← LIST[[docType: YggRep.did, bits: did]];
}
ELSE property ← LIST[newProperty];
IF ~alreadyRoot THEN YggNav.SetProperty[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs", property: property.first, appendProperty: FALSE];
[] ← YggTransaction.Finish[transID: trans, requestedOutcome: commit];
UnlatchRootDID[];
};
RemoveRootDID: PUBLIC PROC [did: YggDID.DID] RETURNS [ok: BOOLFALSE] ~ {
trans: YggEnvironment.TransID;
propertyExists: BOOLFALSE;
property: LIST OF YggRep.Attribute ← NIL;
isARoot: BOOLFALSE;
newProperty: YggRep.Attribute;
LatchRootDID[];
trans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];
[propertyExists: propertyExists, property: property] ← YggNav.GetProperty[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs"];
newProperty ← [attributeName: "$rootDIDs", ordered: TRUE, value: LIST[[fieldName: "", valueSet: LIST [[docType: YggRep.did, bits: did]]]]];
IF propertyExists THEN {
prevDIDs: LIST OF YggRep.TypedPrimitiveElement ← NIL;
IF property.rest # NIL THEN ERROR;
IF property.first.value.rest # NIL THEN ERROR;
FOR lotpe: LIST OF YggRep.TypedPrimitiveElement ← property.first.value.first.valueSet, lotpe.rest UNTIL lotpe = NIL DO
didInList: DID;
IF lotpe.first.docType # YggRep.did THEN ERROR;
didInList ← NARROW[lotpe.first.bits];
IF YggDID.EqualDIDs[did, didInList] THEN {
isARoot← TRUE;
IF prevDIDs = NIL THEN property.first.value.first.valueSet ← lotpe.rest
ELSE prevDIDs.rest ← lotpe.rest;
EXIT;
};
prevDIDs ← lotpe;
ENDLOOP;
};
IF isARoot THEN YggNav.SetProperty[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs", property: property.first, appendProperty: FALSE];
[] ← YggTransaction.Finish[transID: trans, requestedOutcome: commit];
UnlatchRootDID[];
RETURN[isARoot];
};
Exported for document modifications
aShortTime: CONDITION;
grapDoc: ENTRY PROC [doc: YggInternal.Document] ~ {
ENABLE UNWIND => {};
WHILE doc.interlock DO WAIT aShortTime ENDLOOP;
doc.interlock ← TRUE;
};
releaseDoc: ENTRY PROC [doc: YggInternal.Document] ~ {doc.interlock ← FALSE};
RemoveRuns: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT, runList: YggDIDMap.RunList] ~ {
Normally, only called by the file package during it's pre-commit. tid is a top level transaction. Runs listed are removed from the object. Also used when an object is migrated. Caller guarentees that PreCommit will later be called with this tid (if the system doesn't crash).
The document is updated in the cache. PreCommit will make the updates to the disk resident B-tree(s). If Commit is later called, then all is well. If Abort is called, then the cache entry is destroyed. Undo is done in the B-trees by physical pages. Subsequent use of the document requires rebuilding the document object.
doc: YggInternal.Document ← NIL;
IF (doc ← LookupInCache[did, TRUE]) = NIL THEN ERROR
ELSE {
doc.modifiedDIDMap ← TRUE;
grapDoc[doc];
noteUseInTransaction[doc, tid];
FOR rl: YggDIDMap.RunList ← runList, rl.rest UNTIL rl = NIL DO
pdl: LIST OF YggDIDMapPrivate.DocItem;
prevPdl: LIST OF YggDIDMapPrivate.DocItem ← NIL;
runToDelete: YggDIDMap.Run ← rl.first;
FOR pdl ← doc.parsedDocList, pdl.rest UNTIL pdl = NIL DO
segments: LIST OF YggDIDMapPrivate.SegmentItem;
prevSegments: LIST OF YggDIDMapPrivate.SegmentItem;
FOR segments ← pdl.first.segments, segments.rest UNTIL segments = NIL DO
runs: LIST OF YggDIDMapPrivate.RunItem;
prevRuns: LIST OF YggDIDMapPrivate.RunItem;
FOR runs ← segments.first.runs, runs.rest UNTIL runs = NIL DO
firstIntersection: INT ← -1;
lastIntersection: INT ← -1;
runLast: CARD ← 0;
runToDeleteLast: CARD ← 0;
IF runs.first.segmentId # runToDelete.segmentId THEN {-- wrong segment
prevRuns ← runs;
LOOP;
};
firstIntersection ← MAX[runs.first.segmentPage, runToDelete.segmentPage];
lastIntersection ← MIN[
(runLast ← runs.first.segmentPage + runs.first.pages - 1),
(runToDeleteLast ← runToDelete.segmentPage + runToDelete.pages - 1) ];
IF lastIntersection < firstIntersection THEN LOOP; -- no intersection
IF firstIntersection = INT[runs.first.segmentPage] AND lastIntersection = INT[runLast] THEN { -- run is eliminated
IF prevRuns = NIL THEN segments.first.runs ← runs.rest
ELSE prevRuns.rest ← runs.rest;
LOOP;
}
ELSE {
runs.first.segmentPage ← firstIntersection;
runs.first.pages ← lastIntersection - firstIntersection + 1;
};
prevRuns ← runs;
ENDLOOP;
IF segments.first.runs = NIL THEN { -- no more runs, so dump the segment too
IF prevSegments = NIL THEN pdl.first.segments ← segments.rest
ELSE prevSegments.rest ← segments.rest;
LOOP;
};
prevSegments ← segments;
ENDLOOP;
IF pdl.first.segments = NIL THEN { -- no more segments, so dump the DocItem too
IF prevPdl = NIL THEN doc.parsedDocList ← pdl.rest
ELSE prevPdl.rest ← pdl.rest;
LOOP;
};
prevPdl ← pdl;
ENDLOOP;
ENDLOOP;
releaseDoc[doc];
};
};
AddRuns: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT, fileUse: ATOM, runList: YggDIDMap.RunList] ~ {
Normally, only called by the file package during it's pre-commit. tid is a top level transaction. Runs listed are added to the object. Also used when an object is removed from the disk cache.
doc: YggInternal.Document ← NIL;
partType: YggDIDMapPrivate.PartType;
partType ← mapFileUseToPartType[fileUse];
IF (doc ← LookupInCache[did]) = NIL THEN ERROR
ELSE {
lastPDL: LIST OF YggDIDMapPrivate.DocItem ← NIL;
thisPDL: LIST OF YggDIDMapPrivate.DocItem ← NIL;
doc.modifiedDIDMap ← TRUE;
grapDoc[doc];
noteUseInTransaction[doc, tid];
FOR pdl: LIST OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, pdl.rest UNTIL pdl = NIL DO
IF pdl.first.indexName = fileUse THEN {
thisPDL ← pdl;
EXIT;
};
lastPDL ← pdl;
REPEAT FINISHED => {
thisPDL ← LIST[[partType, fileUse, 0, NIL]];
IF lastPDL = NIL THEN doc.parsedDocList ← thisPDL
ELSE lastPDL.rest ← thisPDL;
};
ENDLOOP; -- pdl
IF thisPDL = NIL THEN ERROR;
FOR runs: YggDIDMap.RunList ← runList, runs.rest UNTIL runs = NIL DO
lastSegment: LIST OF YggDIDMapPrivate.SegmentItem ← NIL;
FOR segs: LIST OF YggDIDMapPrivate.SegmentItem ← thisPDL.first.segments, segs.rest UNTIL segs = NIL DO
lastRun: LIST OF YggDIDMapPrivate.RunItem ← NIL;
runPages: YggFile.PageCount ← 0;
FOR lori: LIST OF YggDIDMapPrivate.RunItem ← segs.first.runs, lori.rest UNTIL lori = NIL DO
runPages ← runPages + lori.first.pages;
lastRun ← lori;
ENDLOOP;
IF INT[segs.first.firstPage] + runPages = runs.first.firstPage THEN { -- append at end of segment
lastRun.rest ← CONS[[segmentId: runs.first.segmentId, segmentPage: runs.first.segmentPage, pages: runs.first.pages], NIL];
EXIT; -- appended at end of segment; go do the next run
};
lastSegment ← segs;
REPEAT FINISHED => { -- make a new segment with a singleton run
newItem: LIST OF YggDIDMapPrivate.SegmentItem;
newItem ← LIST[[firstPage: runs.first.firstPage, numberOfBytes: YggFile.BytesForPages[runs.first.pages], runs: LIST[[segmentId: runs.first.segmentId, segmentPage: runs.first.segmentPage, pages: runs.first.pages]]]];
IF lastSegment = NIL THEN thisPDL.first.segments ← newItem
ELSE lastSegment.rest ← newItem;
};
ENDLOOP;
ENDLOOP;
releaseDoc[doc];
};
};
SetByteSize: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT, fileUse: ATOM, bytes: INT] ~ {
Normally, only called by the file package during it's pre-commit. tid is a top level transaction. The size of valid bytes is set to this amount, but the pages for the file are not changed.
doc: YggInternal.Document ← NIL;
partType: YggDIDMapPrivate.PartType;
partType ← mapFileUseToPartType[fileUse];
IF (doc ← LookupInCache[did]) = NIL THEN ERROR
ELSE {
doc.modifiedDIDMap ← TRUE;
grapDoc[doc];
noteUseInTransaction[doc, tid];
FOR pdl: LIST OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, pdl.rest UNTIL pdl = NIL DO
IF pdl.first.indexName = fileUse THEN {
FOR segs: LIST OF YggDIDMapPrivate.SegmentItem ← pdl.first.segments, segs.rest UNTIL segs = NIL DO
firstByte: CARD;
firstByte ← YggFile.BytesForPages[segs.first.firstPage];
IF INT[firstByte] > bytes THEN segs.first.numberOfBytes ← 0
ELSE {
runPages: YggFile.PageCount ← 0;
runBytes: CARD ← 0;
FOR lori: LIST OF YggDIDMapPrivate.RunItem ← segs.first.runs, lori.rest UNTIL lori = NIL DO
runPages ← runPages + lori.first.pages;
ENDLOOP;
runBytes ← YggFile.BytesForPages[runPages];
segs.first.numberOfBytes ← IF INT[firstByte + runBytes] > bytes THEN bytes - firstByte ELSE runBytes;
};
ENDLOOP;
EXIT;
};
REPEAT FINISHED => {
IF bytes > 0 THEN ERROR;
};
ENDLOOP; -- pdl
releaseDoc[doc];
};
};
Utilities
Hash64DID: PROC [did: DID] RETURNS [bucket: CARD] ~ {
RETURN [Hash64Table[Basics.BITAND[03Fh, Hash64[did.didLow]]]];
};
Hash64: PROC [c: CARD] RETURNS [bucket: CARD] ~ {
RETURN [Hash64Table[PBasics.BITAND[03Fh, c]]];
};
mapFileUseToPartType: PROC [fileUse: ATOM] RETURNS [partType: YggDIDMapPrivate.PartType] ~ {
SELECT fileUse FROM
$root => partType ← root;
$contents => partType ← contents;
$outlinks => partType ← outlinks;
$attributes => partType ← attributes;
$directoryContents => partType ← directoryContents;
ENDCASE => {
pName: Rope.Text;
pName ← Atom.GetPName[fileUse];
IF Rope.Equal[YggFixedNames.IndexPrefix, Rope.Substr[pName, 0, YggFixedNames.IndexPrefixSize]] THEN {
partType ← index;
}
ELSE ERROR;
};
};
Exported procedures for transactions
PreCommit: PUBLIC PROC [tid: Camelot.tidT] ~ {
Take all the modified documents from the transaction and write out their changes to the DID Map.
Find all the hash pages that have to be updated. Process them in ascending order, doing all of the updates for the whole page before doing the next page. We can't deadlock within DIDMap update since we've ordered the locks.
First, lock the page for exclusive access. Perform the update(s) including the Camelot pin and log dance. Downgrade to lock to allow readers (they can see uncommitted data).
If the transaction commits, completely unlock the tree. If the transaction aborts, lock the tree for exclusive access, perform undo, and after undo processing is done unlock the tree.
data: RedBlackTree.UserData;
scrTIDObj: TIDObj ← CheckoutTIDObj[tid.top];
refTid: REF Camelot.tidT ← NEW [Camelot.tidT ← tid];
data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scrTIDObj];
RecycleTIDObj[scrTIDObj];
IF data # NIL THEN {
dt: DocTrans;
max64Done: INT ← -1;
dt ← NARROW[data];
DO
minBucket: CARD ← 65;
numberInMinBucket: CARD ← 0;
FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO -- look at all the documents to find the next 64 value to do
hashedDidTo64: CARD;
hashedDidTo64 ← Hash64DID[docList.first.did];
IF hashedDidTo64 = minBucket THEN numberInMinBucket ← numberInMinBucket + 1;
IF INT[hashedDidTo64] > max64Done AND hashedDidTo64 < minBucket THEN {
minBucket ← hashedDidTo64;
numberInMinBucket ← 1;
};
ENDLOOP;
IF minBucket = 65 THEN EXIT;
IF INT[minBucket] = max64Done THEN EXIT;
LockBucket64[minBucket, tid];
dt.bucketsToUnlatch ← CONS[minBucket, dt.bucketsToUnlatch];
FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO -- look at all the documents and process those with the correct hash value
hashedDidTo64: CARD;
hashedDidTo64 ← Hash64DID[docList.first.did];
IF hashedDidTo64 = minBucket THEN {
writeDocumentToMap[docList.first, refTid];
};
ENDLOOP;
max64Done ← minBucket;
ENDLOOP;
};
};
DropLatches: PROC [buckets: LIST OF CARD, tid: Camelot.tidT] ~ {
FOR nowBuck: LIST OF CARD ← buckets, nowBuck.rest UNTIL nowBuck = NIL DO
UnlockBucket64[nowBuck.first, tid];
ENDLOOP;
};
Commit: PUBLIC PROC [tid: Camelot.tidT] ~ {
data: RedBlackTree.UserData;
scrTIDObj: TIDObj ← CheckoutTIDObj[tid.top];
data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scrTIDObj];
IF data # NIL THEN {
dt: DocTrans;
dt ← NARROW[data];
FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO -- look at all the documents and turn off the modified flag
docList.first.modifiedDIDMap ← FALSE;
IF docList.first.destroyed THEN {
Delete[docList.first];
};
ENDLOOP;
DropLatches[dt.bucketsToUnlatch, tid];
};
[] ← RedBlackTree.Delete[ transactionToModifiedDocumentMap, scrTIDObj];
RecycleTIDObj[scrTIDObj];
};
Abort: PUBLIC PROC [tid: Camelot.tidT] ~ {
data: RedBlackTree.UserData;
scrTIDObj: TIDObj ← CheckoutTIDObj[tid.top];
data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scrTIDObj];
IF data # NIL THEN {
dt: DocTrans;
max64Done: INT ← -1;
dt ← NARROW[data];
FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO -- look at all the documents and remove them from the cache
Delete[ docList.first];
ENDLOOP;
DropLatches[dt.bucketsToUnlatch, tid];
};
[] ← RedBlackTree.Delete[ transactionToModifiedDocumentMap, scrTIDObj];
RecycleTIDObj[scrTIDObj];
};
Files
openFilesFromParsedDocList: PROC [doc: YggInternal.Document, did: YggDID.DID, tid: Camelot.tidT] = {
FOR pdl: LIST OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, pdl.rest UNTIL pdl = NIL DO
atomCode: ATOMNIL;
byteSize: INT ← 0;
theRuns: YggDIDMap.RunList ← NIL;
theFile: YggFile.FileHandle;
SELECT pdl.first.partType FROM
root => atomCode ← $root;
contents => atomCode ← $contents;
outlinks => atomCode ← $outlinks;
index => {
atomCode ← pdl.first.indexName;
};
directoryContents => atomCode ← $directoryContents;
attributes => atomCode ← $attributes;
ENDCASE => atomCode ← $huh;
FOR segments: LIST OF YggDIDMapPrivate.SegmentItem ← pdl.first.segments, segments.rest UNTIL segments = NIL DO
nowPage: CARD ← segments.first.firstPage;
byteSize ← MAX[byteSize, YggFile.BytesForPages[segments.first.firstPage] + segments.first.numberOfBytes];
FOR runs: LIST OF YggDIDMapPrivate.RunItem ← segments.first.runs, runs.rest UNTIL runs = NIL DO
theRuns ← CONS[[segmentId: runs.first.segmentId, segmentPage: runs.first.segmentPage, firstPage: nowPage, pages: runs.first.pages, leader: FALSE], theRuns];
nowPage ← nowPage + runs.first.pages;
ENDLOOP;
ENDLOOP;
theFile ← YggFile.Open[runList: theRuns, byteSize: byteSize, did: did, fileUse: atomCode, verifyLeaderNow: FALSE];
doc.componentFiles ← CONS[theFile, doc.componentFiles];
ENDLOOP;
};
Utilities
AppendText: UNSAFE PROC [dPP: YggDIDMapPrivate.DocPartPtr, text: Rope.Text, offset: CARD] RETURNS [textRP: YggDIDMapPrivate.TextRP, words: CARD] = UNCHECKED {
textRep: LONG POINTER TO YggDIDMapPrivate.TextRep;
byteOffset: CARD16 ← offset*UNITS[CARD32];
textRP ← LOOPHOLE[byteOffset];
textRep ← @dPP[textRP];
LOOPHOLE[textRep, LONG POINTER TO CARD16]^ ← Rope.Length[text]; -- sets textRep.length
[] ← PBasics.ByteBlt [
to: [ BASE[DESCRIPTOR[textRep]], 0, textRep.length ],
from: [ BASE[DESCRIPTOR[text]], 0, textRep.length ]
];
words ← (WORDS[YggDIDMapPrivate.TextRep[textRep.length]] + WORDSInWords - 1) / WORDSInWords;
};
GetText: UNSAFE PROC [textRep: LONG POINTER TO YggDIDMapPrivate.TextRep, text: REF TEXT] = UNCHECKED {
nBytes: CARD;
text.length ← textRep.length;
nBytes ← PBasics.ByteBlt [
to: [ BASE[DESCRIPTOR[text]], 0, textRep.length ],
from: [ BASE[DESCRIPTOR[textRep]], 0, textRep.length ]
];
};
DID allocation
didsCondition: CONDITION;
GetNextDID: ENTRY PROC RETURNS [did: YggDID.DID] ~ {
ENABLE UNWIND => {};
loopCount: INT ← 0;
WHILE NextDID.didHigh = LastDIDAllocateable.didHigh AND NextDID.didLow = LastDIDAllocateable.didLow DO
WAIT didsCondition;
loopCount ← loopCount + 1;
IF loopCount > 5 THEN TRUSTED {loopCount ← 0; Process.Detach[FORK GrabSomeDIDs[]];};
ENDLOOP;
did ← NEW[YggDIDPrivate.DIDRep ← [NextDID.didHigh, NextDID.didLow]];
NextDID.didLow ← NextDID.didLow + 1;
IF LastDIDAllocateable.didLow - NextDID.didLow <= ThresholdToGrabDIDs THEN TRUSTED {Process.Detach[FORK GrabSomeDIDs[]];};
};
GrabberCount: CARD ← 0;
GrabSomeDIDs: PROC [] ~ {
newLastDIDAllocateable: REF DIDRep ← NEW[DIDRep];
newHigh: BOOLFALSE;
moreDIDsTID: Camelot.tidT;
kernCode: Mach.kernReturnT;
onlyOne: ENTRY PROC RETURNS [exit: BOOL ← FALSE] ~ {
ENABLE UNWIND => {};
IF GrabberCount # 0 THEN RETURN[TRUE];
GrabberCount ← GrabberCount + 1;
};
IF onlyOne[] THEN RETURN;
newLastDIDAllocateable.didHigh ← LastDIDAllocateable.didHigh;
newLastDIDAllocateable.didLow ← LastDIDAllocateable.didLow + NumberOfDIDsToGrap;
IF newLastDIDAllocateable.didLow > LastDidLow THEN {
newLastDIDAllocateable.didHigh ← newLastDIDAllocateable.didHigh + 1;
newLastDIDAllocateable.didLow ← NumberOfDIDsToGrap;
newHigh ← TRUE;
};
DO
outcome: YggEnvironment.Outcome;
status: INT ← -1;
addrNewLastDIDAllocateable: Mach.pointerT;
[newTid: moreDIDsTID, kernCode: kernCode] ← Camelot.TABegin[taPort: YggEnvironment.taPort, parentTid: YggEnvironment.nullTransID, transType: ttNvServerBased, raiseSignal: TRUE];
moreDIDsTID ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];
kernCode ← Camelot.DSPinObject[dsPort: YggEnvironment.dsPort, tid: moreDIDsTID, optr: MaxDIDOptr, size: BYTES[DIDRep], raiseSignal: TRUE];
TRUSTED {addrNewLastDIDAllocateable ← LOOPHOLE[newLastDIDAllocateable, CARD32]};
kernCode ← Camelot.DSLogNewValue[dsPort: YggEnvironment.dsPort, tid: moreDIDsTID, optr: MaxDIDOptr, newValue: addrNewLastDIDAllocateable, newValueCnt: BYTES[DIDRep], raiseSignal: TRUE];
[status: status, kernCode: kernCode] ← Camelot.TAEnd[taPort: YggEnvironment.taPort, tid: moreDIDsTID, protocolType: ptTwoPhased, raiseSignal: TRUE];
IF kernCode = Mach.KernSuccess AND status = Camelot.ErSuccess THEN EXIT;
[outcome: outcome] ← YggTransaction.Finish[transID: moreDIDsTID, requestedOutcome: commit];
IF outcome = commit THEN EXIT;
ENDLOOP;
IF newHigh THEN {
m: ENTRY PROC ~ {
ENABLE UNWIND => {};
LastDIDAllocateable ← newLastDIDAllocateable^;
NextDID.didHigh ← newLastDIDAllocateable.didHigh;
NextDID.didLow ← 0;
GrabberCount ← 0;
BROADCAST didsCondition;
};
m[];
}
ELSE {
n: ENTRY PROC ~ {
LastDIDAllocateable.didLow ← newLastDIDAllocateable.didLow;
BROADCAST didsCondition;
GrabberCount ← 0;
};
n[];
};
};
DID access to recoverable DID Map
LookupInMap: PROC [did: YggDID.DID, tid: Camelot.tidT, doc: YggInternal.Document] RETURNS [found: BOOLFALSE, gottaBeInCache: BOOLFALSE] = {
notFound: BOOLFALSE;
extendedData: REF EntryDataBuffer ← NIL;
extendedDataCount: CARD ← 0;
lowExtendedToAccept: CARD ← 0;
highExtendedToAccept: CARD ← 1;
expectMore: BOOL ← TRUE;
expectedExtendedDataSize: CARD ← 0;
maxDataCount: CARD ← 0;
fromLink: DID ← YggEnvironment.nullDID;
toLink: DID ← YggEnvironment.nullDID;
linkType: Rope.ROPENIL;
updatesForTrans: LIST OF updates;
lookupProc: UNSAFE PROC [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT, overFlowPageProc: UNSAFE PROC RETURNS [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT]] ~ TRUSTED {
nowEntry: ExtendibleHash.Entry ← entry;
wordsLeft: INT ← wordsForEntriesOnPage;
WHILE wordsLeft > 0 DO
entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[nowEntry, LONG POINTER] + UNITS[CARD32];
IF nowEntry.entryWordSize > CARD[HashPageSizeInWords] THEN ERROR;
IF YggDID.CompareDIDs[did, @entryPtr.did] = equal AND entryPtr.extensionCount >= lowExtendedToAccept AND entryPtr.extensionCount <= highExtendedToAccept THEN {
found ← TRUE;
SELECT entryPtr.extensionCount FROM
0 => { -- normal, non-extended entry
index: INT ← 0;
expectMore ← FALSE; -- everything is in this entry
IF entryPtr.link THEN {
wordIndex: INT ← 0;
noChars: INT ← 0;
nullSeen: BOOLFALSE;
fromLink ← NEW[DIDRep];
PBasics.Move[dst: @fromLink, src: @entryPtr.data[0], nWords: WORDS[DIDRep]/WORDSInWords];
index ← WORDS[DIDRep]/WORDSInWords;
toLink ← NEW[DIDRep];
PBasics.Move[dst: @toLink, src: @entryPtr.data[index], nWords: WORDS[DIDRep]/WORDSInWords];
index ← index + index;
FOR wordIndex ← index, wordIndex + 1 UNTIL nullSeen DO
fourChars: PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR;
LOOPHOLE[fourChars, CARD] ← entryPtr.data[index];
FOR charNo: INT IN [0..Basics.charsPerWord) DO
IF fourChars[charNo] = 0C THEN {nullSeen← TRUE; EXIT;};
noChars ← noChars + 1;
ENDLOOP;
ENDLOOP;
IF noChars = 0 THEN {
index ← index + 1;
}
ELSE {
linkTypeRT: REF TEXTNIL;
linkTypeRT ← RefText.New[noChars];
nullSeen ← FALSE;
UNTIL nullSeen DO
fourChars: PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR;
LOOPHOLE[fourChars, CARD] ← entryPtr.data[index];
index ← index + 1;
FOR charNo: INT IN [0..Basics.charsPerWord) DO
IF fourChars[charNo] = 0C THEN {nullSeen← TRUE; EXIT;};
linkTypeRT[noChars] ← fourChars[charNo];
noChars ← noChars + 1;
ENDLOOP;
ENDLOOP;
linkTypeRT.length ← noChars;
linkType ← RefText.TrustTextAsRope[linkTypeRT];
};
};
[] ← parseDocPart[@entryPtr.data[index], entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry - index, doc];
};
1 => { -- normal, extended entries to follow
expectMore ← TRUE; -- everything is in this entry
expectedExtendedDataSize ← entryPtr.data[0];
extendedData ← NEW[EntryDataBuffer[expectedExtendedDataSize]];
extendedDataCount ← entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry - 1;
PBasics.Move[dst: @extendedData[0], src: @entryPtr.data[1], nWords: extendedDataCount];
};
>1 => { -- extended entry
IF extendedDataCount + entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry > maxDataCount THEN ERROR;
PBasics.Move[dst: @extendedData[extendedDataCount], src: @entryPtr.data[0], nWords: entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry];
extendedDataCount ← extendedDataCount + entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry;
};
ENDCASE => ERROR;
};
wordsLeft ← wordsLeft - nowEntry.entryWordSize - 1 ;
nowEntry ← nowEntry + ((nowEntry.entryWordSize + 1) * UNITS[CARD32]);
REPEAT FINISHED => notFound ← TRUE;
ENDLOOP;
};
updatesForTrans ← getUpdates[tid];
FOR ut: LIST OF updates ← updatesForTrans, ut.rest UNTIL ut = NIL DO
IF YggDID.EqualDIDs[ut.first.did, did] THEN RETURN[FALSE, TRUE]; -- if it has updates, then is should be in the cache - so why are we in this routine?
ENDLOOP;
WHILE expectMore DO
TRUSTED {ExtendibleHash.ReadEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], proc: lookupProc]};
IF notFound THEN EXIT;
ENDLOOP;
IF maxDataCount # 0 THEN TRUSTED {
IF maxDataCount # extendedDataCount THEN ERROR;
[] ← parseDocPart[@extendedData[0], maxDataCount, doc];
};
};
InsertIntoMap: PROC [did: DID, doc: YggInternal.Document, buffer: REF EntryDataBuffer, wordsInBuffer: INT, link: BOOL, refTid: REF Camelot.tidT] = {
extensionCount: INT ← 0;
foundEntry: BOOLTRUE;
lookupProc: UNSAFE PROC [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT, overFlowPageProc: UNSAFE PROC RETURNS [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT]] RETURNS[entryToUpdate: ExtendibleHash.Entry ← NIL] ~ TRUSTED {
Set extensionCount before calling
nowEntry: ExtendibleHash.Entry ← entry;
wordsLeft: INT ← wordsForEntriesOnPage;
WHILE wordsLeft > 0 DO
entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[nowEntry, LONG POINTER] + UNITS[CARD32];
IF nowEntry.entryWordSize > CARD[HashPageSizeInWords] THEN ERROR;
IF YggDID.CompareDIDs[did, @entryPtr.did] = equal AND entryPtr.extensionCount = extensionCount THEN {
entryToUpdate ← nowEntry;
RETURN;
};
wordsLeft ← wordsLeft - nowEntry.entryWordSize - 1 ;
nowEntry ← nowEntry + ((nowEntry.entryWordSize + 1) * UNITS[CARD32]);
ENDLOOP;
};
lookupForDelete: UNSAFE PROC [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT, overFlowPageProc: UNSAFE PROC RETURNS [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT]] RETURNS[entryToDelete: ExtendibleHash.Entry ← NIL] ~ TRUSTED {
Set extensionCount before calling
nowEntry: ExtendibleHash.Entry ← entry;
wordsLeft: INT ← wordsForEntriesOnPage;
foundEntry ← FALSE;
WHILE wordsLeft > 0 DO
entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[nowEntry, LONG POINTER] + UNITS[CARD32];
IF nowEntry.entryWordSize > CARD[HashPageSizeInWords] THEN ERROR;
IF YggDID.CompareDIDs[did, @entryPtr.did] = equal AND entryPtr.extensionCount = extensionCount THEN {
entryToDelete ← nowEntry;
foundEntry ← TRUE;
RETURN;
};
wordsLeft ← wordsLeft - nowEntry.entryWordSize - 1 ;
nowEntry ← nowEntry + ((nowEntry.entryWordSize + 1) * UNITS[CARD32]);
ENDLOOP;
};
totalEntrySize: INT;
totalEntrySize ← YggDIDMapPrivate.Word32OverheadPerEntry + wordsInBuffer;
IF totalEntrySize <= MaxWordsPerEntry THEN {
modifyProc: UNSAFE PROC [entry: ExtendibleHash.Entry] ~ TRUSTED {
entPtr: LONG POINTER TO YggDIDMapPrivate.Entry = LOOPHOLE[entry, LONG POINTER] + UNITS[CARD32];
entPtr.size ← totalEntrySize;
entPtr.extensionCount ← 0;
entPtr.did ← did^;
entPtr.link ← link;
PBasics.Move[dst: @entPtr.data[0], src: LOOPHOLE[buffer], nWords: wordsInBuffer];
};
extensionCount ← 0;
TRUSTED {ExtendibleHash.UpdateEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], words: totalEntrySize, findProc: lookupProc, modifyProc: modifyProc, updateType: insertOrReplace, clientPerCallData: refTid];};
extensionCount ← 1;
}
ELSE {
modifyProcFirst: UNSAFE PROC [entry: ExtendibleHash.Entry] ~ TRUSTED {
writeWords: INT ← MaxWordsPerEntry - YggDIDMapPrivate.Word32OverheadPerEntry - 1;
entPtr: LONG POINTER TO YggDIDMapPrivate.Entry = LOOPHOLE[entry, LONG POINTER] + UNITS[CARD32];
entPtr.size ← MaxWordsPerEntry;
entPtr.extensionCount ← 1;
entPtr.did ← did^;
entPtr.data[0] ← wordsInBuffer;
leftToWrite ← wordsInBuffer - writeWords;
PBasics.Move[dst: @entPtr.data[1], src: LOOPHOLE[buffer], nWords: writeWords];
writeOffset ← writeWords;
};
leftToWrite: INT ← 0;
writeOffset: INT ← 0;
extensionCount ← 1;
TRUSTED {ExtendibleHash.UpdateEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], words: MaxWordsPerEntry, findProc: lookupProc, modifyProc: modifyProcFirst, updateType: insertOrReplace, clientPerCallData: refTid]; };
extensionCount ← 0;
TRUSTED {ExtendibleHash.DeleteEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], proc: lookupForDelete, clientPerCallData: refTid];};
DO
modifyProcExtended: UNSAFE PROCEDURE [entry: ExtendibleHash.Entry] = UNCHECKED{
entPtr: LONG POINTER TO YggDIDMapPrivate.Entry = LOOPHOLE[entry, LONG POINTER] + UNITS[CARD32];
entPtr.size ← writeThisTime+YggDIDMapPrivate.Word32OverheadPerEntry;
entPtr.extensionCount ← extensionCount;
entPtr.did ← did^;
entPtr.data[0] ← writeThisTime;
PBasics.Move[dst: @entPtr.data[1], src: LOOPHOLE[buffer, LONG POINTER]+writeOffset, nWords: writeThisTime];
};
writeThisTime: INTMIN[MaxWordsPerEntry - YggDIDMapPrivate.Word32OverheadPerEntry, leftToWrite];
leftToWrite ← leftToWrite - writeThisTime;
extensionCount ← extensionCount + 1;
TRUSTED {ExtendibleHash.UpdateEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], words: writeThisTime + YggDIDMapPrivate.Word32OverheadPerEntry, findProc: lookupProc, modifyProc: modifyProcExtended, updateType: insertOrReplace, clientPerCallData: refTid]; };
IF leftToWrite < 0 THEN ERROR;
IF leftToWrite = 0 THEN EXIT;
ENDLOOP;
};
foundEntry ← TRUE;
WHILE foundEntry DO -- insure that old extra extentions do not exist
TRUSTED {ExtendibleHash.DeleteEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], proc: lookupForDelete, clientPerCallData: refTid];};
extensionCount ← extensionCount + 1;
ENDLOOP;
};
lock64Condition: CONDITION;
LockBucket64: ENTRY PROC [bucket: INT, tid: Camelot.tidT] ~ {
ENABLE UNWIND => {};
WHILE Lock64Table[bucket].interlock DO WAIT lock64Condition ENDLOOP;
Lock64Table[bucket].interlock ← TRUE;
Lock64Table[bucket].tid ← tid;
};
UnlockBucket64: ENTRY PROC [bucket: INT, tid: Camelot.tidT] ~ {
ENABLE UNWIND => {};
IF ~Lock64Table[bucket].interlock OR Lock64Table[bucket].tid # tid THEN ERROR;
Lock64Table[bucket].interlock ← FALSE;
Lock64Table[bucket].tid ← YggEnvironment.nullTransID;
BROADCAST lock64Condition;
};
getUpdates: PROC [tid: Camelot.tidT] RETURNS [updateList: LIST OF updates] ~ {
ERROR;
};
parseDocPart: PROC [dataToParse: LONG POINTER, size: CARD, doc: YggInternal.Document] RETURNS [found: BOOLFALSE] = {
This code must be the dual of writeDocToBuffer
parsedDocList: LIST OF YggDIDMapPrivate.DocItem ← NIL;
parsedDocListTail: LIST OF YggDIDMapPrivate.DocItem ← NIL;
index: CARD ← 0;
WHILE index < size*WORDSInWords DO
TRUSTED {
segIndex: LONG POINTER TO YggDIDMapPrivate.SegmentRep;
docPart: LONG POINTER TO YggDIDMapPrivate.DocPart;
newDocList: LIST OF YggDIDMapPrivate.DocItem ← NIL;
segmentsListTail: LIST OF YggDIDMapPrivate.SegmentItem ← NIL;
startOffset: CARD ← 0;
offsetToLastSegment: CARD ← 0;
indexName: ATOMNIL;
docPart ← LOOPHOLE[dataToParse + index];
IF index + docPart.docPartSize*WORDSInWords > size*WORDSInWords THEN ERROR;
IF LOOPHOLE[docPart.indexName, CARD16] # 0 THEN {
r: Rope.Text;
docPartPtr: YggDIDMapPrivate.DocPartPtr = LOOPHOLE[docPart];
indexNamePtr: LONG POINTER TO YggDIDMapPrivate.TextRep;
indexNamePtr ← @docPartPtr[docPartPtr.indexName];
IF indexNamePtr.length = 0 THEN ERROR;
offsetToLastSegment ← LOOPHOLE[indexNamePtr, CARD] - LOOPHOLE[docPart, CARD];
r ← Rope.NewText[indexNamePtr.length];
GetText[indexNamePtr, LOOPHOLE[r]];
indexName ← Atom.MakeAtom[r];
}
ELSE {
offsetToLastSegment ← docPart.docPartSize*WORDSInWords;
SELECT docPart.partType FROM
root => indexName ← $root;
contents => indexName ← $contents;
outlinks => indexName ← $outlinks;
attributes => indexName ← $attributes;
ENDCASE => ERROR;
};
newDocList ← LIST[[partType: docPart.partType, indexName: indexName, slotNo: docPart.slotNo, segments: NIL]];
segIndex ← LOOPHOLE[@docPart.segments];
WHILE (startOffset ← LOOPHOLE[segIndex, CARD] - LOOPHOLE[docPart, CARD]) < offsetToLastSegment DO
segSize: CARD ← 0;
runs: LIST OF YggDIDMapPrivate.RunItem ← NIL;
runTail: LIST OF YggDIDMapPrivate.RunItem ← NIL;
newSegListItem: LIST OF YggDIDMapPrivate.SegmentItem;
newSegListItem ← LIST[[firstPage: segIndex.firstPage, numberOfBytes: segIndex.numberOfBytes, runs: NIL]];
IF (segSize ← (WORDS[YggDIDMapPrivate.SegmentRep[segIndex.length]]/WORDSInWords)) > (docPart.docPartSize - (startOffset/WORDSInWords)) THEN ERROR; -- too many runs
FOR inx: CARDINAL IN [0..segIndex.length) DO
newRunList: LIST OF YggDIDMapPrivate.RunItem ← NIL;
newRunList ← LIST[[segmentId: segIndex.runs[inx].segmentId, segmentPage: segIndex.runs[inx].segmentPage, pages: segIndex.runs[inx].pages]];
IF runTail = NIL THEN runs ← runTail ← newRunList
ELSE {
runTail.rest ← newRunList;
runTail ← newRunList;
};
ENDLOOP;
newSegListItem.first.runs ← runs;
IF newDocList.first.segments = NIL THEN newDocList.first.segments ← segmentsListTail ← newSegListItem
ELSE {
segmentsListTail.rest ← newSegListItem;
segmentsListTail ← newSegListItem;
};
segIndex ← segIndex + segSize*WORDSInWords;
ENDLOOP;
IF parsedDocListTail = NIL THEN parsedDocList ← parsedDocListTail ← newDocList
ELSE {
parsedDocListTail.rest ← newDocList;
parsedDocListTail ← newDocList;
};
index ← index + docPart.docPartSize*WORDSInWords;
};
ENDLOOP;
doc.parsedDocList ← parsedDocList;
};
stockBuffer: REF EntryDataBuffer ← NIL;
CheckoutStockBuffer: ENTRY PROC RETURNS [buf: REF EntryDataBuffer] = {
ENABLE UNWIND => {};
IF stockBuffer = NIL THEN buf ← NEW[EntryDataBuffer[256]] ELSE { buf ← stockBuffer; stockBuffer ← NIL };
};
RecycleStockBuffer: ENTRY PROC [buf: REF EntryDataBuffer] = {
stockBuffer ← buf;
};
writeDocumentToMap: PROC [ doc: Document, refTid: REF Camelot.tidT] ~ {
buffer: REF EntryDataBuffer ← NIL;
stockBuffer: REF EntryDataBuffer ← NIL;
bufferSize: CARD ← 256;
link: BOOL;
wordsInBuffer: INT;
IF doc.destroyed THEN {
lookupForDelete: UNSAFE PROC [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT, overFlowPageProc: UNSAFE PROC RETURNS [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT]] RETURNS[entryToDelete: ExtendibleHash.Entry ← NIL] ~ TRUSTED {
Set extensionCount before calling
nowEntry: ExtendibleHash.Entry ← entry;
wordsLeft: INT ← wordsForEntriesOnPage;
foundEntry ← FALSE;
WHILE wordsLeft > 0 DO
entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[nowEntry, LONG POINTER] + UNITS[CARD32];
IF nowEntry.entryWordSize > CARD[HashPageSizeInWords] THEN ERROR;
IF YggDID.CompareDIDs[doc.did, @entryPtr.did] = equal AND entryPtr.extensionCount = extensionCount THEN {
entryToDelete ← nowEntry;
foundEntry ← TRUE;
noteUpdatedBuffer[entry, tid];
RETURN;
};
wordsLeft ← wordsLeft - nowEntry.entryWordSize - 1 ;
nowEntry ← nowEntry + ((nowEntry.entryWordSize + 1) * UNITS[CARD32]);
ENDLOOP;
};
extensionCount: INT ← 0;
foundEntry: BOOLTRUE;
WHILE foundEntry DO -- insure that old extra extentions do not exist
foundEntry ← FALSE;
TRUSTED {ExtendibleHash.DeleteEntry[hashTable: TheDIDMap, hashValue: [doc.did.didHigh, doc.did.didLow], proc: lookupForDelete, clientPerCallData: refTid];};
extensionCount ← extensionCount + 1;
ENDLOOP;
}
ELSE {
stockBuffer ← buffer ← CheckoutStockBuffer[];
DO
didNotFit: BOOLTRUE;
[wordsInBuffer: wordsInBuffer, didNotFit: didNotFit, link: link] ← writeDocToBuffer[doc, buffer, bufferSize];
IF didNotFit THEN {
bufferSize ← bufferSize + bufferSize;
buffer ← NEW[EntryDataBuffer[bufferSize]];
LOOP;
};
EXIT;
ENDLOOP;
InsertIntoMap[did: doc.did, doc: doc, buffer: buffer, wordsInBuffer: wordsInBuffer, link: link, refTid: refTid];
RecycleStockBuffer[stockBuffer];
};
};
writeDocToBuffer: PROC [ doc: Document, buffer: REF EntryDataBuffer, bufferSize: INT] RETURNS [wordsInBuffer: CARD ← 0, didNotFit: BOOLTRUE, link: BOOLFALSE] ~ {
This code must be the dual of parseDocPart
IF doc.fromDID # YggEnvironment.nullDID THEN {
linkTypeLen: INT ← 0;
linkTypeWords: CARD ← 0;
nowCharNum: INT ← 0;
link ← TRUE;
TRUSTED {PBasics.Move[dst: LOOPHOLE[@buffer[0]], src: LOOPHOLE[@(doc.fromDID^)], nWords: WORDS[DIDRep]/WORDSInWords]; };
wordsInBuffer ← WORDS[DIDRep]/WORDSInWords;
TRUSTED {PBasics.Move[dst: LOOPHOLE[@buffer[wordsInBuffer]], src: LOOPHOLE[@(doc.toDID^)], nWords: WORDS[DIDRep]/WORDSInWords]; };
wordsInBuffer ← wordsInBuffer + WORDS[DIDRep]/WORDSInWords;
linkTypeLen ← Rope.Length[doc.linkType];
linkTypeWords ← (linkTypeLen + PBasics.charsPerWord)/PBasics.charsPerWord;
IF linkTypeWords + wordsInBuffer > CARD[bufferSize] THEN RETURN[0, TRUE];
FOR i: INT IN [0..INT[linkTypeWords]) DO
fourChars: PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR;
FOR charNo: INT IN [0..Basics.charsPerWord) DO
IF nowCharNum >= linkTypeLen THEN fourChars[charNo] ← 0C
ELSE fourChars[charNo] ← Rope.Fetch[doc.linkType, nowCharNum];
nowCharNum ← nowCharNum + 1;
ENDLOOP;
TRUSTED{buffer[wordsInBuffer].card ← LOOPHOLE[fourChars, CARD];};
wordsInBuffer ← wordsInBuffer + 1;
ENDLOOP;
};
FOR parsedDocList: LIST OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, parsedDocList.rest UNTIL parsedDocList = NIL DO
indexNameWords: CARD ← 0;
segmentOffset: CARD ← 0;
docPart: LONG POINTER TO YggDIDMapPrivate.DocPart;
IF INT[wordsInBuffer] + WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[1]] > INT[bufferSize] THEN RETURN [0, TRUE];
TRUSTED {
docPart ← LOOPHOLE[@buffer[wordsInBuffer]];
docPart.compression ← YggDIDMapPrivate.notCompressed;
docPart.keepingModifications ← FALSE;
docPart.partType ← parsedDocList.first.partType;
docPart.extraFlags ← 0;
docPart.slotNo ← parsedDocList.first.slotNo;
docPart.indexName ← LOOPHOLE[0];
docPart.versionIdentifierString ← LOOPHOLE[0];
};
FOR segments: LIST OF YggDIDMapPrivate.SegmentItem ← parsedDocList.first.segments, segments.rest UNTIL segments = NIL DO
runIndex: CARD ← 0;
segIndex: LONG POINTER TO YggDIDMapPrivate.SegmentRep;
IF wordsInBuffer + segmentOffset + CARD[WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[1]]] > CARD[bufferSize] THEN RETURN [0, TRUE];
TRUSTED {
segIndex ← LOOPHOLE[@docPart.segments + segmentOffset];
segIndex.firstPage ← segments.first.firstPage;
segIndex.numberOfBytes ← segments.first.numberOfBytes;
segIndex.modificationDeltasOffset ← 0;
segIndex.lastUsedTime ← BasicTime.Now[];
segIndex.backUpTime ← BasicTime.nullGMT;
segIndex.backUpID ← 0;
};
FOR rl: LIST OF YggDIDMapPrivate.RunItem ← segments.first.runs, rl.rest UNTIL rl = NIL DO
IF wordsInBuffer + segmentOffset + CARD[WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[runIndex]]] > CARD[bufferSize] THEN RETURN [0, TRUE];
TRUSTED {segIndex.runs[runIndex] ← [segmentId: rl.first.segmentId, segmentPage: rl.first.segmentPage, pages: rl.first.pages]; };
runIndex ← runIndex + 1;
ENDLOOP;
TRUSTED {segIndex.length ← runIndex; };
segmentOffset ← segmentOffset + WORDS[YggDIDMapPrivate.SegmentRep[runIndex]]/WORDSInWords;
ENDLOOP;
TRUSTED {
docPart.docPartSize ← WORDS[YggDIDMapPrivate.DocPart]/WORDSInWords - WORDS[CARD]/WORDSInWords + segmentOffset + indexNameWords;
wordsInBuffer ← wordsInBuffer + docPart.docPartSize;
};
IF parsedDocList.first.indexName # NIL THEN {
SELECT parsedDocList.first.partType FROM
root, contents, outlinks, attributes => {};
ENDCASE => {
pName: Rope.Text;
pName ← Atom.GetPName[parsedDocList.first.indexName];
IF wordsInBuffer + WORDS[YggDIDMapPrivate.TextRep[Rope.Length[pName]]] > CARD[bufferSize] THEN RETURN [0, TRUE];
TRUSTED {
[docPart.indexName, indexNameWords] ← AppendText[dPP: docPart, text: pName, offset: docPart.docPartSize];
docPart.docPartSize ← docPart.docPartSize + indexNameWords;
};
wordsInBuffer ← wordsInBuffer + indexNameWords;
};
};
ENDLOOP;
RETURN[wordsInBuffer: wordsInBuffer, didNotFit: FALSE];
};
ExtendibleHash storage primitives
StartUpExtendibleHash: PROC [btreeInfo: REF, initialize: BOOL ← FALSE] = {
TheDIDMap ← ExtendibleHash.New [
storPrim: [referencePage: HashReferencePage, releasePage: HashReleasePage],
initialState: suspended
];
IF initialize THEN {
outcome: YggEnvironment.Outcome ← active;
doingInit ← TRUE;
initTrans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];
ExtendibleHash.Open[
hashTable: TheDIDMap,
storage: btreeInfo,
pageSize: HashPageSizeInWords,
initialize: initialize,
initialDirectorySize: 64
];
doingInit ← FALSE;
[outcome: outcome] ← YggTransaction.Finish[transID: initTrans, requestedOutcome: commit];
IF outcome # commit THEN ERROR;
}
ELSE {
ExtendibleHash.Open[
hashTable: TheDIDMap,
storage: btreeInfo,
pageSize: HashPageSizeInWords,
initialize: initialize,
initialDirectorySize: 64
];
};
};
debugReferencePageNumber: CARD ← 0;
debugReferencePageNumberCount: INT ← 0;
doingInit: BOOL ← FALSE;
initTrans: YggEnvironment.TransID;
BufferAndTransList: LIST OF BufferAndTransItem ← NIL;
BufferAndTransItem: TYPE = RECORD [
highBits: CARD,
trans: YggEnvironment.TransID
];
findTransForPage: ENTRY PROC [number: PageNumber] RETURNS [trans: YggEnvironment.TransID] ~ {
};
noteUpdatedBuffer: ENTRY PROC [entry: ExtendibleHash.Entry, trans: YggEnvironment.TransID] ~ {
highBits: CARD;
highBits ← LOOPHOLE[entry, CARD]/ExtendibleHash.wordsPerPage;
BufferAndTransList ← CONS[[highBits, trans], BufferAndTransList];
};
doneWithTransUpdatedBuffer: ENTRY PROC [trans: YggEnvironment.TransID] ~ {
highBits: CARD;
highBits ← LOOPHOLE[entry, CARD]/ExtendibleHash.wordsPerPage;
BufferAndTransList ← CONS[[highBits, trans], BufferAndTransList];
};
HashReferencePage: PUBLIC ExtendibleHash.ReferencePage = {
PROC [storage: PageStorage, number: PageNumber, type: ReferenceType, clientPerCallData: REF] RETURNS [ptr: PagePtr]
addVMPageSet: ENTRY PROC = {
CurrentVMPageSetsForDIDMap ← CONS[vMPageSet, CurrentVMPageSetsForDIDMap];
};
debugReferencePageNumberCount: INT ← 0;
vMPageSet: YggBuffMan.VMPageSet;
IF number = debugReferencePageNumber THEN debugReferencePageNumberCount ← debugReferencePageNumberCount + 1;
IF type = new THEN {
IF YggFile.Info[FileForDIDMap, YggEnvironment.nullTransID].size < (INT[number]+1)*FilePagesPerBtreePage THEN YggFile.SetSize[FileForDIDMap, (number+25)*FilePagesPerBtreePage, YggEnvironment.nullTransID];
vMPageSet ← YggBuffMan.ReadPages[fileHandle: FileForDIDMap, tid: YggEnvironment.nullTransID, pageRun: [firstPage: number+FilePagesPerBtreePage, count: FilePagesPerBtreePage]];
}
ELSE {
vMPageSet ← YggBuffMan.ReadPages[fileHandle: FileForDIDMap, tid: YggEnvironment.nullTransID, pageRun: [firstPage: number*FilePagesPerBtreePage+FilePagesPerBtreePage, count: FilePagesPerBtreePage]];
};
IF vMPageSet = NIL OR vMPageSet.rest # NIL THEN ERROR;
addVMPageSet[];
RETURN[vMPageSet.first.buffer];
};
HashReleasePage: PUBLIC ExtendibleHash.ReleasePage = {
PROC [storage: PageStorage, number: PageNumber, update: UpdateState, clientPerCallData: REF]
findVMPageSet: ENTRY PROC [number: ExtendibleHash.PageNumber] RETURNS [vmPageSet: LIST OF YggBuffMan.VMPageSet] = {
ENABLE UNWIND => {};
prevlovmps: LIST OF YggBuffMan.VMPageSet ← NIL;
FOR lovmps: LIST OF YggBuffMan.VMPageSet ← CurrentVMPageSetsForDIDMap, lovmps.rest UNTIL lovmps = NIL DO
IF lovmps.first.pageRun.segmentPage = INT[number]*FilePagesPerBtreePage+FilePagesPerBtreePage THEN {
RETURN[lovmps.first.vmPageSet];
};
prevlovmps ← lovmps;
ENDLOOP;
ERROR;
};
removeVMPageSet: ENTRY PROC = {
ENABLE UNWIND => {};
prevlovmps: LIST OF YggBuffMan.VMPageSet ← NIL;
FOR lovmps: LIST OF YggBuffMan.VMPageSet ← CurrentVMPageSetsForDIDMap, lovmps.rest UNTIL lovmps = NIL DO
IF lovmps.first.first.pageRun.firstPage = INT[number]*FilePagesPerBtreePage+FilePagesPerBtreePage THEN {
IF prevlovmps = NIL THEN CurrentVMPageSetsForDIDMap ← lovmps.rest
ELSE prevlovmps.rest ← lovmps.rest;
EXIT;
};
prevlovmps ← lovmps;
ENDLOOP;
};
debugReferencePageNumberCount: INT ← 0;
releaseState: YggBuffMan.ReleaseState;
vMPageSet: YggBuffMan.VMPageSet ← NIL;
refTid: REF Camelot.tidT ← NARROW[clientPerCallData];
IF number = debugReferencePageNumber THEN debugReferencePageNumberCount ← debugReferencePageNumberCount + 1;
IF refTid = NIL THEN ERROR;
SELECT update FROM
modified => {
releaseState ← writeBatchedNoWait;
};
unchanged => {
releaseState ← clean;
};
ENDCASE ;
vMPageSet ← findVMPageSet[number];
IF doingInit THEN {
YggBuffMan.WritePages[fileHandle: FileForDIDMap, tid: initTrans, to: number+FilePagesPerBtreePage, nPages: FilePagesPerBtreePage, from: vMPageSet.first.buffer];
}
ELSE IF update = modified THEN {
YggBuffMan.WritePages[fileHandle: FileForDIDMap, tid: refTid^, to: number+FilePagesPerBtreePage, nPages: FilePagesPerBtreePage, from: vMPageSet.first.buffer];
};
IF vMPageSet # NIL THEN {
YggBuffMan.ReleaseVMPageSet[vMPageSet: vMPageSet, releaseState: releaseState,
keep: TRUE];
removeVMPageSet[];
};
};
Internal red black procs
noteUseInTransaction: ENTRY PROC [doc: Document, tid: Camelot.tidT] ~ {
ENABLE UNWIND => {};
data: RedBlackTree.UserData;
scratchTIDObj.btid ← tid.top;
data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scratchTIDObj];
IF data = NIL THEN {
dt: DocTrans ← NEW[DocTransRep];
dt.btidForDoc ← NEW[TIDObjRep ← [tid.top]];
dt.docs ← LIST[doc];
RedBlackTree.Insert[transactionToModifiedDocumentMap, dt, scratchTIDObj];
}
ELSE {
dt: DocTrans;
dt ← NARROW[data];
FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO
IF YggDID.EqualDIDs[docList.first.did, doc.did] THEN RETURN;
ENDLOOP;
dt.docs ← CONS[doc, dt.docs];
};
};
GetKeyProc: RedBlackTree.GetKey = {
PROC [data: UserData] RETURNS [Key]
docTrans: DocTrans ← NARROW[data];
RETURN[ docTrans.btidForDoc ];
};
CompareProc: RedBlackTree.Compare = {
PROC [k: Key, data: UserData] RETURNS [Basics.Comparison]
fileTransData: DocTrans ← NARROW[data];
keyData: TIDObj ← NARROW[k];
SELECT keyData.highTicker FROM
> fileTransData.btidForDoc.highTicker => RETURN [greater];
< fileTransData.btidForDoc.highTicker => RETURN [less];
ENDCASE => {
SELECT keyData.lowTicker FROM
> fileTransData.btidForDoc.lowTicker => RETURN [greater];
< fileTransData.btidForDoc.lowTicker => RETURN [less];
ENDCASE => RETURN [equal];
};
};
stockTIDObj: TIDObj ← NIL;
CheckoutTIDObj: ENTRY PROC [btid: Camelot.btidT] RETURNS [k: TIDObj] = {
ENABLE UNWIND => {};
IF stockTIDObj = NIL THEN k ← NEW [TIDObjRep] ELSE { k ← stockTIDObj; stockTIDObj ← NIL };
k.btid ← btid;
};
RecycleTIDObj: ENTRY PROC [k: TIDObj] = {
stockTIDObj ← k;
};
Old enumeration proc
doc = NIL starts a new enumeration, and GetNext returns nextDocument = NIL when the enumeration is exhausted. The only guaranteed property of the enumeration is that all documents in existence during the entire enumeration will be visited at least once. Some documents may be seen more than once.
GetNext: PUBLIC --ENTRY-- PROCEDURE[doc: Document] RETURNS [nextDocument: Document] =
BEGIN -- errors defined in FileMap: none.
ENABLE UNWIND => NULL;
RETURN[EnumerateNext[doc]];
END;
Hash table management:
Explanation of client-supplied parameters:
The procedure ClientHashInit is called during hash table initialization, to allow the hash function to precompute values based on the range and to make any small adjustments to the range that are necessary.
HashDocument must:
be a REF type.
contain a field "next" of type HashDocument, under the exclusive control of the hash package.
Key is an arbitrary type.
SetKey sets the "key value" associated with a HashDocument. The key value must not change between the time the doc is Inserted into the table and the time it is deleted from the table.
Hash must be a function of the key value of the parameter "hashDocument".
EqualKeys must be the equality relation on the key values of the parameters "hashDocument1" and "hashDocument2".
Interface description:
InitializeHashTable: INTERNAL PROCEDURE[numHashSlotsDesired: NAT, hashTableZone:
ZONE, hashDocument: HashDocument];
errors: HashPkgCallerProgrammingError (numHashSlotsDesired = 0).
Insert: ENTRY PROCEDURE[hashDocument: HashDocument];
errors: HashPkgDuplicateKey.
LookupInCache: ENTRY PROCEDURE[key: Key] RETURNS [hashDocument: HashDocument];
returns hashDocument = NIL if not found.
Delete: ENTRY PROCEDURE[hashDocument: HashDocument];
errors: HashPkgCallerProgrammingError (not found).
EnumerateNext: INTERNAL PROCEDURE[prevHashDocument: HashDocument] RETURNS
[hashDocument: HashDocument];
errors: none.
prevHashDocument = NIL starts the enumeration, returned hashDocument = NIL is the end
of the enumeration. This procedure guarantees that any hashDocument in existence throughout
the entire enumeration will be seen. Other docs may or not not be seen. HashDocuments
may be seen more than once.
EnumerateWithProc: INTERNAL PROCEDURE[proc: PROCEDURE[hashDocument: HashDocument]
RETURNS[stop: BOOLEAN]];
errors: none.
client-supplied parameters to hash package begin here:
HashDocument: TYPE = Document;
Key: TYPE = YggDID.DID;
ClientHashInit: INTERNAL PROCEDURE[numHashSlotsDesired: NAT] RETURNS
[numHashSlotsAllowed: NAT] =
BEGIN
divisor, quotient, remainder: CARDINAL;
DO divisor ← 2;
DO
[quotient, remainder] ← Basics.DivMod[numHashSlotsDesired, divisor];
quotient ← numHashSlotsDesired / divisor;
remainder ← numHashSlotsDesired MOD divisor;
IF remainder = 0 THEN EXIT; -- this number is not prime.
IF quotient < divisor THEN RETURN[numHashSlotsDesired]; -- prime.
divisor ← divisor + 1;
ENDLOOP;
numHashSlotsDesired ← numHashSlotsDesired + 1;
ENDLOOP;
END;
ClientHash: INTERNAL PROCEDURE[hashDocument: HashDocument] RETURNS [index: NAT--[0..numHashSlots)--] = INLINE {
index ← YggDID.HashDID[hashDocument.did] MOD numHashSlots;
};
ClientEqualKeys: INTERNAL PROCEDURE[hashDocument1, hashDocument2: HashDocument] RETURNS [equal: BOOLEAN] = INLINE {
RETURN[YggDID.EqualDIDs[hashDocument1.did, hashDocument2.did]];
};
end of client-supplied parameters, start of invariant hash package code:
The INTERNAL procedures below expect to be called from a client procedure holding the module monitor lock, which protects the following data structures:
hashSlots: REF HashSlots ← NIL;
HashSlots: TYPE = RECORD[SEQUENCE nSlots: NAT OF HashDocument];
numHashSlots: NAT ← 0; -- boy, will they be sorry if they don't init this package.
lookupHashDocument: HashDocument ← NIL; -- for the "package's" use only.
lruListTop: Document ← NIL;
lruListTail: Document ← NIL;
numberOfHashDocuments: INT ← 0;
desiredNumberOfHashDocuments: INT ← 500;
errors:
HashPkgCallerProgrammingError: ERROR = CODE; -- various fatal conditions.
HashPkgDuplicateKey: ERROR = CODE; -- from Insert.
InitializeHashTable: INTERNAL PROCEDURE[numHashSlotsDesired: NAT, hashDocument: HashDocument] =
BEGIN -- errors: HashPkgCallerProgrammingError (numHashSlotsDesired = 0).
numHashSlots ← ClientHashInit[numHashSlotsDesired];
IF numHashSlots = 0 THEN ERROR HashPkgCallerProgrammingError;
lookupHashDocument ← hashDocument;
hashSlots ← NEW[HashSlots[numHashSlots]];
FOR index: NAT IN [0..numHashSlots)
DO hashSlots[index] ← NIL; ENDLOOP;
END;
Insert: ENTRY PROC [hashDocument: HashDocument] = {
errors: HashPkgDuplicateKey.
ENABLE UNWIND => {};
index: NAT ← ClientHash[hashDocument];
FOR newHashDocument: HashDocument ← hashSlots[index], newHashDocument.next
UNTIL newHashDocument = NIL DO
IF ClientEqualKeys[newHashDocument, hashDocument] AND ~hashDocument.destroyed
THEN ERROR HashPkgDuplicateKey;
ENDLOOP;
hashDocument.next ← hashSlots[index];
hashSlots[index] ← hashDocument;
SafeStorage.EnableFinalization[hashDocument];
numberOfHashDocuments ← numberOfHashDocuments + 1;
};
LookupInCache: ENTRY PROC [key: Key, destroyedOK: BOOLFALSE] RETURNS [hashDocument: HashDocument] = {
returns hashDocument = NIL if not found.
ENABLE UNWIND => {};
lookupHashDocument.did ← key;
FOR hashDocument ← hashSlots[ClientHash[lookupHashDocument]], hashDocument.next UNTIL hashDocument = NIL DO
IF ClientEqualKeys[hashDocument, lookupHashDocument] AND (destroyedOK OR ~hashDocument.destroyed) THEN {
SafeStorage.EnableFinalization[hashDocument];
IF hashDocument.lruListPrev # NIL OR hashDocument.lruListNext # NIL OR lruListTop = hashDocument THEN {
On lrulist. Remove it and use it.
IF hashDocument.lruListPrev = NIL THEN {
lruListTop ← hashDocument.lruListNext
}
ELSE {
hashDocument.lruListPrev.lruListNext ← hashDocument.lruListNext;
};
IF lruListTop # NIL THEN lruListTop.lruListPrev ← NIL;
IF hashDocument.lruListNext = NIL THEN {
lruListTail ← hashDocument.lruListPrev
}
ELSE hashDocument.lruListNext.lruListPrev ← hashDocument.lruListPrev;
IF lruListTail # NIL THEN lruListTail.lruListNext ← NIL;
};
hashDocument.lruListPrev ← NIL;
hashDocument.lruListNext ← NIL;
RETURN;
};
ENDLOOP;
RETURN[NIL];
};
FlushDocument: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT] = {
Flush a document out of the cache. Testing only!!
doc: YggInternal.Document ← NIL;
IF (doc ← LookupInCache[did]) = NIL THEN {
Delete[doc];
};
};
Delete: ENTRY PROC [hashDocument: HashDocument] = {
errors: HashPkgCallerProgrammingError (not found).
InnerDelete[hashDocument];
};
InnerDelete: INTERNAL PROC [hashDocument: HashDocument] = {
errors: HashPkgCallerProgrammingError (not found).
ENABLE UNWIND => {};
index: NAT ← ClientHash[hashDocument];
newHashDocument: HashDocument ← NIL;
prevHashDocument: HashDocument ← NIL;
FOR newHashDocument ← hashSlots[index], newHashDocument.next
UNTIL newHashDocument = NIL DO
IF ClientEqualKeys[newHashDocument, hashDocument] THEN EXIT;
prevHashDocument ← newHashDocument;
REPEAT FINISHED => ERROR HashPkgCallerProgrammingError;
ENDLOOP;
IF SafeStorage.IsFinalizationEnabled[newHashDocument] THEN {
newHashDocument.okToRemoveFromDIDMapCache ← TRUE;
}
ELSE {
IF prevHashDocument = NIL THEN hashSlots[index] ← hashDocument.next
ELSE prevHashDocument.next ← hashDocument.next;
numberOfHashDocuments ← numberOfHashDocuments - 1;
};
};
prevHashDocument = NIL starts the enumeration, returned hashDocument = NIL is the end of the enumeration. This procedure guarantees that any hashDocument in existence throughout the entire enumeration will be seen. Other docs may or not not be seen. HashDocuments may be seen more than once.
EnumerateNext: ENTRY PROCEDURE[prevHashDocument: HashDocument] RETURNS
[hashDocument: HashDocument] = {
errors: none.
ENABLE UNWIND => {};
index: NAT;
IF prevHashDocument = NIL THEN index ← 0
ELSE {
index ← ClientHash[prevHashDocument];
FOR hashDocument ← hashSlots[index], hashDocument.next UNTIL hashDocument = NIL DO
IF ClientEqualKeys[hashDocument, prevHashDocument] THEN GOTO found;
REPEAT
found => {
IF hashDocument.next # NIL THEN RETURN[hashDocument.next];
index ← index + 1;
};
ENDLOOP;
};
UNTIL index >= numHashSlots DO
IF hashSlots[index] # NIL THEN RETURN[hashSlots[index]];
index ← index + 1;
ENDLOOP;
RETURN[NIL];
};
EnumerateWithProc: INTERNAL PROCEDURE[proc: PROCEDURE[hashDocument: HashDocument]
RETURNS[stop: BOOLEAN]] = {
errors: none.
FOR index: NAT IN [0..numHashSlots) DO
FOR hashDocument: HashDocument ← hashSlots[index], hashDocument.next UNTIL hashDocument = NIL DO
IF proc[hashDocument] THEN RETURN;
ENDLOOP;
ENDLOOP;
};
end of invariant hash package code.
Finalization
FinalizationProcess: PROCEDURE = {
FinalizeEntry: ENTRY PROCEDURE [docm: Document] = {
ENABLE UNWIND => {};
IF SafeStorage.IsFinalizationEnabled[docm] THEN RETURN;
IF lruListTop = NIL THEN {
lruListTop ← docm;
}
ELSE {
lruListTail.lruListNext ← docm;
docm.lruListPrev ← lruListTail;
};
lruListTail ← docm;
docm ← NIL;
};
DO
doc: Document;
doc ← NARROW[SafeStorage.FQNext[finalizationQ]];
FinalizeEntry[doc];
doc ← NIL;
ENDLOOP;
};
Trim lru list
CacheTrimProcess: PROC = {
ticksToWait: Process.Ticks;
ticksToWait ← Process.MsecToTicks[197];
DO
overage: INT ← 0;
Process.Pause[ticksToWait];
overage ← numberOfHashDocuments - desiredNumberOfHashDocuments;
IF overage > 0 THEN TrimLruList[overage];
ENDLOOP;
};
TrimLruList: PROC [entriesToTrim: INT] = {
FOR entNo: INT IN [1..entriesToTrim] DO
grabTopOffOfLRU: ENTRY PROC ~ {
IF lruListTop # NIL THEN {
document ← lruListTop;
lruListTop ← lruListTop.lruListNext;
IF lruListTop = NIL THEN lruListTail ← NIL ELSE lruListTop.lruListPrev ← NIL;
document.lruListNext ← NIL;
document.lruListPrev ← NIL;
InnerDelete[document ! HashPkgCallerProgrammingError => CONTINUE; ]; -- CONTINUE so that finalization will work if this module is re-run during debugging
};
};
document: Document ← NIL;
grabTopOffOfLRU[];
ENDLOOP;
};
Initialization
InitDIDMap: PUBLIC PROC [firstTime: BOOL] ~ {
FileForDIDMap ← YggFile.OpenDIDMapFile[];
[optr: MaxDIDOptr, address: AddressForMaxDID] ← YggFile.GetMaxDIDAddress[];
RecoverableMaxDID ← LOOPHOLE[AddressForMaxDID];
TRUSTED {NextDID ← RecoverableMaxDID^;};
LastDIDAllocateable ← NextDID;
GrabSomeDIDs[];
StartUpExtendibleHash[NIL, firstTime];
IF firstTime THEN {
Init the Root DID
outcome: YggEnvironment.Outcome ← active;
trans: YggEnvironment.TransID;
WHILE outcome # commit DO
doc: YggInternal.Document ← NIL;
trans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];
doc ← NEW[DocumentRep ← [did: YggDIDPrivate.DIDForRootDIDs, modifiedDIDMap: TRUE, componentFiles: NIL, interlock: FALSE, next: NIL]];
Insert[doc ! HashPkgDuplicateKey => ERROR];
noteUseInTransaction[doc, trans];
YggNav.SetContents[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, contents: [YggRep.did, YggDIDPrivate.DIDForRootDIDs]];
[outcome: outcome] ← YggTransaction.Finish[transID: trans, requestedOutcome: commit];
ENDLOOP;
};
};
Initialize: PUBLIC ENTRY PROCEDURE[numHashSlotsDesired: NAT, fQLength: CARDINAL] = {
ENABLE UNWIND => {};
IF fQLength = 0 THEN RETURN WITH ERROR ZeroIllegal;
WORDSInWords ← PBasics.bytesPerWord/BYTES[WORD];
finalizationQ ← SafeStorage.NewFQ[fQLength]; -- do this before
SafeStorage.EstablishFinalization[CODE[DocumentRep], 1, finalizationQ ! SafeStorage.CantEstablishFinalization => CONTINUE]; -- allocating any FileObjects.
TRUSTED BEGIN Process.Detach[FORK FinalizationProcess[]]; END;
TURN FINALIZATION BACK ON AFTER DEBUGGING
TRUSTED BEGIN Process.Detach[FORK CacheTrimProcess[]]; END;
secondLookupHashDocument ← NEW[DocumentRep ← [did: NIL, componentFiles: NIL, interlock: FALSE, next: NIL]];
InitializeHashTable[numHashSlotsDesired, secondLookupHashDocument];
TRUSTED {
Process.InitializeCondition[@aShortTime, Process.MsecToTicks[50]];
Process.InitializeCondition[@didsCondition, Process.MsecToTicks[177]];
Process.InitializeCondition[@lock64Condition, Process.MsecToTicks[43]];
};
transactionToModifiedDocumentMap ← RedBlackTree.Create[getKey: GetKeyProc, compare: CompareProc];
FOR h: CARD IN [0..64) DO
index: CARD ← 0;
hLeft: CARD ← h;
addPowerOf2: CARD ← 32;
WHILE hLeft # 0 DO
IF hLeft MOD 2 = 1 THEN index ← index + addPowerOf2;
addPowerOf2 ← addPowerOf2 /2;
hLeft ← hLeft / 2;
ENDLOOP;
Hash64Table[h] ← index;
ENDLOOP;
};
Initialize[129, 250];
END.