[Indigo]<Yggdrasil>YggRepCM>YggDIDMapImpl.mesa!76

YggDIDMapImpl.mesa

Last edited by

Bob Hagmann July 7, 1989 9:27:26 am PDT

DIRECTORY

Atom USING [GetPName, MakeAtom],

Basics USING [BITAND, charsPerWord, Comparison],

BasicTime USING [Now, nullGMT],

ExtendibleHash USING [DeleteEntry, Entry, ExtendibleHashTable, New, Open, PageNumber, ReadEntry, ReferencePage, ReleasePage, UpdateEntry],

Camelot USING [btidT, DSLogNewValue, DSPinObject, -- ErSuccess, -- optrT, -- TABegin, TAEnd, -- tidT],

Mach USING [kernReturnT, -- KernSuccess, -- pointerT],

Process USING [Detach, InitializeCondition, MsecToTicks, Pause, Ticks],

PBasics USING [BITAND, ByteBlt, bytesPerWord, charsPerWord, Move],

RedBlackTree USING [Compare, Create, Delete, GetKey, Insert, Lookup, Table, UserData],

RefText USING [New, TrustTextAsRope],

Rope USING [Equal, Fetch, Length, NewText, ROPE, Substr, Text],

SafeStorage USING [CantEstablishFinalization, EnableFinalization, EstablishFinalization, FinalizationQueue, FQNext, IsFinalizationEnabled, NewFQ],

YggBuffMan USING [ReadPages, ReleaseState, ReleaseVMPageSet, VMPageSet, WritePages],

YggDID USING [CompareDIDs, DID, EqualDIDs, HashDID, ValidateDID],

YggDIDMap,

YggDIDMapPrivate USING [DocItem, DocPart, DocPartPtr, Document, DocumentRep, Entry, EntryPtr, notCompressed, PartType, RunItem, SegmentItem, SegmentRep, TextRep, TextRP, Word32OverheadPerEntry],

YggDIDPrivate USING [DIDForRootDIDs, DIDRep],

YggEnvironment USING [dsPort, nullDID, nullTransID, Outcome, -- taPort, -- TransID],

YggFile USING [BytesForPages, FileHandle, GetMaxDIDAddress, Info, Open, OpenDIDMapFile, PageCount, SetSize],

YggFixedNames USING [IndexPrefix, IndexPrefixSize],

YggInternal USING [Document, FileHandle],

YggNav USING [GetProperty, SetContents, SetProperty],

YggRep USING [Attribute, did, TypedPrimitiveElement],

YggTransaction USING [CreateTrans, Finish];

YggDIDMapImpl: CEDAR MONITOR

IMPORTS Atom, Basics, BasicTime, Camelot, ExtendibleHash, PBasics, Process, RedBlackTree, RefText, Rope, SafeStorage, YggBuffMan, YggDID, YggDIDPrivate, YggEnvironment, YggFile, YggFixedNames, YggNav, YggTransaction

EXPORTS YggInternal, YggDIDMap, YggDID =

BEGIN

Data types and variables

Document: TYPE = REF DocumentRep;

DocumentRep: PUBLIC TYPE = YggDIDMapPrivate.DocumentRep;

DID: PUBLIC TYPE ~ REF DIDRep;

DIDRep: PUBLIC TYPE ~ YggDIDPrivate.DIDRep;

HashPageSizeInWords: INT = 1024;

MaxWordsPerEntry: INT = HashPageSizeInWords/3 - 5;

FilePagesPerBtreePage: INT = 1;

Hash64Table: ARRAY [0..64) OF CARD;

Lock64Table: ARRAY [0..64) OF Lock64Item;

Lock64Item: TYPE = RECORD[

interlock: BOOL ← FALSE,

tid: Camelot.tidT

];

TheDIDMap: ExtendibleHash.ExtendibleHashTable;

FileForDIDMap: YggFile.FileHandle;

CurrentVMPageSetsForDIDMap: LIST OF YggBuffMan.VMPageSet ← NIL;

NextDID: DIDRep;

LastDIDAllocateable: DIDRep;

MaxDIDOptr: Camelot.optrT;

AddressForMaxDID: Mach.pointerT;

RecoverableMaxDID: LONG POINTER TO DIDRep;

LastDidLow: CARD ← 1000000000;

NumberOfDIDsToGrap: CARD ← 200;

ThresholdToGrabDIDs: CARD ← 20;

KeyObject: TYPE = RECORD [did: YggDID.DID, extensionCount: INT32];

EntryDataBuffer: TYPE = RECORD[SEQUENCE COMPUTED CARD16 OF EntryWord];

EntryWord: TYPE = MACHINE DEPENDENT RECORD[

SELECT OVERLAID * FROM

card => [card: CARD32],

int => [int: INT32],

pair => [hi, lo: CARD16],

bytes => [hh, hl, lh, ll: BYTE],

bits => [bits: PACKED ARRAY [0..32) OF BOOL],

ENDCASE

];

updateAction: TYPE = {create, delete, newValue};

updates: TYPE = RECORD[

did: YggDID.DID,

tid: Camelot.tidT,

action: updateAction,

doc: Document

];

TIDObj: TYPE = REF TIDObjRep;

TIDObjRep: TYPE = RECORD [ btid: Camelot.btidT];

scratchTIDObj: TIDObj ← NEW[TIDObjRep];

DocTrans: TYPE = REF DocTransRep;

DocTransRep: TYPE = RECORD [

btidForDoc: TIDObj,

docs: LIST OF Document ← NIL,

bucketsToUnlatch: LIST OF CARD ← NIL

];

transactionToModifiedDocumentMap: RedBlackTree.Table;

finalizationQ: SafeStorage.FinalizationQueue;

ZeroIllegal: --CALLING-- ERROR = CODE;

secondLookupHashDocument: HashDocument ← NIL; -- wierdness to protect object from finalization.

WORDSInWords: CARD ← 0;

Exported procedures for documents

MarkInterestInDID: PUBLIC PROC [did: YggDID.DID] ~ {

Notes continued interest in this did.

[] ← LookupInCache[did, FALSE];

};

OpenDocumentFromDID: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT, destroyedOK: BOOL] RETURNS [doc: YggInternal.Document ← NIL] = {

Finds or creates a Document for the supplied did and tid.

tryCount: CARD ← 0;

tryAgain: BOOL ← FALSE;

IF (doc ← LookupInCache[did, destroyedOK]) = NIL THEN {

found: BOOL ← FALSE;

gottaBeInCache: BOOL ← FALSE;

doc ← NEW[DocumentRep ← [did: did, componentFiles: NIL, interlock: TRUE, next: NIL]];

[found, gottaBeInCache] ← LookupInMap[did, tid, doc];

IF gottaBeInCache THEN {

doc ← LookupInCache[did, destroyedOK];

IF doc = NIL THEN ERROR;

}

ELSE {

IF ~found THEN RETURN [NIL];

Insert[doc ! HashPkgDuplicateKey => {

tryAgain ← TRUE;

CONTINUE;

};

];

Document is now inserted into the cache. Interlock is set. Open up the componentFiles.

IF ~tryAgain THEN openFilesFromParsedDocList[doc, did, tid];

};

}

ELSE { -- found in cache

waitForInterlockDone: ENTRY PROC ~ {

ENABLE UNWIND => {};

WHILE doc.interlock DO WAIT aShortTime ENDLOOP;

};

IF doc.interlock THEN waitForInterlockDone[];

};

IF ~tryAgain THEN {

releaseDoc[doc];

EXIT;

};

tryCount ← tryCount + 1;

IF tryCount > 25 THEN ERROR;

ENDLOOP;

};

GetDID: PUBLIC PROC [doc: Document] RETURNS [did: YggDID.DID] ~ {

errors defined in this interface: none.

RETURN[doc.did];

};

GetLinkInfo: PUBLIC PROC [doc: Document] RETURNS [fromDID, toDID: YggDID.DID, linkType: Rope.ROPE] ~ {

Get the link info for the Document.

RETURN[doc.fromDID, doc.toDID, doc.linkType];

};

SwapLinkInfo: PUBLIC PROC [doc: Document, newFromDID, newToDID: YggDID.DID, newLinkType: Rope.ROPE] RETURNS [oldFromDID, oldToDID: YggDID.DID, oldLinkType: Rope.ROPE] ~ {

Swap the link info for the document, and report what it used to be.

oldFromDID ← doc.fromDID;

doc.fromDID ← newFromDID;

oldToDID ← doc.toDID;

doc.toDID ← newToDID;

oldLinkType ← doc.linkType;

doc.linkType ← newLinkType;

};

GetComponentFiles: PUBLIC PROC [doc: Document] RETURNS [componentFiles: LIST OF YggInternal.FileHandle] ~ {

no errors possible, but may return NIL.

RETURN[doc.componentFiles];

};

AddComponentFile: PUBLIC ENTRY PROC [doc: Document, componentFile: YggInternal.FileHandle, tid: Camelot.tidT] ~ {

ENABLE UNWIND => {};

lastCF: LIST OF YggInternal.FileHandle ← NIL;

FOR files: LIST OF YggInternal.FileHandle ← doc.componentFiles, files.rest UNTIL files = NIL DO

lastCF ← files;

ENDLOOP;

IF lastCF = NIL THEN doc.componentFiles ← CONS[componentFile, NIL]

ELSE lastCF.rest ← CONS[componentFile, NIL];

};

RemoveComponentFile: PUBLIC PROC [doc: Document, componentFile: YggInternal.FileHandle, tid: Camelot.tidT] ~ {

lastCF: LIST OF YggInternal.FileHandle ← NIL;

FOR files: LIST OF YggInternal.FileHandle ← doc.componentFiles, files.rest UNTIL files = NIL DO

IF files.first = componentFile THEN {

IF lastCF = NIL THEN doc.componentFiles ← files.rest

ELSE lastCF.rest ← files.rest;

EXIT;

};

lastCF ← files;

ENDLOOP;

};

GetName: PUBLIC PROC [doc: Document] RETURNS [Rope.ROPE] ~ {

no errors possible, but may return NIL.

RETURN[doc.name];

};

CreateNewDID: PUBLIC PROC [tid: Camelot.tidT] RETURNS [did: YggDID.DID ] ~ {

doc: YggInternal.Document ← NIL;

did ← GetNextDID[];

doc ← NEW[DocumentRep ← [did: did, modifiedDIDMap: TRUE, componentFiles: NIL, interlock: FALSE, next: NIL]];

Insert[doc ! HashPkgDuplicateKey => ERROR];

noteUseInTransaction[doc, tid];

};

CreateExplicitDID: PUBLIC PROC [tid: Camelot.tidT, did: YggDID.DID] RETURNS [ok: BOOL ← TRUE ] ~ {

doc: YggInternal.Document ← NIL;

IF YggDID.ValidateDID[did] THEN RETURN [FALSE];

doc ← NEW[DocumentRep ← [did: did, modifiedDIDMap: TRUE, componentFiles: NIL, interlock: FALSE, next: NIL]];

Insert[doc ! HashPkgDuplicateKey => {ok ← FALSE; CONTINUE}];

IF ok THEN noteUseInTransaction[doc, tid];

};

DestroyDID: PUBLIC PROC [tid: Camelot.tidT, did: YggDID.DID] ~ {

doc: YggInternal.Document ← NIL;

doc ← OpenDocumentFromDID[did: did, tid: tid, destroyedOK: FALSE];

doc.destroyed ← TRUE;

};

GetRootDIDs: PUBLIC PROC RETURNS [dids: LIST OF YggDID.DID ← NIL] ~ {

trans: YggEnvironment.TransID;

propertyExists: BOOL ← FALSE;

property: LIST OF YggRep.Attribute ← NIL;

trans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];

[propertyExists: propertyExists, property: property] ← YggNav.GetProperty[trans: YggEnvironment.nullTransID, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs"];

IF propertyExists THEN {

lastDIDs: LIST OF YggDID.DID ← NIL;

IF property.rest # NIL THEN ERROR;

IF property.first.value.rest # NIL THEN ERROR;

FOR lotpe: LIST OF YggRep.TypedPrimitiveElement ← property.first.value.first.valueSet, lotpe.rest UNTIL lotpe = NIL DO

didInList: DID;

IF lotpe.first.docType # YggRep.did THEN ERROR;

didInList ← NARROW[lotpe.first.bits];

IF dids = NIL THEN {dids ← LIST[didInList]; lastDIDs ← dids}

ELSE {lastDIDs.rest ← LIST[didInList]; lastDIDs ← lastDIDs.rest;};

ENDLOOP;

};

[] ← YggTransaction.Finish[transID: trans, requestedOutcome: commit];

};

RootDIDLatch: BOOL ← FALSE;

LatchRootDID: ENTRY PROC ~ {

ENABLE UNWIND => {};

WHILE RootDIDLatch DO WAIT aShortTime ENDLOOP;

RootDIDLatch ← TRUE;

};

UnlatchRootDID: ENTRY PROC ~ {

RootDIDLatch ← FALSE;

};

AddRootDID: PUBLIC PROC [did: YggDID.DID] RETURNS [ok: BOOL ← FALSE] ~ {

trans: YggEnvironment.TransID;

propertyExists: BOOL ← FALSE;

property: LIST OF YggRep.Attribute ← NIL;

alreadyRoot: BOOL ← FALSE;

newProperty: YggRep.Attribute;

LatchRootDID[];

trans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];

[propertyExists: propertyExists, property: property] ← YggNav.GetProperty[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs"];

newProperty ← [attributeName: "$rootDIDs", ordered: TRUE, value: LIST[[fieldName: "", valueSet: LIST [[docType: YggRep.did, bits: did]]]]];

IF propertyExists THEN {

lastDIDs: LIST OF YggRep.TypedPrimitiveElement ← NIL;

IF property.rest # NIL THEN ERROR;

IF property.first.value.rest # NIL THEN ERROR;

FOR lotpe: LIST OF YggRep.TypedPrimitiveElement ← property.first.value.first.valueSet, lotpe.rest UNTIL lotpe = NIL DO

didInList: DID;

IF lotpe.first.docType # YggRep.did THEN ERROR;

didInList ← NARROW[lotpe.first.bits];

IF YggDID.EqualDIDs[did, didInList] THEN {alreadyRoot ← TRUE; EXIT};

lastDIDs ← lotpe;

ENDLOOP;

IF ~alreadyRoot THEN lastDIDs.rest ← LIST[[docType: YggRep.did, bits: did]];

}

ELSE property ← LIST[newProperty];

IF ~alreadyRoot THEN YggNav.SetProperty[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs", property: property.first, appendProperty: FALSE];

[] ← YggTransaction.Finish[transID: trans, requestedOutcome: commit];

UnlatchRootDID[];

};

RemoveRootDID: PUBLIC PROC [did: YggDID.DID] RETURNS [ok: BOOL ← FALSE] ~ {

trans: YggEnvironment.TransID;

propertyExists: BOOL ← FALSE;

property: LIST OF YggRep.Attribute ← NIL;

isARoot: BOOL ← FALSE;

newProperty: YggRep.Attribute;

LatchRootDID[];

trans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];

[propertyExists: propertyExists, property: property] ← YggNav.GetProperty[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs"];

newProperty ← [attributeName: "$rootDIDs", ordered: TRUE, value: LIST[[fieldName: "", valueSet: LIST [[docType: YggRep.did, bits: did]]]]];

IF propertyExists THEN {

prevDIDs: LIST OF YggRep.TypedPrimitiveElement ← NIL;

IF property.rest # NIL THEN ERROR;

IF property.first.value.rest # NIL THEN ERROR;

FOR lotpe: LIST OF YggRep.TypedPrimitiveElement ← property.first.value.first.valueSet, lotpe.rest UNTIL lotpe = NIL DO

didInList: DID;

IF lotpe.first.docType # YggRep.did THEN ERROR;

didInList ← NARROW[lotpe.first.bits];

IF YggDID.EqualDIDs[did, didInList] THEN {

isARoot← TRUE;

IF prevDIDs = NIL THEN property.first.value.first.valueSet ← lotpe.rest

ELSE prevDIDs.rest ← lotpe.rest;

EXIT;

};

prevDIDs ← lotpe;

ENDLOOP;

};

IF isARoot THEN YggNav.SetProperty[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, propertyName: "$rootDIDs", property: property.first, appendProperty: FALSE];

[] ← YggTransaction.Finish[transID: trans, requestedOutcome: commit];

UnlatchRootDID[];

RETURN[isARoot];

};

Exported for document modifications

aShortTime: CONDITION;

grapDoc: ENTRY PROC [doc: YggInternal.Document] ~ {

ENABLE UNWIND => {};

WHILE doc.interlock DO WAIT aShortTime ENDLOOP;

doc.interlock ← TRUE;

};

releaseDoc: ENTRY PROC [doc: YggInternal.Document] ~ {doc.interlock ← FALSE};

RemoveRuns: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT, runList: YggDIDMap.RunList] ~ {

Normally, only called by the file package during it's pre-commit. tid is a top level transaction. Runs listed are removed from the object. Also used when an object is migrated. Caller guarentees that PreCommit will later be called with this tid (if the system doesn't crash).

The document is updated in the cache. PreCommit will make the updates to the disk resident B-tree(s). If Commit is later called, then all is well. If Abort is called, then the cache entry is destroyed. Undo is done in the B-trees by physical pages. Subsequent use of the document requires rebuilding the document object.

doc: YggInternal.Document ← NIL;

IF (doc ← LookupInCache[did, TRUE]) = NIL THEN ERROR

ELSE {

doc.modifiedDIDMap ← TRUE;

grapDoc[doc];

noteUseInTransaction[doc, tid];

FOR rl: YggDIDMap.RunList ← runList, rl.rest UNTIL rl = NIL DO

pdl: LIST OF YggDIDMapPrivate.DocItem;

prevPdl: LIST OF YggDIDMapPrivate.DocItem ← NIL;

runToDelete: YggDIDMap.Run ← rl.first;

FOR pdl ← doc.parsedDocList, pdl.rest UNTIL pdl = NIL DO

segments: LIST OF YggDIDMapPrivate.SegmentItem;

prevSegments: LIST OF YggDIDMapPrivate.SegmentItem;

FOR segments ← pdl.first.segments, segments.rest UNTIL segments = NIL DO

runs: LIST OF YggDIDMapPrivate.RunItem;

prevRuns: LIST OF YggDIDMapPrivate.RunItem;

FOR runs ← segments.first.runs, runs.rest UNTIL runs = NIL DO

firstIntersection: INT ← -1;

lastIntersection: INT ← -1;

runLast: CARD ← 0;

runToDeleteLast: CARD ← 0;

IF runs.first.segmentId # runToDelete.segmentId THEN {-- wrong segment

prevRuns ← runs;

LOOP;

};

firstIntersection ← MAX[runs.first.segmentPage, runToDelete.segmentPage];

lastIntersection ← MIN[
(runLast ← runs.first.segmentPage + runs.first.pages - 1),
(runToDeleteLast ← runToDelete.segmentPage + runToDelete.pages - 1) ];

IF lastIntersection < firstIntersection THEN LOOP; -- no intersection

IF firstIntersection = INT[runs.first.segmentPage] AND lastIntersection = INT[runLast] THEN { -- run is eliminated

IF prevRuns = NIL THEN segments.first.runs ← runs.rest

ELSE prevRuns.rest ← runs.rest;

LOOP;

}

ELSE {

runs.first.segmentPage ← firstIntersection;

runs.first.pages ← lastIntersection - firstIntersection + 1;

};

prevRuns ← runs;

ENDLOOP;

IF segments.first.runs = NIL THEN { -- no more runs, so dump the segment too

IF prevSegments = NIL THEN pdl.first.segments ← segments.rest

ELSE prevSegments.rest ← segments.rest;

LOOP;

};

prevSegments ← segments;

ENDLOOP;

IF pdl.first.segments = NIL THEN { -- no more segments, so dump the DocItem too

IF prevPdl = NIL THEN doc.parsedDocList ← pdl.rest

ELSE prevPdl.rest ← pdl.rest;

LOOP;

};

prevPdl ← pdl;

ENDLOOP;

releaseDoc[doc];

};

AddRuns: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT, fileUse: ATOM, runList: YggDIDMap.RunList] ~ {

Normally, only called by the file package during it's pre-commit. tid is a top level transaction. Runs listed are added to the object. Also used when an object is removed from the disk cache.

doc: YggInternal.Document ← NIL;

partType: YggDIDMapPrivate.PartType;

partType ← mapFileUseToPartType[fileUse];

IF (doc ← LookupInCache[did]) = NIL THEN ERROR

ELSE {

lastPDL: LIST OF YggDIDMapPrivate.DocItem ← NIL;

thisPDL: LIST OF YggDIDMapPrivate.DocItem ← NIL;

doc.modifiedDIDMap ← TRUE;

grapDoc[doc];

noteUseInTransaction[doc, tid];

FOR pdl: LIST OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, pdl.rest UNTIL pdl = NIL DO

IF pdl.first.indexName = fileUse THEN {

thisPDL ← pdl;

EXIT;

};

lastPDL ← pdl;

REPEAT FINISHED => {

thisPDL ← LIST[[partType, fileUse, 0, NIL]];

IF lastPDL = NIL THEN doc.parsedDocList ← thisPDL

ELSE lastPDL.rest ← thisPDL;

};

ENDLOOP; -- pdl

IF thisPDL = NIL THEN ERROR;

FOR runs: YggDIDMap.RunList ← runList, runs.rest UNTIL runs = NIL DO

lastSegment: LIST OF YggDIDMapPrivate.SegmentItem ← NIL;

FOR segs: LIST OF YggDIDMapPrivate.SegmentItem ← thisPDL.first.segments, segs.rest UNTIL segs = NIL DO

lastRun: LIST OF YggDIDMapPrivate.RunItem ← NIL;

runPages: YggFile.PageCount ← 0;

FOR lori: LIST OF YggDIDMapPrivate.RunItem ← segs.first.runs, lori.rest UNTIL lori = NIL DO

runPages ← runPages + lori.first.pages;

lastRun ← lori;

ENDLOOP;

IF INT[segs.first.firstPage] + runPages = runs.first.firstPage THEN { -- append at end of segment

lastRun.rest ← CONS[[segmentId: runs.first.segmentId, segmentPage: runs.first.segmentPage, pages: runs.first.pages], NIL];

EXIT; -- appended at end of segment; go do the next run

};

lastSegment ← segs;

REPEAT FINISHED => { -- make a new segment with a singleton run

newItem: LIST OF YggDIDMapPrivate.SegmentItem;

newItem ← LIST[[firstPage: runs.first.firstPage, numberOfBytes: YggFile.BytesForPages[runs.first.pages], runs: LIST[[segmentId: runs.first.segmentId, segmentPage: runs.first.segmentPage, pages: runs.first.pages]]]];

IF lastSegment = NIL THEN thisPDL.first.segments ← newItem

ELSE lastSegment.rest ← newItem;

};

ENDLOOP;

releaseDoc[doc];

};

SetByteSize: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT, fileUse: ATOM, bytes: INT] ~ {

Normally, only called by the file package during it's pre-commit. tid is a top level transaction. The size of valid bytes is set to this amount, but the pages for the file are not changed.

doc: YggInternal.Document ← NIL;

partType: YggDIDMapPrivate.PartType;

partType ← mapFileUseToPartType[fileUse];

IF (doc ← LookupInCache[did]) = NIL THEN ERROR

ELSE {

doc.modifiedDIDMap ← TRUE;

grapDoc[doc];

noteUseInTransaction[doc, tid];

FOR pdl: LIST OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, pdl.rest UNTIL pdl = NIL DO

IF pdl.first.indexName = fileUse THEN {

FOR segs: LIST OF YggDIDMapPrivate.SegmentItem ← pdl.first.segments, segs.rest UNTIL segs = NIL DO

firstByte: CARD;

firstByte ← YggFile.BytesForPages[segs.first.firstPage];

IF INT[firstByte] > bytes THEN segs.first.numberOfBytes ← 0

ELSE {

runPages: YggFile.PageCount ← 0;

runBytes: CARD ← 0;

FOR lori: LIST OF YggDIDMapPrivate.RunItem ← segs.first.runs, lori.rest UNTIL lori = NIL DO

runPages ← runPages + lori.first.pages;

ENDLOOP;

runBytes ← YggFile.BytesForPages[runPages];

segs.first.numberOfBytes ← IF INT[firstByte + runBytes] > bytes THEN bytes - firstByte ELSE runBytes;

};

ENDLOOP;

EXIT;

};

REPEAT FINISHED => {

IF bytes > 0 THEN ERROR;

};

ENDLOOP; -- pdl

releaseDoc[doc];

};

Utilities

Hash64DID: PROC [did: DID] RETURNS [bucket: CARD] ~ {

RETURN [Hash64Table[Basics.BITAND[03Fh, Hash64[did.didLow]]]];

};

Hash64: PROC [c: CARD] RETURNS [bucket: CARD] ~ {

RETURN [Hash64Table[PBasics.BITAND[03Fh, c]]];

};

mapFileUseToPartType: PROC [fileUse: ATOM] RETURNS [partType: YggDIDMapPrivate.PartType] ~ {

SELECT fileUse FROM

$root => partType ← root;

$contents => partType ← contents;

$outlinks => partType ← outlinks;

$attributes => partType ← attributes;

$directoryContents => partType ← directoryContents;

ENDCASE => {

pName: Rope.Text;

pName ← Atom.GetPName[fileUse];

IF Rope.Equal[YggFixedNames.IndexPrefix, Rope.Substr[pName, 0, YggFixedNames.IndexPrefixSize]] THEN {

partType ← index;

}

ELSE ERROR;

};

Exported procedures for transactions

PreCommit: PUBLIC PROC [tid: Camelot.tidT] ~ {

Take all the modified documents from the transaction and write out their changes to the DID Map.

Find all the hash pages that have to be updated. Process them in ascending order, doing all of the updates for the whole page before doing the next page. We can't deadlock within DIDMap update since we've ordered the locks.

First, lock the page for exclusive access. Perform the update(s) including the Camelot pin and log dance. Downgrade to lock to allow readers (they can see uncommitted data).

If the transaction commits, completely unlock the tree. If the transaction aborts, lock the tree for exclusive access, perform undo, and after undo processing is done unlock the tree.

data: RedBlackTree.UserData;

scrTIDObj: TIDObj ← CheckoutTIDObj[tid.top];

refTid: REF Camelot.tidT ← NEW [Camelot.tidT ← tid];

data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scrTIDObj];

RecycleTIDObj[scrTIDObj];

IF data # NIL THEN {

dt: DocTrans;

max64Done: INT ← -1;

dt ← NARROW[data];

minBucket: CARD ← 65;

numberInMinBucket: CARD ← 0;

FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO -- look at all the documents to find the next 64 value to do

hashedDidTo64: CARD;

hashedDidTo64 ← Hash64DID[docList.first.did];

IF hashedDidTo64 = minBucket THEN numberInMinBucket ← numberInMinBucket + 1;

IF INT[hashedDidTo64] > max64Done AND hashedDidTo64 < minBucket THEN {

minBucket ← hashedDidTo64;

numberInMinBucket ← 1;

};

ENDLOOP;

IF minBucket = 65 THEN EXIT;

IF INT[minBucket] = max64Done THEN EXIT;

LockBucket64[minBucket, tid];

dt.bucketsToUnlatch ← CONS[minBucket, dt.bucketsToUnlatch];

FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO -- look at all the documents and process those with the correct hash value

hashedDidTo64: CARD;

hashedDidTo64 ← Hash64DID[docList.first.did];

IF hashedDidTo64 = minBucket THEN {

writeDocumentToMap[docList.first, refTid];

};

ENDLOOP;

max64Done ← minBucket;

ENDLOOP;

};

DropLatches: PROC [buckets: LIST OF CARD, tid: Camelot.tidT] ~ {

FOR nowBuck: LIST OF CARD ← buckets, nowBuck.rest UNTIL nowBuck = NIL DO

UnlockBucket64[nowBuck.first, tid];

ENDLOOP;

};

Commit: PUBLIC PROC [tid: Camelot.tidT] ~ {

data: RedBlackTree.UserData;

scrTIDObj: TIDObj ← CheckoutTIDObj[tid.top];

data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scrTIDObj];

IF data # NIL THEN {

dt: DocTrans;

dt ← NARROW[data];

FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO -- look at all the documents and turn off the modified flag

docList.first.modifiedDIDMap ← FALSE;

IF docList.first.destroyed THEN {

Delete[docList.first];

};

ENDLOOP;

DropLatches[dt.bucketsToUnlatch, tid];

};

[] ← RedBlackTree.Delete[ transactionToModifiedDocumentMap, scrTIDObj];

RecycleTIDObj[scrTIDObj];

};

Abort: PUBLIC PROC [tid: Camelot.tidT] ~ {

data: RedBlackTree.UserData;

scrTIDObj: TIDObj ← CheckoutTIDObj[tid.top];

data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scrTIDObj];

IF data # NIL THEN {

dt: DocTrans;

max64Done: INT ← -1;

dt ← NARROW[data];

FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO -- look at all the documents and remove them from the cache

Delete[ docList.first];

ENDLOOP;

DropLatches[dt.bucketsToUnlatch, tid];

};

[] ← RedBlackTree.Delete[ transactionToModifiedDocumentMap, scrTIDObj];

RecycleTIDObj[scrTIDObj];

};

Files

openFilesFromParsedDocList: PROC [doc: YggInternal.Document, did: YggDID.DID, tid: Camelot.tidT] = {

FOR pdl: LIST OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, pdl.rest UNTIL pdl = NIL DO

atomCode: ATOM ← NIL;

byteSize: INT ← 0;

theRuns: YggDIDMap.RunList ← NIL;

theFile: YggFile.FileHandle;

SELECT pdl.first.partType FROM

root => atomCode ← $root;

contents => atomCode ← $contents;

outlinks => atomCode ← $outlinks;

index => {

atomCode ← pdl.first.indexName;

};

directoryContents => atomCode ← $directoryContents;

attributes => atomCode ← $attributes;

ENDCASE => atomCode ← $huh;

FOR segments: LIST OF YggDIDMapPrivate.SegmentItem ← pdl.first.segments, segments.rest UNTIL segments = NIL DO

nowPage: CARD ← segments.first.firstPage;

byteSize ← MAX[byteSize, YggFile.BytesForPages[segments.first.firstPage] + segments.first.numberOfBytes];

FOR runs: LIST OF YggDIDMapPrivate.RunItem ← segments.first.runs, runs.rest UNTIL runs = NIL DO

theRuns ← CONS[[segmentId: runs.first.segmentId, segmentPage: runs.first.segmentPage, firstPage: nowPage, pages: runs.first.pages, leader: FALSE], theRuns];

nowPage ← nowPage + runs.first.pages;

ENDLOOP;

theFile ← YggFile.Open[runList: theRuns, byteSize: byteSize, did: did, fileUse: atomCode, verifyLeaderNow: FALSE];

doc.componentFiles ← CONS[theFile, doc.componentFiles];

ENDLOOP;

};

Utilities

AppendText: UNSAFE PROC [dPP: YggDIDMapPrivate.DocPartPtr, text: Rope.Text, offset: CARD] RETURNS [textRP: YggDIDMapPrivate.TextRP, words: CARD] = UNCHECKED {

textRep: LONG POINTER TO YggDIDMapPrivate.TextRep;

byteOffset: CARD16 ← offset*UNITS[CARD32];

textRP ← LOOPHOLE[byteOffset];

textRep ← @dPP[textRP];

LOOPHOLE[textRep, LONG POINTER TO CARD16]^ ← Rope.Length[text]; -- sets textRep.length

[] ← PBasics.ByteBlt [

to: [ BASE[DESCRIPTOR[textRep]], 0, textRep.length ],

from: [ BASE[DESCRIPTOR[text]], 0, textRep.length ]

];

words ← (WORDS[YggDIDMapPrivate.TextRep[textRep.length]] + WORDSInWords - 1) / WORDSInWords;

};

GetText: UNSAFE PROC [textRep: LONG POINTER TO YggDIDMapPrivate.TextRep, text: REF TEXT] = UNCHECKED {

nBytes: CARD;

text.length ← textRep.length;

nBytes ← PBasics.ByteBlt [

to: [ BASE[DESCRIPTOR[text]], 0, textRep.length ],

from: [ BASE[DESCRIPTOR[textRep]], 0, textRep.length ]

];

};

DID allocation

didsCondition: CONDITION;

GetNextDID: ENTRY PROC RETURNS [did: YggDID.DID] ~ {

ENABLE UNWIND => {};

loopCount: INT ← 0;

WHILE NextDID.didHigh = LastDIDAllocateable.didHigh AND NextDID.didLow = LastDIDAllocateable.didLow DO

WAIT didsCondition;

loopCount ← loopCount + 1;

IF loopCount > 5 THEN TRUSTED {loopCount ← 0; Process.Detach[FORK GrabSomeDIDs[]];};

ENDLOOP;

did ← NEW[YggDIDPrivate.DIDRep ← [NextDID.didHigh, NextDID.didLow]];

NextDID.didLow ← NextDID.didLow + 1;

IF LastDIDAllocateable.didLow - NextDID.didLow <= ThresholdToGrabDIDs THEN TRUSTED {Process.Detach[FORK GrabSomeDIDs[]];};

};

GrabberCount: CARD ← 0;

GrabSomeDIDs: PROC [] ~ {

newLastDIDAllocateable: REF DIDRep ← NEW[DIDRep];

newHigh: BOOL ← FALSE;

moreDIDsTID: Camelot.tidT;

kernCode: Mach.kernReturnT;

onlyOne: ENTRY PROC RETURNS [exit: BOOL ← FALSE] ~ {

ENABLE UNWIND => {};

IF GrabberCount # 0 THEN RETURN[TRUE];

GrabberCount ← GrabberCount + 1;

};

IF onlyOne[] THEN RETURN;

newLastDIDAllocateable.didHigh ← LastDIDAllocateable.didHigh;

newLastDIDAllocateable.didLow ← LastDIDAllocateable.didLow + NumberOfDIDsToGrap;

IF newLastDIDAllocateable.didLow > LastDidLow THEN {

newLastDIDAllocateable.didHigh ← newLastDIDAllocateable.didHigh + 1;

newLastDIDAllocateable.didLow ← NumberOfDIDsToGrap;

newHigh ← TRUE;

};

outcome: YggEnvironment.Outcome;

status: INT ← -1;

addrNewLastDIDAllocateable: Mach.pointerT;

[newTid: moreDIDsTID, kernCode: kernCode] ← Camelot.TABegin[taPort: YggEnvironment.taPort, parentTid: YggEnvironment.nullTransID, transType: ttNvServerBased, raiseSignal: TRUE];

moreDIDsTID ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];

kernCode ← Camelot.DSPinObject[dsPort: YggEnvironment.dsPort, tid: moreDIDsTID, optr: MaxDIDOptr, size: BYTES[DIDRep], raiseSignal: TRUE];

TRUSTED {addrNewLastDIDAllocateable ← LOOPHOLE[newLastDIDAllocateable, CARD32]};

kernCode ← Camelot.DSLogNewValue[dsPort: YggEnvironment.dsPort, tid: moreDIDsTID, optr: MaxDIDOptr, newValue: addrNewLastDIDAllocateable, newValueCnt: BYTES[DIDRep], raiseSignal: TRUE];

[status: status, kernCode: kernCode] ← Camelot.TAEnd[taPort: YggEnvironment.taPort, tid: moreDIDsTID, protocolType: ptTwoPhased, raiseSignal: TRUE];

IF kernCode = Mach.KernSuccess AND status = Camelot.ErSuccess THEN EXIT;

[outcome: outcome] ← YggTransaction.Finish[transID: moreDIDsTID, requestedOutcome: commit];

IF outcome = commit THEN EXIT;

ENDLOOP;

IF newHigh THEN {

m: ENTRY PROC ~ {

ENABLE UNWIND => {};

LastDIDAllocateable ← newLastDIDAllocateable^;

NextDID.didHigh ← newLastDIDAllocateable.didHigh;

NextDID.didLow ← 0;

GrabberCount ← 0;

BROADCAST didsCondition;

};

m[];

}

ELSE {

n: ENTRY PROC ~ {

LastDIDAllocateable.didLow ← newLastDIDAllocateable.didLow;

BROADCAST didsCondition;

GrabberCount ← 0;

};

n[];

};

DID access to recoverable DID Map

LookupInMap: PROC [did: YggDID.DID, tid: Camelot.tidT, doc: YggInternal.Document] RETURNS [found: BOOL ← FALSE, gottaBeInCache: BOOL ← FALSE] = {

notFound: BOOL ← FALSE;

extendedData: REF EntryDataBuffer ← NIL;

extendedDataCount: CARD ← 0;

lowExtendedToAccept: CARD ← 0;

highExtendedToAccept: CARD ← 1;

expectMore: BOOL ← TRUE;

expectedExtendedDataSize: CARD ← 0;

maxDataCount: CARD ← 0;

fromLink: DID ← YggEnvironment.nullDID;

toLink: DID ← YggEnvironment.nullDID;

linkType: Rope.ROPE ← NIL;

updatesForTrans: LIST OF updates;

lookupProc: UNSAFE PROC [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT, overFlowPageProc: UNSAFE PROC RETURNS [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT]] ~ TRUSTED {

nowEntry: ExtendibleHash.Entry ← entry;

wordsLeft: INT ← wordsForEntriesOnPage;

WHILE wordsLeft > 0 DO

entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[nowEntry, LONG POINTER] + UNITS[CARD32];

IF nowEntry.entryWordSize > CARD[HashPageSizeInWords] THEN ERROR;

IF YggDID.CompareDIDs[did, @entryPtr.did] = equal AND entryPtr.extensionCount >= lowExtendedToAccept AND entryPtr.extensionCount <= highExtendedToAccept THEN {

found ← TRUE;

SELECT entryPtr.extensionCount FROM

0 => { -- normal, non-extended entry

index: INT ← 0;

expectMore ← FALSE; -- everything is in this entry

IF entryPtr.link THEN {

wordIndex: INT ← 0;

noChars: INT ← 0;

nullSeen: BOOL ← FALSE;

fromLink ← NEW[DIDRep];

PBasics.Move[dst: @fromLink, src: @entryPtr.data[0], nWords: WORDS[DIDRep]/WORDSInWords];

index ← WORDS[DIDRep]/WORDSInWords;

toLink ← NEW[DIDRep];

PBasics.Move[dst: @toLink, src: @entryPtr.data[index], nWords: WORDS[DIDRep]/WORDSInWords];

index ← index + index;

FOR wordIndex ← index, wordIndex + 1 UNTIL nullSeen DO

fourChars: PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR;

LOOPHOLE[fourChars, CARD] ← entryPtr.data[index];

FOR charNo: INT IN [0..Basics.charsPerWord) DO

IF fourChars[charNo] = 0C THEN {nullSeen← TRUE; EXIT;};

noChars ← noChars + 1;

ENDLOOP;

IF noChars = 0 THEN {

index ← index + 1;

}

ELSE {

linkTypeRT: REF TEXT ← NIL;

linkTypeRT ← RefText.New[noChars];

nullSeen ← FALSE;

UNTIL nullSeen DO

fourChars: PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR;

LOOPHOLE[fourChars, CARD] ← entryPtr.data[index];

index ← index + 1;

FOR charNo: INT IN [0..Basics.charsPerWord) DO

IF fourChars[charNo] = 0C THEN {nullSeen← TRUE; EXIT;};

linkTypeRT[noChars] ← fourChars[charNo];

noChars ← noChars + 1;

ENDLOOP;

linkTypeRT.length ← noChars;

linkType ← RefText.TrustTextAsRope[linkTypeRT];

};

[] ← parseDocPart[@entryPtr.data[index], entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry - index, doc];

};

1 => { -- normal, extended entries to follow

expectMore ← TRUE; -- everything is in this entry

expectedExtendedDataSize ← entryPtr.data[0];

extendedData ← NEW[EntryDataBuffer[expectedExtendedDataSize]];

extendedDataCount ← entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry - 1;

PBasics.Move[dst: @extendedData[0], src: @entryPtr.data[1], nWords: extendedDataCount];

};

>1 => { -- extended entry

IF extendedDataCount + entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry > maxDataCount THEN ERROR;

PBasics.Move[dst: @extendedData[extendedDataCount], src: @entryPtr.data[0], nWords: entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry];

extendedDataCount ← extendedDataCount + entryPtr.size - YggDIDMapPrivate.Word32OverheadPerEntry;

};

ENDCASE => ERROR;

};

wordsLeft ← wordsLeft - nowEntry.entryWordSize - 1 ;

nowEntry ← nowEntry + ((nowEntry.entryWordSize + 1) * UNITS[CARD32]);

REPEAT FINISHED => notFound ← TRUE;

ENDLOOP;

};

updatesForTrans ← getUpdates[tid];

FOR ut: LIST OF updates ← updatesForTrans, ut.rest UNTIL ut = NIL DO

IF YggDID.EqualDIDs[ut.first.did, did] THEN RETURN[FALSE, TRUE]; -- if it has updates, then is should be in the cache - so why are we in this routine?

ENDLOOP;

WHILE expectMore DO

TRUSTED {ExtendibleHash.ReadEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], proc: lookupProc]};

IF notFound THEN EXIT;

ENDLOOP;

IF maxDataCount # 0 THEN TRUSTED {

IF maxDataCount # extendedDataCount THEN ERROR;

[] ← parseDocPart[@extendedData[0], maxDataCount, doc];

};

InsertIntoMap: PROC [did: DID, doc: YggInternal.Document, buffer: REF EntryDataBuffer, wordsInBuffer: INT, link: BOOL, refTid: REF Camelot.tidT] = {

extensionCount: INT ← 0;

foundEntry: BOOL ← TRUE;

lookupProc: UNSAFE PROC [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT, overFlowPageProc: UNSAFE PROC RETURNS [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT]] RETURNS[entryToUpdate: ExtendibleHash.Entry ← NIL] ~ TRUSTED {

Set extensionCount before calling

nowEntry: ExtendibleHash.Entry ← entry;

wordsLeft: INT ← wordsForEntriesOnPage;

WHILE wordsLeft > 0 DO

entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[nowEntry, LONG POINTER] + UNITS[CARD32];

IF nowEntry.entryWordSize > CARD[HashPageSizeInWords] THEN ERROR;

IF YggDID.CompareDIDs[did, @entryPtr.did] = equal AND entryPtr.extensionCount = extensionCount THEN {

entryToUpdate ← nowEntry;

RETURN;

};

wordsLeft ← wordsLeft - nowEntry.entryWordSize - 1 ;

nowEntry ← nowEntry + ((nowEntry.entryWordSize + 1) * UNITS[CARD32]);

ENDLOOP;

};

lookupForDelete: UNSAFE PROC [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT, overFlowPageProc: UNSAFE PROC RETURNS [entry: ExtendibleHash.Entry, wordsForEntriesOnPage: INT]] RETURNS[entryToDelete: ExtendibleHash.Entry ← NIL] ~ TRUSTED {

Set extensionCount before calling

nowEntry: ExtendibleHash.Entry ← entry;

wordsLeft: INT ← wordsForEntriesOnPage;

foundEntry ← FALSE;

WHILE wordsLeft > 0 DO

entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[nowEntry, LONG POINTER] + UNITS[CARD32];

IF nowEntry.entryWordSize > CARD[HashPageSizeInWords] THEN ERROR;

IF YggDID.CompareDIDs[did, @entryPtr.did] = equal AND entryPtr.extensionCount = extensionCount THEN {

entryToDelete ← nowEntry;

foundEntry ← TRUE;

RETURN;

};

wordsLeft ← wordsLeft - nowEntry.entryWordSize - 1 ;

nowEntry ← nowEntry + ((nowEntry.entryWordSize + 1) * UNITS[CARD32]);

ENDLOOP;

};

totalEntrySize: INT;

totalEntrySize ← YggDIDMapPrivate.Word32OverheadPerEntry + wordsInBuffer;

IF totalEntrySize <= MaxWordsPerEntry THEN {

modifyProc: UNSAFE PROC [entry: ExtendibleHash.Entry] ~ TRUSTED {

entPtr: LONG POINTER TO YggDIDMapPrivate.Entry = LOOPHOLE[entry, LONG POINTER] + UNITS[CARD32];

entPtr.size ← totalEntrySize;

entPtr.extensionCount ← 0;

entPtr.did ← did^;

entPtr.link ← link;

PBasics.Move[dst: @entPtr.data[0], src: LOOPHOLE[buffer], nWords: wordsInBuffer];

};

extensionCount ← 0;

TRUSTED {ExtendibleHash.UpdateEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], words: totalEntrySize, findProc: lookupProc, modifyProc: modifyProc, updateType: insertOrReplace, clientPerCallData: refTid];};

extensionCount ← 1;

}

ELSE {

modifyProcFirst: UNSAFE PROC [entry: ExtendibleHash.Entry] ~ TRUSTED {

writeWords: INT ← MaxWordsPerEntry - YggDIDMapPrivate.Word32OverheadPerEntry - 1;

entPtr: LONG POINTER TO YggDIDMapPrivate.Entry = LOOPHOLE[entry, LONG POINTER] + UNITS[CARD32];

entPtr.size ← MaxWordsPerEntry;

entPtr.extensionCount ← 1;

entPtr.did ← did^;

entPtr.data[0] ← wordsInBuffer;

leftToWrite ← wordsInBuffer - writeWords;

PBasics.Move[dst: @entPtr.data[1], src: LOOPHOLE[buffer], nWords: writeWords];

writeOffset ← writeWords;

};

leftToWrite: INT ← 0;

writeOffset: INT ← 0;

extensionCount ← 1;

TRUSTED {ExtendibleHash.UpdateEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], words: MaxWordsPerEntry, findProc: lookupProc, modifyProc: modifyProcFirst, updateType: insertOrReplace, clientPerCallData: refTid]; };

extensionCount ← 0;

TRUSTED {ExtendibleHash.DeleteEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], proc: lookupForDelete, clientPerCallData: refTid];};

modifyProcExtended: UNSAFE PROCEDURE [entry: ExtendibleHash.Entry] = UNCHECKED{

entPtr: LONG POINTER TO YggDIDMapPrivate.Entry = LOOPHOLE[entry, LONG POINTER] + UNITS[CARD32];

entPtr.size ← writeThisTime+YggDIDMapPrivate.Word32OverheadPerEntry;

entPtr.extensionCount ← extensionCount;

entPtr.did ← did^;

entPtr.data[0] ← writeThisTime;

PBasics.Move[dst: @entPtr.data[1], src: LOOPHOLE[buffer, LONG POINTER]+writeOffset, nWords: writeThisTime];

};

writeThisTime: INT ← MIN[MaxWordsPerEntry - YggDIDMapPrivate.Word32OverheadPerEntry, leftToWrite];

leftToWrite ← leftToWrite - writeThisTime;

extensionCount ← extensionCount + 1;

TRUSTED {ExtendibleHash.UpdateEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], words: writeThisTime + YggDIDMapPrivate.Word32OverheadPerEntry, findProc: lookupProc, modifyProc: modifyProcExtended, updateType: insertOrReplace, clientPerCallData: refTid]; };

IF leftToWrite < 0 THEN ERROR;

IF leftToWrite = 0 THEN EXIT;

ENDLOOP;

};

foundEntry ← TRUE;

WHILE foundEntry DO -- insure that old extra extentions do not exist

TRUSTED {ExtendibleHash.DeleteEntry[hashTable: TheDIDMap, hashValue: [did.didHigh, did.didLow], proc: lookupForDelete, clientPerCallData: refTid];};

extensionCount ← extensionCount + 1;

ENDLOOP;

};

lock64Condition: CONDITION;

LockBucket64: ENTRY PROC [bucket: INT, tid: Camelot.tidT] ~ {

ENABLE UNWIND => {};

WHILE Lock64Table[bucket].interlock DO WAIT lock64Condition ENDLOOP;

Lock64Table[bucket].interlock ← TRUE;

Lock64Table[bucket].tid ← tid;

};

UnlockBucket64: ENTRY PROC [bucket: INT, tid: Camelot.tidT] ~ {

ENABLE UNWIND => {};

IF ~Lock64Table[bucket].interlock OR Lock64Table[bucket].tid # tid THEN ERROR;

Lock64Table[bucket].interlock ← FALSE;

Lock64Table[bucket].tid ← YggEnvironment.nullTransID;

BROADCAST lock64Condition;

};

getUpdates: PROC [tid: Camelot.tidT] RETURNS [updateList: LIST OF updates] ~ {

ERROR;

};

parseDocPart: PROC [dataToParse: LONG POINTER, size: CARD, doc: YggInternal.Document] RETURNS [found: BOOL ← FALSE] = {

This code must be the dual of writeDocToBuffer

parsedDocList: LIST OF YggDIDMapPrivate.DocItem ← NIL;

parsedDocListTail: LIST OF YggDIDMapPrivate.DocItem ← NIL;

index: CARD ← 0;

WHILE index < size*WORDSInWords DO

TRUSTED {

segIndex: LONG POINTER TO YggDIDMapPrivate.SegmentRep;

docPart: LONG POINTER TO YggDIDMapPrivate.DocPart;

newDocList: LIST OF YggDIDMapPrivate.DocItem ← NIL;

segmentsListTail: LIST OF YggDIDMapPrivate.SegmentItem ← NIL;

startOffset: CARD ← 0;

offsetToLastSegment: CARD ← 0;

indexName: ATOM ← NIL;

docPart ← LOOPHOLE[dataToParse + index];

IF index + docPart.docPartSize*WORDSInWords > size*WORDSInWords THEN ERROR;

IF LOOPHOLE[docPart.indexName, CARD16] # 0 THEN {

r: Rope.Text;

docPartPtr: YggDIDMapPrivate.DocPartPtr = LOOPHOLE[docPart];

indexNamePtr: LONG POINTER TO YggDIDMapPrivate.TextRep;

indexNamePtr ← @docPartPtr[docPartPtr.indexName];

IF indexNamePtr.length = 0 THEN ERROR;

offsetToLastSegment ← LOOPHOLE[indexNamePtr, CARD] - LOOPHOLE[docPart, CARD];

r ← Rope.NewText[indexNamePtr.length];

GetText[indexNamePtr, LOOPHOLE[r]];

indexName ← Atom.MakeAtom[r];

}

ELSE {

offsetToLastSegment ← docPart.docPartSize*WORDSInWords;

SELECT docPart.partType FROM

root => indexName ← $root;

contents => indexName ← $contents;

outlinks => indexName ← $outlinks;

attributes => indexName ← $attributes;

ENDCASE => ERROR;

};

newDocList ← LIST[[partType: docPart.partType, indexName: indexName, slotNo: docPart.slotNo, segments: NIL]];

segIndex ← LOOPHOLE[@docPart.segments];

WHILE (startOffset ← LOOPHOLE[segIndex, CARD] - LOOPHOLE[docPart, CARD]) < offsetToLastSegment DO

segSize: CARD ← 0;

runs: LIST OF YggDIDMapPrivate.RunItem ← NIL;

runTail: LIST OF YggDIDMapPrivate.RunItem ← NIL;

newSegListItem: LIST OF YggDIDMapPrivate.SegmentItem;

newSegListItem ← LIST[[firstPage: segIndex.firstPage, numberOfBytes: segIndex.numberOfBytes, runs: NIL]];

IF (segSize ← (WORDS[YggDIDMapPrivate.SegmentRep[segIndex.length]]/WORDSInWords)) > (docPart.docPartSize - (startOffset/WORDSInWords)) THEN ERROR; -- too many runs

FOR inx: CARDINAL IN [0..segIndex.length) DO

newRunList: LIST OF YggDIDMapPrivate.RunItem ← NIL;

newRunList ← LIST[[segmentId: segIndex.runs[inx].segmentId, segmentPage: segIndex.runs[inx].segmentPage, pages: segIndex.runs[inx].pages]];

IF runTail = NIL THEN runs ← runTail ← newRunList

ELSE {

runTail.rest ← newRunList;

runTail ← newRunList;

};

ENDLOOP;

newSegListItem.first.runs ← runs;

IF newDocList.first.segments = NIL THEN newDocList.first.segments ← segmentsListTail ← newSegListItem

ELSE {

segmentsListTail.rest ← newSegListItem;

segmentsListTail ← newSegListItem;

};

segIndex ← segIndex + segSize*WORDSInWords;

ENDLOOP;

IF parsedDocListTail = NIL THEN parsedDocList ← parsedDocListTail ← newDocList

ELSE {

parsedDocListTail.rest ← newDocList;

parsedDocListTail ← newDocList;

};

index ← index + docPart.docPartSize*WORDSInWords;

};

ENDLOOP;

doc.parsedDocList ← parsedDocList;

};

stockBuffer: REF EntryDataBuffer ← NIL;

CheckoutStockBuffer: ENTRY PROC RETURNS [buf: REF EntryDataBuffer] = {

ENABLE UNWIND => {};

IF stockBuffer = NIL THEN buf ← NEW[EntryDataBuffer[256]] ELSE { buf ← stockBuffer; stockBuffer ← NIL };

};

RecycleStockBuffer: ENTRY PROC [buf: REF EntryDataBuffer] = {

stockBuffer ← buf;

};

writeDocumentToMap: PROC [ doc: Document, refTid: REF Camelot.tidT] ~ {

buffer: REF EntryDataBuffer ← NIL;

stockBuffer: REF EntryDataBuffer ← NIL;

bufferSize: CARD ← 256;

link: BOOL;

wordsInBuffer: INT;

IF doc.destroyed THEN {

Set extensionCount before calling

nowEntry: ExtendibleHash.Entry ← entry;

wordsLeft: INT ← wordsForEntriesOnPage;

foundEntry ← FALSE;

WHILE wordsLeft > 0 DO

entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[nowEntry, LONG POINTER] + UNITS[CARD32];

IF nowEntry.entryWordSize > CARD[HashPageSizeInWords] THEN ERROR;

IF YggDID.CompareDIDs[doc.did, @entryPtr.did] = equal AND entryPtr.extensionCount = extensionCount THEN {

entryToDelete ← nowEntry;

foundEntry ← TRUE;

noteUpdatedBuffer[entry, tid];

RETURN;

};

wordsLeft ← wordsLeft - nowEntry.entryWordSize - 1 ;

nowEntry ← nowEntry + ((nowEntry.entryWordSize + 1) * UNITS[CARD32]);

ENDLOOP;

};

extensionCount: INT ← 0;

foundEntry: BOOL ← TRUE;

WHILE foundEntry DO -- insure that old extra extentions do not exist

foundEntry ← FALSE;

TRUSTED {ExtendibleHash.DeleteEntry[hashTable: TheDIDMap, hashValue: [doc.did.didHigh, doc.did.didLow], proc: lookupForDelete, clientPerCallData: refTid];};

extensionCount ← extensionCount + 1;

ENDLOOP;

}

ELSE {

stockBuffer ← buffer ← CheckoutStockBuffer[];

didNotFit: BOOL ← TRUE;

[wordsInBuffer: wordsInBuffer, didNotFit: didNotFit, link: link] ← writeDocToBuffer[doc, buffer, bufferSize];

IF didNotFit THEN {

bufferSize ← bufferSize + bufferSize;

buffer ← NEW[EntryDataBuffer[bufferSize]];

LOOP;

};

EXIT;

ENDLOOP;

InsertIntoMap[did: doc.did, doc: doc, buffer: buffer, wordsInBuffer: wordsInBuffer, link: link, refTid: refTid];

RecycleStockBuffer[stockBuffer];

};

writeDocToBuffer: PROC [ doc: Document, buffer: REF EntryDataBuffer, bufferSize: INT] RETURNS [wordsInBuffer: CARD ← 0, didNotFit: BOOL ← TRUE, link: BOOL ← FALSE] ~ {

This code must be the dual of parseDocPart

IF doc.fromDID # YggEnvironment.nullDID THEN {

linkTypeLen: INT ← 0;

linkTypeWords: CARD ← 0;

nowCharNum: INT ← 0;

link ← TRUE;

TRUSTED {PBasics.Move[dst: LOOPHOLE[@buffer[0]], src: LOOPHOLE[@(doc.fromDID^)], nWords: WORDS[DIDRep]/WORDSInWords]; };

wordsInBuffer ← WORDS[DIDRep]/WORDSInWords;

TRUSTED {PBasics.Move[dst: LOOPHOLE[@buffer[wordsInBuffer]], src: LOOPHOLE[@(doc.toDID^)], nWords: WORDS[DIDRep]/WORDSInWords]; };

wordsInBuffer ← wordsInBuffer + WORDS[DIDRep]/WORDSInWords;

linkTypeLen ← Rope.Length[doc.linkType];

linkTypeWords ← (linkTypeLen + PBasics.charsPerWord)/PBasics.charsPerWord;

IF linkTypeWords + wordsInBuffer > CARD[bufferSize] THEN RETURN[0, TRUE];

FOR i: INT IN [0..INT[linkTypeWords]) DO

fourChars: PACKED ARRAY [0..PBasics.charsPerWord) OF CHAR;

FOR charNo: INT IN [0..Basics.charsPerWord) DO

IF nowCharNum >= linkTypeLen THEN fourChars[charNo] ← 0C

ELSE fourChars[charNo] ← Rope.Fetch[doc.linkType, nowCharNum];

nowCharNum ← nowCharNum + 1;

ENDLOOP;

TRUSTED{buffer[wordsInBuffer].card ← LOOPHOLE[fourChars, CARD];};

wordsInBuffer ← wordsInBuffer + 1;

ENDLOOP;

};

FOR parsedDocList: LIST OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, parsedDocList.rest UNTIL parsedDocList = NIL DO

indexNameWords: CARD ← 0;

segmentOffset: CARD ← 0;

docPart: LONG POINTER TO YggDIDMapPrivate.DocPart;

IF INT[wordsInBuffer] + WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[1]] > INT[bufferSize] THEN RETURN [0, TRUE];

TRUSTED {

docPart ← LOOPHOLE[@buffer[wordsInBuffer]];

docPart.compression ← YggDIDMapPrivate.notCompressed;

docPart.keepingModifications ← FALSE;

docPart.partType ← parsedDocList.first.partType;

docPart.extraFlags ← 0;

docPart.slotNo ← parsedDocList.first.slotNo;

docPart.indexName ← LOOPHOLE[0];

docPart.versionIdentifierString ← LOOPHOLE[0];

};

FOR segments: LIST OF YggDIDMapPrivate.SegmentItem ← parsedDocList.first.segments, segments.rest UNTIL segments = NIL DO

runIndex: CARD ← 0;

segIndex: LONG POINTER TO YggDIDMapPrivate.SegmentRep;

IF wordsInBuffer + segmentOffset + CARD[WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[1]]] > CARD[bufferSize] THEN RETURN [0, TRUE];

TRUSTED {

segIndex ← LOOPHOLE[@docPart.segments + segmentOffset];

segIndex.firstPage ← segments.first.firstPage;

segIndex.numberOfBytes ← segments.first.numberOfBytes;

segIndex.modificationDeltasOffset ← 0;

segIndex.lastUsedTime ← BasicTime.Now[];

segIndex.backUpTime ← BasicTime.nullGMT;

segIndex.backUpID ← 0;

};

FOR rl: LIST OF YggDIDMapPrivate.RunItem ← segments.first.runs, rl.rest UNTIL rl = NIL DO

IF wordsInBuffer + segmentOffset + CARD[WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[runIndex]]] > CARD[bufferSize] THEN RETURN [0, TRUE];

TRUSTED {segIndex.runs[runIndex] ← [segmentId: rl.first.segmentId, segmentPage: rl.first.segmentPage, pages: rl.first.pages]; };

runIndex ← runIndex + 1;

ENDLOOP;

TRUSTED {segIndex.length ← runIndex; };

segmentOffset ← segmentOffset + WORDS[YggDIDMapPrivate.SegmentRep[runIndex]]/WORDSInWords;

ENDLOOP;

TRUSTED {

docPart.docPartSize ← WORDS[YggDIDMapPrivate.DocPart]/WORDSInWords - WORDS[CARD]/WORDSInWords + segmentOffset + indexNameWords;

wordsInBuffer ← wordsInBuffer + docPart.docPartSize;

};

IF parsedDocList.first.indexName # NIL THEN {

SELECT parsedDocList.first.partType FROM

root, contents, outlinks, attributes => {};

ENDCASE => {

pName: Rope.Text;

pName ← Atom.GetPName[parsedDocList.first.indexName];

IF wordsInBuffer + WORDS[YggDIDMapPrivate.TextRep[Rope.Length[pName]]] > CARD[bufferSize] THEN RETURN [0, TRUE];

TRUSTED {

[docPart.indexName, indexNameWords] ← AppendText[dPP: docPart, text: pName, offset: docPart.docPartSize];

docPart.docPartSize ← docPart.docPartSize + indexNameWords;

};

wordsInBuffer ← wordsInBuffer + indexNameWords;

};

ENDLOOP;

RETURN[wordsInBuffer: wordsInBuffer, didNotFit: FALSE];

};

ExtendibleHash storage primitives

StartUpExtendibleHash: PROC [btreeInfo: REF, initialize: BOOL ← FALSE] = {

TheDIDMap ← ExtendibleHash.New [

storPrim: [referencePage: HashReferencePage, releasePage: HashReleasePage],

initialState: suspended

];

IF initialize THEN {

outcome: YggEnvironment.Outcome ← active;

doingInit ← TRUE;

initTrans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];

ExtendibleHash.Open[

hashTable: TheDIDMap,

storage: btreeInfo,

pageSize: HashPageSizeInWords,

initialize: initialize,

initialDirectorySize: 64

];

doingInit ← FALSE;

[outcome: outcome] ← YggTransaction.Finish[transID: initTrans, requestedOutcome: commit];

IF outcome # commit THEN ERROR;

}

ELSE {

ExtendibleHash.Open[

hashTable: TheDIDMap,

storage: btreeInfo,

pageSize: HashPageSizeInWords,

initialize: initialize,

initialDirectorySize: 64

];

};

debugReferencePageNumber: CARD ← 0;

debugReferencePageNumberCount: INT ← 0;

doingInit: BOOL ← FALSE;

initTrans: YggEnvironment.TransID;

BufferAndTransList: LIST OF BufferAndTransItem ← NIL;

BufferAndTransItem: TYPE = RECORD [

highBits: CARD,

trans: YggEnvironment.TransID

];

findTransForPage: ENTRY PROC [number: PageNumber] RETURNS [trans: YggEnvironment.TransID] ~ {

};

noteUpdatedBuffer: ENTRY PROC [entry: ExtendibleHash.Entry, trans: YggEnvironment.TransID] ~ {

highBits: CARD;

highBits ← LOOPHOLE[entry, CARD]/ExtendibleHash.wordsPerPage;

BufferAndTransList ← CONS[[highBits, trans], BufferAndTransList];

};

doneWithTransUpdatedBuffer: ENTRY PROC [trans: YggEnvironment.TransID] ~ {

highBits: CARD;

highBits ← LOOPHOLE[entry, CARD]/ExtendibleHash.wordsPerPage;

BufferAndTransList ← CONS[[highBits, trans], BufferAndTransList];

};

HashReferencePage: PUBLIC ExtendibleHash.ReferencePage = {

PROC [storage: PageStorage, number: PageNumber, type: ReferenceType, clientPerCallData: REF] RETURNS [ptr: PagePtr]

addVMPageSet: ENTRY PROC = {

CurrentVMPageSetsForDIDMap ← CONS[vMPageSet, CurrentVMPageSetsForDIDMap];

};

debugReferencePageNumberCount: INT ← 0;

vMPageSet: YggBuffMan.VMPageSet;

IF number = debugReferencePageNumber THEN debugReferencePageNumberCount ← debugReferencePageNumberCount + 1;

IF type = new THEN {

IF YggFile.Info[FileForDIDMap, YggEnvironment.nullTransID].size < (INT[number]+1)*FilePagesPerBtreePage THEN YggFile.SetSize[FileForDIDMap, (number+25)*FilePagesPerBtreePage, YggEnvironment.nullTransID];

vMPageSet ← YggBuffMan.ReadPages[fileHandle: FileForDIDMap, tid: YggEnvironment.nullTransID, pageRun: [firstPage: number+FilePagesPerBtreePage, count: FilePagesPerBtreePage]];

}

ELSE {

vMPageSet ← YggBuffMan.ReadPages[fileHandle: FileForDIDMap, tid: YggEnvironment.nullTransID, pageRun: [firstPage: number*FilePagesPerBtreePage+FilePagesPerBtreePage, count: FilePagesPerBtreePage]];

};

IF vMPageSet = NIL OR vMPageSet.rest # NIL THEN ERROR;

addVMPageSet[];

RETURN[vMPageSet.first.buffer];

};

HashReleasePage: PUBLIC ExtendibleHash.ReleasePage = {

PROC [storage: PageStorage, number: PageNumber, update: UpdateState, clientPerCallData: REF]

findVMPageSet: ENTRY PROC [number: ExtendibleHash.PageNumber] RETURNS [vmPageSet: LIST OF YggBuffMan.VMPageSet] = {

ENABLE UNWIND => {};

prevlovmps: LIST OF YggBuffMan.VMPageSet ← NIL;

FOR lovmps: LIST OF YggBuffMan.VMPageSet ← CurrentVMPageSetsForDIDMap, lovmps.rest UNTIL lovmps = NIL DO

IF lovmps.first.first.pageRun.segmentPage = number*CARD[FilePagesPerBtreePage]+CARD[FilePagesPerBtreePage] THEN {

RETURN[lovmps];

};

prevlovmps ← lovmps;

ENDLOOP;

ERROR;

};

removeVMPageSet: ENTRY PROC = {

ENABLE UNWIND => {};

prevlovmps: LIST OF YggBuffMan.VMPageSet ← NIL;

FOR lovmps: LIST OF YggBuffMan.VMPageSet ← CurrentVMPageSetsForDIDMap, lovmps.rest UNTIL lovmps = NIL DO

IF lovmps.first.first.pageRun.firstPage = INT[number]*FilePagesPerBtreePage+FilePagesPerBtreePage THEN {

IF prevlovmps = NIL THEN CurrentVMPageSetsForDIDMap ← lovmps.rest

ELSE prevlovmps.rest ← lovmps.rest;

EXIT;

};

prevlovmps ← lovmps;

ENDLOOP;

};

debugReferencePageNumberCount: INT ← 0;

releaseState: YggBuffMan.ReleaseState;

vMPageSet: YggBuffMan.VMPageSet ← NIL;

refTid: REF Camelot.tidT ← NARROW[clientPerCallData];

IF number = debugReferencePageNumber THEN debugReferencePageNumberCount ← debugReferencePageNumberCount + 1;

IF refTid = NIL THEN ERROR;

SELECT update FROM

modified => {

releaseState ← writeBatchedNoWait;

};

unchanged => {

releaseState ← clean;

};

ENDCASE ;

vMPageSet ← findVMPageSet[number].first;

IF doingInit THEN {

YggBuffMan.WritePages[fileHandle: FileForDIDMap, tid: initTrans, to: number+FilePagesPerBtreePage, nPages: FilePagesPerBtreePage, from: vMPageSet.first.buffer];

}

ELSE IF update = modified THEN {

YggBuffMan.WritePages[fileHandle: FileForDIDMap, tid: refTid^, to: number+FilePagesPerBtreePage, nPages: FilePagesPerBtreePage, from: vMPageSet.first.buffer];

};

IF vMPageSet # NIL THEN {

YggBuffMan.ReleaseVMPageSet[vMPageSet: vMPageSet, releaseState: releaseState,

keep: TRUE];

removeVMPageSet[];

};

Internal red black procs

noteUseInTransaction: ENTRY PROC [doc: Document, tid: Camelot.tidT] ~ {

ENABLE UNWIND => {};

data: RedBlackTree.UserData;

scratchTIDObj.btid ← tid.top;

data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scratchTIDObj];

IF data = NIL THEN {

dt: DocTrans ← NEW[DocTransRep];

dt.btidForDoc ← NEW[TIDObjRep ← [tid.top]];

dt.docs ← LIST[doc];

RedBlackTree.Insert[transactionToModifiedDocumentMap, dt, scratchTIDObj];

}

ELSE {

dt: DocTrans;

dt ← NARROW[data];

FOR docList: LIST OF Document ← dt.docs, docList.rest UNTIL docList = NIL DO

IF YggDID.EqualDIDs[docList.first.did, doc.did] THEN RETURN;

ENDLOOP;

dt.docs ← CONS[doc, dt.docs];

};

GetKeyProc: RedBlackTree.GetKey = {

PROC [data: UserData] RETURNS [Key]

docTrans: DocTrans ← NARROW[data];

RETURN[ docTrans.btidForDoc ];

};

CompareProc: RedBlackTree.Compare = {

PROC [k: Key, data: UserData] RETURNS [Basics.Comparison]

fileTransData: DocTrans ← NARROW[data];

keyData: TIDObj ← NARROW[k];

SELECT keyData.highTicker FROM

> fileTransData.btidForDoc.highTicker => RETURN [greater];

< fileTransData.btidForDoc.highTicker => RETURN [less];

ENDCASE => {

SELECT keyData.lowTicker FROM

> fileTransData.btidForDoc.lowTicker => RETURN [greater];

< fileTransData.btidForDoc.lowTicker => RETURN [less];

ENDCASE => RETURN [equal];

};

stockTIDObj: TIDObj ← NIL;

CheckoutTIDObj: ENTRY PROC [btid: Camelot.btidT] RETURNS [k: TIDObj] = {

ENABLE UNWIND => {};

IF stockTIDObj = NIL THEN k ← NEW [TIDObjRep] ELSE { k ← stockTIDObj; stockTIDObj ← NIL };

k.btid ← btid;

};

RecycleTIDObj: ENTRY PROC [k: TIDObj] = {

stockTIDObj ← k;

};

Old enumeration proc

doc = NIL starts a new enumeration, and GetNext returns nextDocument = NIL when the enumeration is exhausted. The only guaranteed property of the enumeration is that all documents in existence during the entire enumeration will be visited at least once. Some documents may be seen more than once.

GetNext: PUBLIC --ENTRY-- PROCEDURE[doc: Document] RETURNS [nextDocument: Document] =

BEGIN -- errors defined in FileMap: none.

ENABLE UNWIND => NULL;

RETURN[EnumerateNext[doc]];

END;

Hash table management:

Explanation of client-supplied parameters:

The procedure ClientHashInit is called during hash table initialization, to allow the hash function to precompute values based on the range and to make any small adjustments to the range that are necessary.

HashDocument must:

be a REF type.

contain a field "next" of type HashDocument, under the exclusive control of the hash package.

Key is an arbitrary type.

SetKey sets the "key value" associated with a HashDocument. The key value must not change between the time the doc is Inserted into the table and the time it is deleted from the table.

Hash must be a function of the key value of the parameter "hashDocument".

EqualKeys must be the equality relation on the key values of the parameters "hashDocument1" and "hashDocument2".

Interface description:

InitializeHashTable: INTERNAL PROCEDURE[numHashSlotsDesired: NAT, hashTableZone:

ZONE, hashDocument: HashDocument];

errors: HashPkgCallerProgrammingError (numHashSlotsDesired = 0).

Insert: ENTRY PROCEDURE[hashDocument: HashDocument];

errors: HashPkgDuplicateKey.

LookupInCache: ENTRY PROCEDURE[key: Key] RETURNS [hashDocument: HashDocument];

returns hashDocument = NIL if not found.

Delete: ENTRY PROCEDURE[hashDocument: HashDocument];

errors: HashPkgCallerProgrammingError (not found).

EnumerateNext: INTERNAL PROCEDURE[prevHashDocument: HashDocument] RETURNS

[hashDocument: HashDocument];

errors: none.

prevHashDocument = NIL starts the enumeration, returned hashDocument = NIL is the end

of the enumeration. This procedure guarantees that any hashDocument in existence throughout

the entire enumeration will be seen. Other docs may or not not be seen. HashDocuments

may be seen more than once.

EnumerateWithProc: INTERNAL PROCEDURE[proc: PROCEDURE[hashDocument: HashDocument]

RETURNS[stop: BOOLEAN]];

errors: none.

client-supplied parameters to hash package begin here:

HashDocument: TYPE = Document;

Key: TYPE = YggDID.DID;

ClientHashInit: INTERNAL PROCEDURE[numHashSlotsDesired: NAT] RETURNS

[numHashSlotsAllowed: NAT] =

BEGIN

divisor, quotient, remainder: CARDINAL;

DO divisor ← 2;

[quotient, remainder] ← Basics.DivMod[numHashSlotsDesired, divisor];

quotient ← numHashSlotsDesired / divisor;

remainder ← numHashSlotsDesired MOD divisor;

IF remainder = 0 THEN EXIT; -- this number is not prime.

IF quotient < divisor THEN RETURN[numHashSlotsDesired]; -- prime.

divisor ← divisor + 1;

ENDLOOP;

numHashSlotsDesired ← numHashSlotsDesired + 1;

ENDLOOP;

END;

ClientHash: INTERNAL PROCEDURE[hashDocument: HashDocument] RETURNS [index: NAT--[0..numHashSlots)--] = INLINE {

index ← YggDID.HashDID[hashDocument.did] MOD numHashSlots;

};

ClientEqualKeys: INTERNAL PROCEDURE[hashDocument1, hashDocument2: HashDocument] RETURNS [equal: BOOLEAN] = INLINE {

RETURN[YggDID.EqualDIDs[hashDocument1.did, hashDocument2.did]];

};

end of client-supplied parameters, start of invariant hash package code:

The INTERNAL procedures below expect to be called from a client procedure holding the module monitor lock, which protects the following data structures:

hashSlots: REF HashSlots ← NIL;

HashSlots: TYPE = RECORD[SEQUENCE nSlots: NAT OF HashDocument];

numHashSlots: NAT ← 0; -- boy, will they be sorry if they don't init this package.

lookupHashDocument: HashDocument ← NIL; -- for the "package's" use only.

lruListTop: Document ← NIL;

lruListTail: Document ← NIL;

numberOfHashDocuments: INT ← 0;

desiredNumberOfHashDocuments: INT ← 500;

errors:

HashPkgCallerProgrammingError: ERROR = CODE; -- various fatal conditions.

HashPkgDuplicateKey: ERROR = CODE; -- from Insert.

InitializeHashTable: INTERNAL PROCEDURE[numHashSlotsDesired: NAT, hashDocument: HashDocument] =

BEGIN -- errors: HashPkgCallerProgrammingError (numHashSlotsDesired = 0).

numHashSlots ← ClientHashInit[numHashSlotsDesired];

IF numHashSlots = 0 THEN ERROR HashPkgCallerProgrammingError;

lookupHashDocument ← hashDocument;

hashSlots ← NEW[HashSlots[numHashSlots]];

FOR index: NAT IN [0..numHashSlots)

DO hashSlots[index] ← NIL; ENDLOOP;

END;

Insert: ENTRY PROC [hashDocument: HashDocument] = {

errors: HashPkgDuplicateKey.

ENABLE UNWIND => {};

index: NAT ← ClientHash[hashDocument];

FOR newHashDocument: HashDocument ← hashSlots[index], newHashDocument.next

UNTIL newHashDocument = NIL DO

IF ClientEqualKeys[newHashDocument, hashDocument] AND ~hashDocument.destroyed

THEN ERROR HashPkgDuplicateKey;

ENDLOOP;

hashDocument.next ← hashSlots[index];

hashSlots[index] ← hashDocument;

SafeStorage.EnableFinalization[hashDocument];

numberOfHashDocuments ← numberOfHashDocuments + 1;

};

LookupInCache: ENTRY PROC [key: Key, destroyedOK: BOOL ← FALSE] RETURNS [hashDocument: HashDocument] = {

returns hashDocument = NIL if not found.

ENABLE UNWIND => {};

lookupHashDocument.did ← key;

FOR hashDocument ← hashSlots[ClientHash[lookupHashDocument]], hashDocument.next UNTIL hashDocument = NIL DO

IF ClientEqualKeys[hashDocument, lookupHashDocument] AND (destroyedOK OR ~hashDocument.destroyed) THEN {

SafeStorage.EnableFinalization[hashDocument];

IF hashDocument.lruListPrev # NIL OR hashDocument.lruListNext # NIL OR lruListTop = hashDocument THEN {

On lrulist. Remove it and use it.

IF hashDocument.lruListPrev = NIL THEN {

lruListTop ← hashDocument.lruListNext

}

ELSE {

hashDocument.lruListPrev.lruListNext ← hashDocument.lruListNext;

};

IF lruListTop # NIL THEN lruListTop.lruListPrev ← NIL;

IF hashDocument.lruListNext = NIL THEN {

lruListTail ← hashDocument.lruListPrev

}

ELSE hashDocument.lruListNext.lruListPrev ← hashDocument.lruListPrev;

IF lruListTail # NIL THEN lruListTail.lruListNext ← NIL;

};

hashDocument.lruListPrev ← NIL;

hashDocument.lruListNext ← NIL;

RETURN;

};

ENDLOOP;

RETURN[NIL];

};

FlushDocument: PUBLIC PROC [did: YggDID.DID, tid: Camelot.tidT] = {

Flush a document out of the cache. Testing only!!

doc: YggInternal.Document ← NIL;

IF (doc ← LookupInCache[did]) = NIL THEN {

Delete[doc];

};

Delete: ENTRY PROC [hashDocument: HashDocument] = {

errors: HashPkgCallerProgrammingError (not found).

InnerDelete[hashDocument];

};

InnerDelete: INTERNAL PROC [hashDocument: HashDocument] = {

errors: HashPkgCallerProgrammingError (not found).

ENABLE UNWIND => {};

index: NAT ← ClientHash[hashDocument];

newHashDocument: HashDocument ← NIL;

prevHashDocument: HashDocument ← NIL;

FOR newHashDocument ← hashSlots[index], newHashDocument.next

UNTIL newHashDocument = NIL DO

IF ClientEqualKeys[newHashDocument, hashDocument] THEN EXIT;

prevHashDocument ← newHashDocument;

REPEAT FINISHED => ERROR HashPkgCallerProgrammingError;

ENDLOOP;

IF SafeStorage.IsFinalizationEnabled[newHashDocument] THEN {

newHashDocument.okToRemoveFromDIDMapCache ← TRUE;

}

ELSE {

IF prevHashDocument = NIL THEN hashSlots[index] ← hashDocument.next

ELSE prevHashDocument.next ← hashDocument.next;

numberOfHashDocuments ← numberOfHashDocuments - 1;

};

prevHashDocument = NIL starts the enumeration, returned hashDocument = NIL is the end of the enumeration. This procedure guarantees that any hashDocument in existence throughout the entire enumeration will be seen. Other docs may or not not be seen. HashDocuments may be seen more than once.

EnumerateNext: ENTRY PROCEDURE[prevHashDocument: HashDocument] RETURNS

[hashDocument: HashDocument] = {

errors: none.

ENABLE UNWIND => {};

index: NAT;

IF prevHashDocument = NIL THEN index ← 0

ELSE {

index ← ClientHash[prevHashDocument];

FOR hashDocument ← hashSlots[index], hashDocument.next UNTIL hashDocument = NIL DO

IF ClientEqualKeys[hashDocument, prevHashDocument] THEN GOTO found;

REPEAT

found => {

IF hashDocument.next # NIL THEN RETURN[hashDocument.next];

index ← index + 1;

};

ENDLOOP;

};

UNTIL index >= numHashSlots DO

IF hashSlots[index] # NIL THEN RETURN[hashSlots[index]];

index ← index + 1;

ENDLOOP;

RETURN[NIL];

};

EnumerateWithProc: INTERNAL PROCEDURE[proc: PROCEDURE[hashDocument: HashDocument]

RETURNS[stop: BOOLEAN]] = {

errors: none.

FOR index: NAT IN [0..numHashSlots) DO

FOR hashDocument: HashDocument ← hashSlots[index], hashDocument.next UNTIL hashDocument = NIL DO

IF proc[hashDocument] THEN RETURN;

ENDLOOP;

};

end of invariant hash package code.

Finalization

FinalizationProcess: PROCEDURE = {

FinalizeEntry: ENTRY PROCEDURE [docm: Document] = {

ENABLE UNWIND => {};

IF SafeStorage.IsFinalizationEnabled[docm] THEN RETURN;

IF lruListTop = NIL THEN {

lruListTop ← docm;

}

ELSE {

lruListTail.lruListNext ← docm;

docm.lruListPrev ← lruListTail;

};

lruListTail ← docm;

docm ← NIL;

};

doc: Document;

doc ← NARROW[SafeStorage.FQNext[finalizationQ]];

FinalizeEntry[doc];

doc ← NIL;

ENDLOOP;

};

Trim lru list

CacheTrimProcess: PROC = {

ticksToWait: Process.Ticks;

ticksToWait ← Process.MsecToTicks[197];

overage: INT ← 0;

Process.Pause[ticksToWait];

overage ← numberOfHashDocuments - desiredNumberOfHashDocuments;

IF overage > 0 THEN TrimLruList[overage];

ENDLOOP;

};

TrimLruList: PROC [entriesToTrim: INT] = {

FOR entNo: INT IN [1..entriesToTrim] DO

grabTopOffOfLRU: ENTRY PROC ~ {

IF lruListTop # NIL THEN {

document ← lruListTop;

lruListTop ← lruListTop.lruListNext;

IF lruListTop = NIL THEN lruListTail ← NIL ELSE lruListTop.lruListPrev ← NIL;

document.lruListNext ← NIL;

document.lruListPrev ← NIL;

InnerDelete[document ! HashPkgCallerProgrammingError => CONTINUE; ]; -- CONTINUE so that finalization will work if this module is re-run during debugging

};

document: Document ← NIL;

grabTopOffOfLRU[];

ENDLOOP;

};

Initialization

InitDIDMap: PUBLIC PROC [firstTime: BOOL] ~ {

FileForDIDMap ← YggFile.OpenDIDMapFile[];

[optr: MaxDIDOptr, address: AddressForMaxDID] ← YggFile.GetMaxDIDAddress[];

RecoverableMaxDID ← LOOPHOLE[AddressForMaxDID];

TRUSTED {NextDID ← RecoverableMaxDID^;};

LastDIDAllocateable ← NextDID;

GrabSomeDIDs[];

StartUpExtendibleHash[NIL, firstTime];

IF firstTime THEN {

Init the Root DID

outcome: YggEnvironment.Outcome ← active;

trans: YggEnvironment.TransID;

WHILE outcome # commit DO

doc: YggInternal.Document ← NIL;

trans ← YggTransaction.CreateTrans[YggEnvironment.nullTransID];

doc ← NEW[DocumentRep ← [did: YggDIDPrivate.DIDForRootDIDs, modifiedDIDMap: TRUE, componentFiles: NIL, interlock: FALSE, next: NIL]];

Insert[doc ! HashPkgDuplicateKey => ERROR];

noteUseInTransaction[doc, trans];

YggNav.SetContents[trans: trans, did: YggDIDPrivate.DIDForRootDIDs, contents: [YggRep.did, YggDIDPrivate.DIDForRootDIDs]];

[outcome: outcome] ← YggTransaction.Finish[transID: trans, requestedOutcome: commit];

ENDLOOP;

};

Initialize: PUBLIC ENTRY PROCEDURE[numHashSlotsDesired: NAT, fQLength: CARDINAL] = {

ENABLE UNWIND => {};

IF fQLength = 0 THEN RETURN WITH ERROR ZeroIllegal;

WORDSInWords ← PBasics.bytesPerWord/BYTES[WORD];

finalizationQ ← SafeStorage.NewFQ[fQLength]; -- do this before

SafeStorage.EstablishFinalization[CODE[DocumentRep], 1, finalizationQ ! SafeStorage.CantEstablishFinalization => CONTINUE]; -- allocating any FileObjects.

TRUSTED BEGIN Process.Detach[FORK FinalizationProcess[]]; END;

TURN FINALIZATION BACK ON AFTER DEBUGGING

TRUSTED BEGIN Process.Detach[FORK CacheTrimProcess[]]; END;

secondLookupHashDocument ← NEW[DocumentRep ← [did: NIL, componentFiles: NIL, interlock: FALSE, next: NIL]];

InitializeHashTable[numHashSlotsDesired, secondLookupHashDocument];

TRUSTED {

Process.InitializeCondition[@aShortTime, Process.MsecToTicks[50]];

Process.InitializeCondition[@didsCondition, Process.MsecToTicks[177]];

Process.InitializeCondition[@lock64Condition, Process.MsecToTicks[43]];

};

transactionToModifiedDocumentMap ← RedBlackTree.Create[getKey: GetKeyProc, compare: CompareProc];

FOR h: CARD IN [0..64) DO

index: CARD ← 0;

hLeft: CARD ← h;

addPowerOf2: CARD ← 32;

WHILE hLeft # 0 DO

IF hLeft MOD 2 = 1 THEN index ← index + addPowerOf2;

addPowerOf2 ← addPowerOf2 /2;

hLeft ← hLeft / 2;

ENDLOOP;

Hash64Table[h] ← index;

ENDLOOP;

};

Initialize[129, 250];

END.