YggDIDMapImpl.mesa
Copyright Ó 1988 by Xerox Corporation. All rights reserved.
Last edited by
Bob Hagmann August 10, 1988 4:27:57 pm PDT
DIRECTORY
Atom USING [GetPName, MakeAtom],
Basics USING [BITAND, ByteBlt, charsPerWord, Comparison, Move],
BasicTime USING [Now, nullGMT],
BTree USING [DeleteKey, Entry, EntSize, EnumerateEntries, Key, New, NewPathStk, Open, PageSize, PathStk, ReferencePage, ReleasePage, Tree, UpdateEntry],
Camelot USING [btidT, tidT],
Process USING [Detach, InitializeCondition, MsecToTicks],
RedBlackTree USING [Compare, Create, Delete, GetKey, Insert, Lookup, Table, UserData],
RefText USING [New, TrustTextAsRope],
Rope USING [Equal, Fetch, Length, NewText, ROPE, Substr, Text, TextBound],
SafeStorage USING [EnableFinalization, EstablishFinalization, FinalizationQueue, FQNext, IsFinalizationEnabled, NewFQ],
YggBuffMan USING [ReadPages, ReleaseState, ReleaseVMPageSet, UsePages, VMPageSet],
YggDID USING [CompareDIDs, DID, EqualDIDs, HashDID, StableHashDID],
YggDIDMap,
YggDIDMapPrivate USING [DocItem, DocPart, DocPartPtr, Document, DocumentRep, Entry, EntryPtr, OverheadPerEntry, notCompressed, PartType, RunItem, SegmentItem, SegmentRep, TextRep, TextRP],
YggDIDPrivate USING [DIDRep],
YggEnvironment USING [nullDID],
YggFile USING [BytesForPages, FileHandle, Info, Open, PageCount, SetSize],
YggFixedNames USING [IndexPrefix],
YggInternal
USING [Document, FileHandle];
YggDIDMapImpl:
CEDAR
MONITOR
IMPORTS Atom, Basics, BasicTime, BTree, Process, RedBlackTree, RefText, Rope, SafeStorage, YggBuffMan, YggDID, YggFile, YggFixedNames
EXPORTS YggInternal, YggDIDMap, YggDID =
BEGIN
Data types and variables
Document: TYPE = REF DocumentRep;
DocumentRep: PUBLIC TYPE = YggDIDMapPrivate.DocumentRep;
DID: PUBLIC TYPE ~ REF DIDRep;
DIDRep: PUBLIC TYPE ~ YggDIDPrivate.DIDRep;
TreeEntry: TYPE = REF TreeEntryRep;
TreeEntryRep:
TYPE =
RECORD [
interlock: BOOL ← FALSE,
tid: Camelot.tidT, -- holder of interlock
tree: BTree.Tree,
file: YggFile.FileHandle,
currentVMPageSets: LIST OF YggBuffMan.VMPageSet ← NIL
];
Btrees: ARRAY [0..64) OF TreeEntry;
BtreePageSizeInWords: INT = 4096;
FilePagesPerBtreePage: INT = 8;
MaxWordsPerEntry: INT = (BtreePageSizeInWords - 6)/5;
NextDID: YggDIDPrivate.DIDRep;
LastDIDAllocateable: YggDIDPrivate.DIDRep;
LastDidLow: CARD ← 1000000000;
NumberOfDIDsToGrap: CARD ← 200;
ThresholdToGrabDIDs: CARD ← 20;
KeyObject: TYPE = RECORD [did: YggDID.DID, extensionCount: INT32];
EntryDataBuffer: TYPE = RECORD[SEQUENCE COMPUTED CARD OF EntryWord];
EntryWord:
TYPE =
MACHINE
DEPENDENT
RECORD[
SELECT
OVERLAID *
FROM
card => [card: CARD32],
int => [int: INT32],
pair => [hi, lo: CARD16],
bytes => [hh, hl, lh, ll: BYTE],
bits => [bits: PACKED ARRAY [0..32) OF BOOL],
ENDCASE
];
updateAction: TYPE = {create, delete, newValue};
updates:
TYPE =
RECORD[
did: YggDID.DID,
tid: Camelot.tidT,
action: updateAction,
doc: Document
];
TIDObj: TYPE = REF TIDObjRep;
TIDObjRep: TYPE = RECORD [ btid: Camelot.btidT];
scratchTIDObj: TIDObj ← NEW[TIDObjRep];
DocTrans: TYPE = REF DocTransRep;
DocTransRep:
TYPE =
RECORD [
btidForDoc: TIDObj,
docs: LIST OF Document ← NIL
];
transactionToModifiedDocumentMap: RedBlackTree.Table;
finalizationQ: SafeStorage.FinalizationQueue;
ZeroIllegal: --CALLING-- ERROR = CODE;
secondLookupHashDocument: HashDocument ← NIL; -- wierdness to protect object from finalization.
Exported procedures for documents
OpenDocumentFromDID:
PUBLIC
PROC [did: YggDID.
DID, tid: Camelot.tidT]
RETURNS [doc: YggInternal.Document ←
NIL] = {
Finds or creates a Document for the supplied did and tid.
tryCount: CARD ← 0;
DO
tryAgain: BOOL ← FALSE;
IF (doc ← LookupInCache[did]) =
NIL
THEN {
found: BOOL ← FALSE;
gottaBeInCache: BOOL ← FALSE;
doc ← NEW[DocumentRep ← [did: did, componentFiles: NIL, interlock: TRUE, next: NIL]];
[found, gottaBeInCache] ← LookupInBtree[did, tid, doc];
IF gottaBeInCache
THEN {
doc ← LookupInCache[did];
IF doc = NIL THEN ERROR;
}
ELSE {
Insert[doc ! HashPkgDuplicateKey => {
tryAgain ← TRUE;
CONTINUE;
};
];
Document is now inserted into the cache. Interlock is set. Open up the componentFiles.
openFilesFromParsedDocList[doc, did, tid];
};
}
ELSE {
-- found in cache
waitForInterlockDone:
ENTRY
PROC ~ {
WHILE doc.interlock DO WAIT aShortTime ENDLOOP;
};
IF doc.interlock THEN waitForInterlockDone[];
};
IF ~tryAgain THEN EXIT;
tryCount ← tryCount + 1;
IF tryCount > 25 THEN ERROR;
ENDLOOP;
GetDID:
PUBLIC
PROC [doc: Document]
RETURNS [did: YggDID.
DID] ~ {
errors defined in this interface: none.
RETURN[doc.did];
};
GetLinkInfo:
PUBLIC PROC [doc: Document]
RETURNS [fromDID, toDID: YggDID.
DID, linkType: Rope.
ROPE] ~ {
Get the link info for the Document.
RETURN[doc.fromDID, doc.toDID, doc.linkType];
};
SwapLinkInfo:
PUBLIC PROC [doc: Document, newFromDID, newToDID: YggDID.
DID, newLinkType: Rope.
ROPE]
RETURNS [oldFromDID, oldToDID: YggDID.
DID, oldLinkType: Rope.
ROPE] ~ {
Swap the link info for the document, and report what it used to be.
oldFromDID ← doc.fromDID;
doc.fromDID ← newFromDID;
oldToDID ← doc.toDID;
doc.toDID ← newToDID;
oldLinkType ← doc.linkType;
doc.linkType ← newLinkType;
};
GetComponentFiles:
PUBLIC
PROC [doc: Document]
RETURNS [componentFiles:
LIST
OF YggInternal.FileHandle] ~ {
no errors possible, but may return NIL.
RETURN[doc.componentFiles];
};
AddComponentFile:
PUBLIC ENTRY PROC [doc: Document, componentFile: YggInternal.FileHandle, tid: Camelot.tidT] ~ {
lastCF: LIST OF YggInternal.FileHandle ← NIL;
FOR files:
LIST
OF YggInternal.FileHandle ← doc.componentFiles, files.rest UNTIL files = NIL DO
lastCF ← files;
ENDLOOP;
IF lastCF = NIL THEN doc.componentFiles ← CONS[componentFile, NIL]
ELSE lastCF.rest ← CONS[componentFile, doc.componentFiles];
};
RemoveComponentFile:
PUBLIC
PROC [doc: Document, componentFile: YggInternal.FileHandle, tid: Camelot.tidT] ~ {
lastCF: LIST OF YggInternal.FileHandle ← NIL;
FOR files:
LIST
OF YggInternal.FileHandle ← doc.componentFiles, files.rest
UNTIL files =
NIL
DO
IF files.first = componentFile THEN {
IF lastCF = NIL THEN doc.componentFiles ← files.rest
ELSE lastCF.rest ← files.rest;
EXIT;
};
lastCF ← files;
ENDLOOP;
};
GetName:
PUBLIC
PROC [doc: Document]
RETURNS [Rope.
ROPE] ~ {
no errors possible, but may return NIL.
RETURN[doc.name];
};
CreateNewDID:
PUBLIC
PROC [tid: Camelot.tidT]
RETURNS [did: YggDID.
DID ] ~ {
doc: YggInternal.Document ← NIL;
did ← GetNextDID[];
doc ← NEW[DocumentRep ← [did: did, modifiedDIDMap: TRUE, componentFiles: NIL, interlock: TRUE, next: NIL]];
Insert[doc ! HashPkgDuplicateKey => ERROR];
noteUseInTransaction[doc, tid];
};
DestroyDID:
PUBLIC
PROC [tid: Camelot.tidT, did: YggDID.
DID] ~ {
ERROR;
};
GetRootDIDs:
PUBLIC
PROC
RETURNS [dids:
LIST
OF YggDID.
DID] ~ {
};
AddRootDID:
PUBLIC
PROC [did: YggDID.
DID]
RETURNS [ok:
BOOL ← FALSE] ~ {
};
RemoveRootDID:
PUBLIC
PROC [did: YggDID.
DID]
RETURNS [ok:
BOOL ← FALSE] ~ {
};
Exported for document modifications
aShortTime: CONDITION;
grapDoc:
ENTRY
PROC [doc: YggInternal.Document] ~ {
WHILE doc.interlock DO WAIT aShortTime ENDLOOP;
doc.interlock ← TRUE;
};
releaseDoc: ENTRY PROC [doc: YggInternal.Document] ~ {doc.interlock ← FALSE};
RemoveRuns:
PUBLIC
PROC [did: YggDID.
DID, tid: Camelot.tidT, runList: YggDIDMap.RunList] ~ {
Normally, only called by the file package during it's pre-commit. tid is a top level transaction. Runs listed are removed from the object. Also used when an object is migrated. Caller guarentees that PreCommit will later be called with this tid (if the system doesn't crash).
The document is updated in the cache. PreCommit will make the updates to the disk resident B-tree(s). If Commit is later called, then all is well. If Abort is called, then the cache entry is destroyed. Undo is done in the B-trees by physical pages. Subsequent use of the document requires rebuilding the document object.
doc: YggInternal.Document ← NIL;
IF (doc ← LookupInCache[did]) = NIL THEN ERROR
ELSE {
doc.modifiedDIDMap ← TRUE;
grapDoc[doc];
noteUseInTransaction[doc, tid];
FOR rl: YggDIDMap.RunList ← runList, rl.rest
UNTIL rl =
NIL
DO
pdl: LIST OF YggDIDMapPrivate.DocItem;
prevPdl: LIST OF YggDIDMapPrivate.DocItem ← NIL;
runToDelete: YggDIDMap.Run ← rl.first;
FOR pdl ← doc.parsedDocList, pdl.rest
UNTIL pdl =
NIL
DO
segments: LIST OF YggDIDMapPrivate.SegmentItem;
prevSegments: LIST OF YggDIDMapPrivate.SegmentItem;
FOR segments ← pdl.first.segments, segments.rest
UNTIL segments =
NIL
DO
runs: LIST OF YggDIDMapPrivate.RunItem;
prevRuns: LIST OF YggDIDMapPrivate.RunItem;
FOR runs ← segments.first.runs, runs.rest
UNTIL runs =
NIL
DO
firstIntersection: INT ← -1;
lastIntersection: INT ← -1;
runLast: CARD ← 0;
runToDeleteLast: CARD ← 0;
IF runs.first.segmentId # runToDelete.segmentId
THEN {
-- wrong segment
prevRuns ← runs;
LOOP;
};
firstIntersection ← MAX[runs.first.segmentPage, runToDelete.segmentPage];
lastIntersection ← MIN[
(runLast ← runs.first.segmentPage + runs.first.pages - 1),
(runToDeleteLast ← runToDelete.segmentPage + runToDelete.pages - 1) ];
IF lastIntersection < firstIntersection THEN LOOP; -- no intersection
IF firstIntersection = runs.first.segmentPage
AND lastIntersection = runLast
THEN {
-- run is eliminated
IF prevRuns = NIL THEN segments.first.runs ← runs.rest
ELSE prevRuns.rest ← runs.rest;
LOOP;
}
ELSE {
runs.first.segmentPage ← firstIntersection;
runs.first.pages ← lastIntersection - firstIntersection + 1;
};
prevRuns ← runs;
ENDLOOP;
IF segments.first.runs =
NIL
THEN {
-- no more runs, so dump the segment too
IF prevSegments = NIL THEN pdl.first.segments ← segments.rest
ELSE prevSegments.rest ← segments.rest;
LOOP;
};
prevSegments ← segments;
ENDLOOP;
IF pdl.first.segments =
NIL
THEN {
-- no more segments, so dump the DocItem too
IF prevPdl = NIL THEN doc.parsedDocList ← pdl.rest
ELSE prevPdl.rest ← pdl.rest;
LOOP;
};
prevPdl ← pdl;
ENDLOOP;
ENDLOOP;
releaseDoc[doc];
};
};
AddRuns:
PUBLIC
PROC [did: YggDID.
DID, tid: Camelot.tidT, fileUse:
ATOM, runList: YggDIDMap.RunList] ~ {
Normally, only called by the file package during it's pre-commit. tid is a top level transaction. Runs listed are added to the object. Also used when an object is removed from the disk cache.
doc: YggInternal.Document ← NIL;
partType: YggDIDMapPrivate.PartType;
partType ← mapFileUseToPartType[fileUse];
IF (doc ← LookupInCache[did]) = NIL THEN ERROR
ELSE {
lastPDL: LIST OF YggDIDMapPrivate.DocItem ← NIL;
thisPDL: LIST OF YggDIDMapPrivate.DocItem ← NIL;
doc.modifiedDIDMap ← TRUE;
grapDoc[doc];
noteUseInTransaction[doc, tid];
FOR pdl:
LIST
OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, pdl.rest
UNTIL pdl =
NIL
DO
IF pdl.first.indexName = fileUse
THEN {
lastPDL ← pdl;
EXIT;
};
lastPDL ← pdl;
REPEAT
FINISHED => {
thisPDL ← LIST[[partType, fileUse, 0, NIL]];
IF lastPDL = NIL THEN doc.parsedDocList ← thisPDL
ELSE lastPDL.rest ← thisPDL;
};
ENDLOOP; -- pdl
IF thisPDL = NIL THEN ERROR;
FOR runs: YggDIDMap.RunList ← runList, runs.rest
UNTIL runs =
NIL
DO
lastSegment: LIST OF YggDIDMapPrivate.SegmentItem ← NIL;
FOR segs:
LIST
OF YggDIDMapPrivate.SegmentItem ← thisPDL.first.segments, segs.rest
UNTIL segs =
NIL
DO
lastRun: LIST OF YggDIDMapPrivate.RunItem ← NIL;
runPages: YggFile.PageCount ← 0;
FOR lori:
LIST
OF YggDIDMapPrivate.RunItem ← segs.first.runs, lori.rest
UNTIL lori =
NIL
DO
runPages ← runPages + lori.first.pages;
lastRun ← lori;
ENDLOOP;
IF segs.first.firstPage + runPages = runs.first.firstPage
THEN {
-- append at end of segment
lastRun.rest ← CONS[[segmentId: runs.first.segmentId, segmentPage: runs.first.segmentPage, pages: runs.first.pages], NIL];
EXIT; -- appended at end of segment; go do the next run
};
lastSegment ← segs;
REPEAT
FINISHED => {
-- make a new segment with a singleton run
lastSegment.rest ← CONS[[firstPage: runs.first.firstPage, numberOfBytes: YggFile.BytesForPages[runs.first.pages], runs: LIST[[segmentId: runs.first.segmentId, segmentPage: runs.first.segmentPage, pages: runs.first.pages]]], NIL];
};
ENDLOOP;
ENDLOOP;
releaseDoc[doc];
};
};
SetByteSize:
PUBLIC
PROC [did: YggDID.
DID, tid: Camelot.tidT, fileUse:
ATOM, bytes:
INT] ~ {
Normally, only called by the file package during it's pre-commit. tid is a top level transaction. The size of valid bytes is set to this amount, but the pages for the file are not changed.
doc: YggInternal.Document ← NIL;
partType: YggDIDMapPrivate.PartType;
partType ← mapFileUseToPartType[fileUse];
IF (doc ← LookupInCache[did]) = NIL THEN ERROR
ELSE {
doc.modifiedDIDMap ← TRUE;
grapDoc[doc];
noteUseInTransaction[doc, tid];
FOR pdl:
LIST
OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, pdl.rest
UNTIL pdl =
NIL
DO
IF pdl.first.indexName = fileUse
THEN {
FOR segs:
LIST
OF YggDIDMapPrivate.SegmentItem ← pdl.first.segments, segs.rest
UNTIL segs =
NIL
DO
firstByte: CARD;
firstByte ← YggFile.BytesForPages[segs.first.firstPage];
IF firstByte > bytes THEN segs.first.numberOfBytes ← 0
ELSE {
runPages: YggFile.PageCount ← 0;
runBytes: CARD ← 0;
FOR lori:
LIST
OF YggDIDMapPrivate.RunItem ← segs.first.runs, lori.rest
UNTIL lori =
NIL
DO
runPages ← runPages + lori.first.pages;
ENDLOOP;
runBytes ← YggFile.BytesForPages[runPages];
segs.first.numberOfBytes ← IF firstByte + runBytes > bytes THEN bytes - firstByte ELSE runBytes;
};
ENDLOOP;
EXIT;
};
REPEAT
FINISHED => {
ERROR;
};
ENDLOOP; -- pdl
};
};
Utilities
mapFileUseToPartType:
PROC [fileUse:
ATOM]
RETURNS [partType: YggDIDMapPrivate.PartType] ~ {
SELECT fileUse
FROM
$root => partType ← root;
$contents => partType ← contents;
$outlinks => partType ← outlinks;
$attributes => partType ← attributes;
ENDCASE => {
pName: Rope.Text;
pName ← Atom.GetPName[fileUse];
IF Rope.Equal[YggFixedNames.IndexPrefix, Rope.Substr[pName, 0, 7]]
THEN {
partType ← index;
}
ELSE ERROR;
};
};
Exported procedures for transactions
PreCommit:
PUBLIC
PROC [tid: Camelot.tidT] ~ {
Take all the modified documents from the transaction and write out their changes to the DID Map.
Find all the B-trees that have to be updated. Process them in ascending order, doing all of the updates for the whole tree. We can't deadlock within B-tree update since we've ordered the locks.
First, lock the tree for exclusive access. Perform the update(s) including the Camelot pin and log dance. Downgrade to lock to allow readers (they can see uncommitted data).
If the transaction commits, completely unlock the tree. If the transaction aborts, lock the tree for exclusive access, perform undo, and after undo processing is done unlock the tree.
data: RedBlackTree.UserData;
scrTIDObj: TIDObj ← CheckoutTIDObj[tid.top];
data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scrTIDObj];
RecycleTIDObj[scrTIDObj];
IF data #
NIL
THEN {
dt: DocTrans;
maxBTreeDone: INT ← -1;
dt ← NARROW[data];
DO
minBTree: INT ← 65;
FOR docList:
LIST
OF Document ← dt.docs, docList.rest
UNTIL docList =
NIL
DO
-- look at all the documents to find the next hash value to do
hashedDidToBTree: INT;
hashedDidToBTree ← HashToBtree[docList.first.did];
IF hashedDidToBTree > maxBTreeDone AND hashedDidToBTree < minBTree THEN minBTree ← hashedDidToBTree;
ENDLOOP;
IF minBTree = maxBTreeDone THEN EXIT;
LockBTree[minBTree, tid];
FOR docList:
LIST
OF Document ← dt.docs, docList.rest
UNTIL docList =
NIL
DO
-- look at all the documents and process those with the correct hash value
hashedDidToBTree: INT;
hashedDidToBTree ← HashToBtree[docList.first.did];
IF hashedDidToBTree = minBTree
THEN {
writeDocumentToBTree[hashedDidToBTree, docList.first];
};
ENDLOOP;
maxBTreeDone ← minBTree;
ENDLOOP;
};
};
Commit:
PUBLIC
PROC [tid: Camelot.tidT] ~ {
data: RedBlackTree.UserData;
scrTIDObj: TIDObj ← CheckoutTIDObj[tid.top];
data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scrTIDObj];
IF data #
NIL
THEN {
dt: DocTrans;
maxBTreeDone: INT ← -1;
dt ← NARROW[data];
FOR docList:
LIST
OF Document ← dt.docs, docList.rest
UNTIL docList =
NIL
DO
-- look at all the documents and turn off the modified flag
docList.first.modifiedDIDMap ← FALSE;
ENDLOOP;
DO
minBTree: INT ← 65;
FOR docList:
LIST
OF Document ← dt.docs, docList.rest
UNTIL docList =
NIL
DO
-- look at all the documents to find the next hash value to do
hashedDidToBTree: INT;
hashedDidToBTree ← HashToBtree[docList.first.did];
IF hashedDidToBTree > maxBTreeDone AND hashedDidToBTree < minBTree THEN minBTree ← hashedDidToBTree;
ENDLOOP;
IF minBTree = maxBTreeDone THEN EXIT;
maxBTreeDone ← minBTree;
UnlockBTree[minBTree, tid];
ENDLOOP;
};
[] ← RedBlackTree.Delete[ transactionToModifiedDocumentMap, scrTIDObj];
RecycleTIDObj[scrTIDObj];
};
Abort:
PUBLIC
PROC [tid: Camelot.tidT] ~ {
data: RedBlackTree.UserData;
scrTIDObj: TIDObj ← CheckoutTIDObj[tid.top];
data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scrTIDObj];
IF data #
NIL
THEN {
dt: DocTrans;
maxBTreeDone: INT ← -1;
dt ← NARROW[data];
FOR docList:
LIST
OF Document ← dt.docs, docList.rest
UNTIL docList =
NIL
DO
-- look at all the documents and remove them from the cache
Delete[ docList.first];
ENDLOOP;
DO
minBTree: INT ← 65;
FOR docList:
LIST
OF Document ← dt.docs, docList.rest
UNTIL docList =
NIL
DO
-- look at all the documents to find the next hash value to do
hashedDidToBTree: INT;
hashedDidToBTree ← HashToBtree[docList.first.did];
IF hashedDidToBTree > maxBTreeDone AND hashedDidToBTree < minBTree THEN minBTree ← hashedDidToBTree;
ENDLOOP;
IF minBTree = maxBTreeDone THEN EXIT;
maxBTreeDone ← minBTree;
UnlockBTree[minBTree, tid];
ENDLOOP;
};
[] ← RedBlackTree.Delete[ transactionToModifiedDocumentMap, scrTIDObj];
RecycleTIDObj[scrTIDObj];
};
Files
openFilesFromParsedDocList:
PROC [doc: YggInternal.Document, did: YggDID.
DID, tid: Camelot.tidT] = {
FOR pdl:
LIST
OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, pdl.rest
UNTIL pdl =
NIL
DO
atomCode: ATOM ← NIL;
theRuns: YggDIDMap.RunList ← NIL;
theFile: YggFile.FileHandle;
SELECT pdl.first.partType
FROM
root => atomCode ← $root;
contents => atomCode ← $contents;
outlinks => atomCode ← $outlinks;
index => {
atomCode ← pdl.first.indexName;
};
attributes => atomCode ← $attributes;
ENDCASE => atomCode ← $huh;
FOR segments:
LIST
OF YggDIDMapPrivate.SegmentItem ← pdl.first.segments, segments.rest
UNTIL segments =
NIL
DO
nowPage: CARD ← segments.first.firstPage;
FOR runs:
LIST
OF YggDIDMapPrivate.RunItem ← segments.first.runs, runs.rest
UNTIL runs =
NIL
DO
theRuns ← CONS[[segmentId: runs.first.segmentId, segmentPage: runs.first.segmentPage, firstPage: nowPage, pages: runs.first.pages, leader: FALSE], theRuns];
nowPage ← nowPage + runs.first.pages;
ENDLOOP;
ENDLOOP;
theFile ← YggFile.Open[runList: theRuns, did: did, fileUse: atomCode, verifyLeaderNow: FALSE];
doc.componentFiles ← CONS[theFile, doc.componentFiles];
ENDLOOP;
};
Utilities
AppendText:
UNSAFE
PROC [dPP: YggDIDMapPrivate.DocPartPtr, text: Rope.Text, offset: CARD]
RETURNS [textRP: YggDIDMapPrivate.TextRP, words: CARD] =
UNCHECKED {
textRep: POINTER TO YggDIDMapPrivate.TextRep;
textRP ← LOOPHOLE[offset];
textRep ← @dPP[textRP];
LOOPHOLE[textRep, LONG POINTER TO Rope.TextBound]^ ← Rope.Length[text]; -- sets textRep.length
[] ← Basics.ByteBlt [
to: [ BASE[DESCRIPTOR[textRep]], 0, textRep.length ],
from: [ BASE[DESCRIPTOR[text]], 0, textRep.length ]
];
words ← WORDS[YggDIDMapPrivate.TextRep[textRep.length]];
};
GetText:
UNSAFE
PROC [textRep:
LONG
POINTER
TO YggDIDMapPrivate.TextRep, text:
REF
TEXT] =
UNCHECKED {
nBytes: CARD;
text.length ← textRep.length;
nBytes ← Basics.ByteBlt [
to: [ BASE[DESCRIPTOR[text]], 0, textRep.length ],
from: [ BASE[DESCRIPTOR[textRep]], 0, textRep.length ]
];
};
DID allocation
didsCondition: CONDITION;
GetNextDID:
ENTRY
PROC
RETURNS [did: YggDID.
DID] ~ {
WHILE NextDID.didHigh = LastDIDAllocateable.didHigh AND NextDID.didLow = LastDIDAllocateable.didLow DO WAIT didsCondition ENDLOOP;
did ← NEW[YggDIDPrivate.DIDRep ← [NextDID.didHigh, NextDID.didLow]];
NextDID.didLow ← NextDID.didLow + 1;
IF LastDIDAllocateable.didLow - NextDID.didLow <= ThresholdToGrabDIDs THEN Process.Detach[FORK GrabSomeDIDs[]];
};
GrabberCount: CARD ← 0;
GrabSomeDIDs:
PROC [] ~ {
newLastDIDAllocateable: YggDIDPrivate.DIDRep;
newHigh: BOOL ← FALSE;
onlyOne:
ENTRY
PROC
RETURNS [exit:
BOOL] ~ {
IF GrabberCount # 0 THEN RETURN[TRUE];
GrabberCount ← GrabberCount + 1;
};
IF onlyOne[] THEN RETURN;
newLastDIDAllocateable.didHigh ← LastDIDAllocateable.didHigh;
newLastDIDAllocateable.didLow ← LastDIDAllocateable.didLow + NumberOfDIDsToGrap;
IF newLastDIDAllocateable.didLow > LastDidLow
THEN {
newLastDIDAllocateable.didHigh ← newLastDIDAllocateable.didHigh + 1;
newLastDIDAllocateable.didLow ← NumberOfDIDsToGrap;
newHigh ← TRUE;
};
Do a transaction that updates the next did field in the DID Map segment metadata. Repeat until one commits. It's gotta commit in a healthy system - nothing can conflict with it.
IF newHigh
THEN {
m:
ENTRY
PROC ~ {
LastDIDAllocateable ← newLastDIDAllocateable;
NextDID.didHigh ← newLastDIDAllocateable.didHigh;
NextDID.didLow ← 0;
BROADCAST didsCondition;
};
m[];
}
ELSE {
n:
ENTRY
PROC ~ {
LastDIDAllocateable.didLow ← newLastDIDAllocateable.didLow;
BROADCAST didsCondition;
};
n[];
};
};
DID B-tree access
bTreeCondition: CONDITION;
LockBTree:
ENTRY
PROC [treeNo:
INT, tid: Camelot.tidT] ~ {
WHILE Btrees[treeNo].interlock DO WAIT bTreeCondition ENDLOOP;
Btrees[treeNo].interlock ← TRUE;
Btrees[treeNo].tid ← tid;
};
UnlockBTree:
ENTRY
PROC [treeNo:
INT, tid: Camelot.tidT] ~ {
IF ~Btrees[treeNo].interlock OR Btrees[treeNo].tid # tid THEN ERROR;
};
stockKey: REF KeyObject ← NIL;
CheckoutKey:
ENTRY
PROC [did: YggDID.
DID, extendedDataCount:
INT32]
RETURNS [k:
REF KeyObject] = {
IF stockKey = NIL THEN k ← NEW [KeyObject] ELSE { k ← stockKey; stockKey ← NIL };
k^ ← KeyObject[did, extendedDataCount];
};
RecycleKey:
ENTRY
PROC [k:
REF KeyObject] = {
stockKey ← k;
};
HashToBtree:
PROC [did: YggDID.
DID]
RETURNS [h:
INT] = {
h ← Basics.BITAND[YggDID.StableHashDID[did], 03Fh]; -- low 6 bits
};
LookupInBtree:
PROC [did: YggDID.
DID, tid: Camelot.tidT, doc: YggInternal.Document]
RETURNS [found:
BOOL ←
FALSE, gottaBeInCache:
BOOL ←
FALSE] = {
hashedDidToBTree: INT;
btreeEntry: TreeEntry;
pathStk: BTree.PathStk ← BTree.NewPathStk[];
keyRef: REF KeyObject = CheckoutKey[did, -3];
extendedData: REF EntryDataBuffer ← NIL;
extendedDataCount: CARD ← 0;
expectedExtendedDataSize: CARD ← 0;
maxDataCount: CARD ← 0;
fromLink: DID ← YggEnvironment.nullDID;
toLink: DID ← YggEnvironment.nullDID;
linkType: Rope.ROPE ← NIL;
updatesForTrans: LIST OF updates;
getAll:
UNSAFE
PROC [entry: BTree.Entry]
RETURNS [continue:
BOOL ←
FALSE] = {
entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[entry];
IF entry #
NIL
THEN
TRUSTED {
IF YggDID.CompareDIDs[did, @entryPtr.did] # equal THEN RETURN;
found ←
TRUE;
SELECT entryPtr.extensionCount
FROM
0 => {
-- normal, non-extended entry
index: INT ← 0;
IF entryPtr.link
THEN {
wordIndex: INT ← 0;
noChars: INT ← 0;
nullSeen: BOOL ← FALSE;
fromLink ← NEW[DIDRep];
Basics.Move[dst: @fromLink, src: @entryPtr.data[0], nWords: WORDS[DIDRep]];
index ← WORDS[DIDRep];
toLink ← NEW[DIDRep];
Basics.Move[dst: @toLink, src: @entryPtr.data[index], nWords: WORDS[DIDRep]];
index ← index + index;
FOR wordIndex ← index, wordIndex + 1
UNTIL nullSeen
DO
fourChars: PACKED ARRAY [0..Basics.charsPerWord) OF CHAR;
fourChars ← LOOPHOLE[@entryPtr.data[index]];
FOR charNo:
INT
IN [0..Basics.charsPerWord)
DO
IF fourChars[charNo] = 0C THEN {nullSeen← TRUE; EXIT;};
noChars ← noChars + 1;
ENDLOOP;
ENDLOOP;
IF noChars = 0
THEN {
index ← index + 1;
}
ELSE {
linkTypeRT: REF TEXT ← NIL;
linkTypeRT ← RefText.New[noChars];
nullSeen ← FALSE;
UNTIL nullSeen
DO
fourChars: PACKED ARRAY [0..Basics.charsPerWord) OF CHAR;
fourChars ← LOOPHOLE[@entryPtr.data[index]];
index ← index + 1;
FOR charNo:
INT
IN [0..Basics.charsPerWord)
DO
IF fourChars[charNo] = 0C THEN {nullSeen← TRUE; EXIT;};
linkTypeRT[noChars] ← fourChars[charNo];
noChars ← noChars + 1;
ENDLOOP;
ENDLOOP;
linkTypeRT.length ← noChars;
linkType ← RefText.TrustTextAsRope[linkTypeRT];
};
};
[] ← parseDocPart[@entryPtr.data[index], entryPtr.size - YggDIDMapPrivate.OverheadPerEntry - index, doc];
};
1 => {
-- normal, extended entries to follow
expectedExtendedDataSize ← entryPtr.data[0];
extendedData ← NEW[EntryDataBuffer[expectedExtendedDataSize]];
extendedDataCount ← entryPtr.size - YggDIDMapPrivate.OverheadPerEntry - 1;
Basics.Move[dst: @extendedData[0], src: @entryPtr.data[1], nWords: extendedDataCount];
};
>1 => {
-- extended entry
IF extendedDataCount + entryPtr.size - YggDIDMapPrivate.OverheadPerEntry > maxDataCount THEN ERROR;
Basics.Move[dst: @extendedData[extendedDataCount], src: @entryPtr.data[0], nWords: entryPtr.size - YggDIDMapPrivate.OverheadPerEntry];
extendedDataCount ← extendedDataCount + entryPtr.size - YggDIDMapPrivate.OverheadPerEntry;
};
ENDCASE => ERROR;
};
};
hashedDidToBTree ← HashToBtree[did];
btreeEntry ← Btrees[hashedDidToBTree];
updatesForTrans ← getUpdates[tid];
FOR ut:
LIST
OF updates ← updatesForTrans, ut.rest
UNTIL ut =
NIL
DO
IF YggDID.EqualDIDs[ut.first.did, did] THEN RETURN[FALSE, TRUE]; -- if it has updates, then is should be in the cache - so why are we in this routine?
ENDLOOP;
DO
TRUSTED {IF BTree.EnumerateEntries[tree: btreeEntry.tree, relation: greater, key: keyRef, pathStk: pathStk, useExistingPath: TRUE, Proc: getAll] THEN EXIT; };
ENDLOOP;
IF maxDataCount # 0
THEN
TRUSTED {
IF maxDataCount # extendedDataCount THEN ERROR;
[] ← parseDocPart[@extendedData[0], maxDataCount, doc];
};
RecycleKey[keyRef];
};
InsertIntoBtree:
PROC [bTree: BTree.Tree, did:
DID, doc: YggInternal.Document, buffer:
REF EntryDataBuffer, wordsInBuffer:
INT, link:
BOOL] = {
totalEntrySize: INT;
pathStk: BTree.PathStk ← BTree.NewPathStk[];
keyRef: REF KeyObject = CheckoutKey[did, -3];
totalEntrySize ← YggDIDMapPrivate.OverheadPerEntry + wordsInBuffer;
IF totalEntrySize <= MaxWordsPerEntry
THEN {
writeEntrySingle:
UNSAFE
PROCEDURE [entry: BTree.Entry] =
UNCHECKED {
entPtr: POINTER TO YggDIDMapPrivate.Entry = LOOPHOLE[entry];
entPtr.size ← totalEntrySize;
entPtr.extensionCount ← 0;
entPtr.did ← did^;
entPtr.link ← link;
Basics.Move[dst: @entPtr.data[0], src: LOOPHOLE[buffer], nWords: wordsInBuffer];
};
keyRef. extensionCount ← 0;
TRUSTED {BTree.UpdateEntry[tree: bTree, key: keyRef, pathStk: pathStk, words: totalEntrySize, Proc: writeEntrySingle, updateType: insertOrReplace];};
insure that an old "normal with extentions" entry does not exist
keyRef.extensionCount ← 1;
}
ELSE {
writeEntryFirst:
UNSAFE
PROCEDURE [entry: BTree.Entry] =
UNCHECKED {
writeWords: INT ← MaxWordsPerEntry - YggDIDMapPrivate.OverheadPerEntry - 1;
entPtr: POINTER TO YggDIDMapPrivate.Entry = LOOPHOLE[entry];
entPtr.size ← MaxWordsPerEntry;
entPtr.extensionCount ← 1;
entPtr.did ← did^;
entPtr.data[0] ← wordsInBuffer;
leftToWrite ← wordsInBuffer - writeWords;
Basics.Move[dst: @entPtr.data[1], src: LOOPHOLE[buffer], nWords: writeWords];
writeOffset ← writeWords;
};
leftToWrite: INT ← 0;
writeOffset: INT ← 0;
keyRef.extensionCount ← 1;
TRUSTED {BTree.UpdateEntry[tree: bTree, key: keyRef, pathStk: pathStk, words: MaxWordsPerEntry, Proc: writeEntryFirst, updateType: insertOrReplace]; };
keyRef.extensionCount ← 0;
[] ← BTree.DeleteKey[tree: bTree, key: keyRef, pathStk: pathStk, useExistingPath: TRUE];
DO
writeEntryExtended:
UNSAFE
PROCEDURE [entry: BTree.Entry] =
UNCHECKED{
entPtr: POINTER TO YggDIDMapPrivate.Entry = LOOPHOLE[entry];
entPtr.size ← writeThisTime+YggDIDMapPrivate.OverheadPerEntry;
entPtr.extensionCount ← keyRef.extensionCount;
entPtr.did ← did^;
entPtr.data[0] ← writeThisTime;
Basics.Move[dst: @entPtr.data[1], src: LOOPHOLE[buffer, POINTER]+writeOffset, nWords: writeThisTime];
};
writeThisTime: INT ← MIN[MaxWordsPerEntry - YggDIDMapPrivate.OverheadPerEntry, leftToWrite];
leftToWrite ← leftToWrite - writeThisTime;
keyRef.extensionCount ← keyRef.extensionCount + 1;
TRUSTED {BTree.UpdateEntry[tree: bTree, key: keyRef, pathStk: pathStk, useExistingPath: TRUE, words: writeThisTime + YggDIDMapPrivate.OverheadPerEntry, Proc: writeEntryFirst, updateType: insertOrReplace]; };
IF leftToWrite < 0 THEN ERROR;
IF leftToWrite = 0 THEN EXIT;
ENDLOOP;
};
DO
-- insure that old extra extentions do not exist
IF ~BTree.DeleteKey[tree: bTree, key: keyRef, pathStk: pathStk, useExistingPath: TRUE] THEN EXIT;
keyRef.extensionCount ← keyRef.extensionCount + 1;
ENDLOOP;
RecycleKey[keyRef];
};
getUpdates:
PROC [tid: Camelot.tidT]
RETURNS [updateList:
LIST
OF updates] ~ {
};
parseDocPart:
PROC [dataToParse:
LONG
POINTER, size:
CARD, doc: YggInternal.Document]
RETURNS [found:
BOOL ←
FALSE] = {
This code must be the dual of writeDocToBuffer
parsedDocList: LIST OF YggDIDMapPrivate.DocItem ← NIL;
parsedDocListTail: LIST OF YggDIDMapPrivate.DocItem ← NIL;
index: CARD ← 0;
WHILE index < size
DO
TRUSTED {
segIndex: POINTER TO YggDIDMapPrivate.SegmentRep;
docPart: POINTER TO YggDIDMapPrivate.DocPart;
newDocList: LIST OF YggDIDMapPrivate.DocItem ← NIL;
segmentsListTail: LIST OF YggDIDMapPrivate.SegmentItem ← NIL;
startOffset: CARD ← 0;
offsetToLastSegment: CARD ← 0;
indexName: ATOM ← NIL;
docPart ← LOOPHOLE[dataToParse + index];
IF index + docPart.docPartSize > size THEN ERROR;
IF docPart.indexName #
NIL
THEN {
r: Rope.Text;
docPartPtr: YggDIDMapPrivate.DocPartPtr = LOOPHOLE[docPart];
indexNamePtr: POINTER TO YggDIDMapPrivate.TextRep;
indexNamePtr ← @docPartPtr[docPartPtr.indexName];
IF indexNamePtr.length = 0 THEN ERROR;
offsetToLastSegment ← LOOPHOLE[indexNamePtr, CARD] - LOOPHOLE[docPart, CARD];
r ← Rope.NewText[indexNamePtr.length];
GetText[indexNamePtr, LOOPHOLE[r]];
indexName ← Atom.MakeAtom[r];
}
ELSE offsetToLastSegment ← docPart.docPartSize;
newDocList ← LIST[[partType: docPart.partType, indexName: indexName, slotNo: docPart.slotNo, segments: NIL]];
segIndex ← LOOPHOLE[@docPart.segments];
WHILE (startOffset ←
LOOPHOLE[segIndex,
CARD] -
LOOPHOLE[docPart,
CARD]) < offsetToLastSegment
DO
segSize: CARD ← 0;
runs: LIST OF YggDIDMapPrivate.RunItem ← NIL;
runTail: LIST OF YggDIDMapPrivate.RunItem ← NIL;
newSegListItem: LIST OF YggDIDMapPrivate.SegmentItem;
newSegListItem ← LIST[[firstPage: segIndex.firstPage, numberOfBytes: segIndex.numberOfBytes, runs: NIL]];
IF (segSize ← WORDS[YggDIDMapPrivate.SegmentRep[segIndex.length]]) > docPart.docPartSize - startOffset THEN ERROR; -- too many runs
FOR inx:
CARDINAL
IN [0..segIndex.length)
DO
newRunList: LIST OF YggDIDMapPrivate.RunItem ← NIL;
newRunList ← LIST[[segmentId: segIndex.runs[inx].segmentId, segmentPage: segIndex.runs[inx].segmentPage, pages: segIndex.runs[inx].pages]];
IF runTail = NIL THEN runs ← runTail ← newRunList
ELSE {
runTail.rest ← newRunList;
runTail ← newRunList;
};
ENDLOOP;
newSegListItem.first.runs ← runs;
IF newDocList.first.segments = NIL THEN newDocList.first.segments ← segmentsListTail ← newSegListItem
ELSE {
segmentsListTail.rest ← newSegListItem;
segmentsListTail ← newSegListItem;
};
segIndex ← segIndex + segSize;
ENDLOOP;
IF startOffset # docPart.docPartSize THEN ERROR;
IF parsedDocListTail = NIL THEN parsedDocList ← parsedDocListTail ← newDocList
ELSE {
parsedDocListTail.rest ← newDocList;
parsedDocListTail ← newDocList;
};
index ← index + docPart.docPartSize;
};
ENDLOOP;
doc.parsedDocList ← parsedDocList;
};
stockBuffer: REF EntryDataBuffer ← NIL;
CheckoutStockBuffer:
ENTRY
PROC
RETURNS [buf:
REF EntryDataBuffer] = {
IF stockBuffer = NIL THEN buf ← NEW[EntryDataBuffer[256]] ELSE { buf ← stockBuffer; stockBuffer ← NIL };
};
RecycleStockBuffer:
ENTRY
PROC [buf:
REF EntryDataBuffer] = {
stockBuffer ← buf;
};
writeDocumentToBTree:
PROC [ hashedDidToBTree:
INT, doc: Document] ~ {
buffer: REF EntryDataBuffer ← NIL;
stockBuffer: REF EntryDataBuffer ← NIL;
bufferSize: INT ← 256;
link: BOOL;
wordsInBuffer: INT;
stockBuffer ← buffer ← CheckoutStockBuffer[];
DO
didNotFit: BOOL ← TRUE;
[wordsInBuffer: wordsInBuffer, didNotFit: didNotFit, link: link] ← writeDocToBuffer[doc, buffer, bufferSize];
IF didNotFit
THEN {
bufferSize ← bufferSize + bufferSize;
buffer ← NEW[EntryDataBuffer[bufferSize]];
LOOP;
};
EXIT;
ENDLOOP;
InsertIntoBtree[bTree: Btrees[hashedDidToBTree].tree, did: doc.did, doc: doc, buffer: buffer, wordsInBuffer: wordsInBuffer, link: link];
RecycleStockBuffer[stockBuffer];
};
writeDocToBuffer:
PROC [ doc: Document, buffer:
REF EntryDataBuffer, bufferSize:
INT]
RETURNS [wordsInBuffer:
INT ← 0, didNotFit:
BOOL ←
TRUE, link: BOOL ← FALSE] ~ {
This code must be the dual of parseDocPart
IF doc.fromDID # YggEnvironment.nullDID
THEN {
linkTypeLen: INT ← 0;
linkTypeWords: INT ← 0;
nowCharNum: INT ← 0;
link ← TRUE;
TRUSTED {Basics.Move[dst: LOOPHOLE[@buffer[0]], src: LOOPHOLE[@(doc.fromDID^)], nWords: WORDS[DIDRep]]; };
wordsInBuffer ← WORDS[DIDRep];
TRUSTED {Basics.Move[dst: LOOPHOLE[@buffer[wordsInBuffer]], src: LOOPHOLE[@(doc.toDID^)], nWords: WORDS[DIDRep]]; };
wordsInBuffer ← wordsInBuffer + WORDS[DIDRep];
linkTypeLen ← Rope.Length[doc.linkType];
linkTypeWords ← (linkTypeLen/Basics.charsPerWord) + 1;
IF linkTypeWords + wordsInBuffer > bufferSize THEN RETURN[0, TRUE];
FOR i:
INT
IN [0..linkTypeWords)
DO
fourChars: PACKED ARRAY [0..Basics.charsPerWord) OF CHAR;
FOR charNo:
INT
IN [0..Basics.charsPerWord)
DO
IF nowCharNum >= linkTypeLen THEN fourChars[charNo] ← 0C
ELSE fourChars[charNo] ← Rope.Fetch[doc.linkType, nowCharNum];
nowCharNum ← nowCharNum + 1;
ENDLOOP;
TRUSTED{buffer[wordsInBuffer] ← LOOPHOLE[fourChars];};
wordsInBuffer ← wordsInBuffer + 1;
ENDLOOP;
};
FOR parsedDocList:
LIST
OF YggDIDMapPrivate.DocItem ← doc.parsedDocList, parsedDocList.rest
UNTIL parsedDocList =
NIL
DO
indexNameWords: CARD;
segmentOffset: CARD ← 0;
docPart: POINTER TO YggDIDMapPrivate.DocPart;
IF wordsInBuffer + WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[1]] > bufferSize THEN RETURN [0, TRUE];
TRUSTED {
docPart ← LOOPHOLE[@buffer[wordsInBuffer]];
docPart.compression ← YggDIDMapPrivate.notCompressed;
docPart.keepingModifications ← FALSE;
docPart.partType ← parsedDocList.first.partType;
docPart.extraFlags ← 0;
docPart.slotNo ← parsedDocList.first.slotNo;
docPart.indexName ← NIL;
docPart.versionIdentifierString ← NIL;
};
FOR segments:
LIST
OF YggDIDMapPrivate.SegmentItem ← parsedDocList.first.segments, segments.rest
UNTIL segments =
NIL
DO
runIndex: CARD ← 0;
segIndex: POINTER TO YggDIDMapPrivate.SegmentRep;
IF wordsInBuffer + segmentOffset + WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[1]] > bufferSize THEN RETURN [0, TRUE];
TRUSTED {
segIndex ← LOOPHOLE[@docPart.segments + segmentOffset];
segIndex.firstPage ← segments.first.firstPage;
segIndex.numberOfBytes ← segments.first.numberOfBytes;
segIndex.modificationDeltasOffset ← 0;
segIndex.lastUsedTime ← BasicTime.Now[];
segIndex.backUpTime ← BasicTime.nullGMT;
segIndex.backUpID ← 0;
};
FOR rl:
LIST
OF YggDIDMapPrivate.RunItem ← segments.first.runs, rl.rest
UNTIL rl =
NIL
DO
IF wordsInBuffer + segmentOffset + WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[runIndex]] > bufferSize THEN RETURN [0, TRUE];
TRUSTED {segIndex.runs[runIndex] ← [segmentId: rl.first.segmentId, segmentPage: rl.first.segmentPage, pages: rl.first.pages]; };
runIndex ← runIndex + 1;
ENDLOOP;
TRUSTED {segIndex.length ← runIndex; };
segmentOffset ← segmentOffset + WORDS[YggDIDMapPrivate.SegmentRep[runIndex]];
ENDLOOP;
TRUSTED {
docPart.docPartSize ← WORDS[YggDIDMapPrivate.DocPart] - WORDS[CARD] + segmentOffset + indexNameWords;
wordsInBuffer ← wordsInBuffer + docPart.docPartSize;
};
IF parsedDocList.first.indexName #
NIL
THEN {
pName: Rope.Text;
pName ← Atom.GetPName[parsedDocList.first.indexName];
IF wordsInBuffer + WORDS[YggDIDMapPrivate.TextRep[Rope.Length[pName]]] > bufferSize THEN RETURN [0, TRUE];
TRUSTED {
[docPart.indexName, indexNameWords] ← AppendText[dPP: docPart, text: pName, offset: docPart.docPartSize];
docPart.docPartSize ← docPart.docPartSize + indexNameWords;
};
wordsInBuffer ← wordsInBuffer + indexNameWords;
};
ENDLOOP;
RETURN[wordsInBuffer: wordsInBuffer, didNotFit: FALSE];
};
BTree access primitives
BtreeCompare:
UNSAFE
PROC [key: BTree.Key, entry: BTree.Entry]
RETURNS [comp: Basics.Comparison] =
UNCHECKED {
keyRef: REF KeyObject = NARROW [key];
entryPtr: YggDIDMapPrivate.EntryPtr = LOOPHOLE[entry];
entryDID: POINTER TO DIDRep = @entryPtr.did;
SELECT YggDID.CompareDIDs[keyRef.did, entryDID]
FROM
less => RETURN[less];
greater => RETURN[greater];
ENDCASE =>
SELECT keyRef.extensionCount
FROM
< entryPtr.extensionCount => RETURN [less];
> entryPtr.extensionCount => RETURN [greater];
ENDCASE => RETURN[equal];
};
BtreeEntrySize:
UNSAFE
PROC [entry: BTree.Entry]
RETURNS [words: BTree.EntSize] =
UNCHECKED {
RETURN [ MIN[LOOPHOLE[entry, YggDIDMapPrivate.EntryPtr].size, BTree.PageSize.LAST] ];
};
BTree storage primitives
StartUpBTree:
PROC [btreeInfo:
REF]
RETURNS [tree: BTree.Tree] = {
tree ← BTree.New [
repPrim: [compare: BtreeCompare, entrySize: BtreeEntrySize],
storPrim: [referencePage: BTreeReferencePage, releasePage: BTreeReleasePage],
minEntrySize: WORDS[YggDIDMapPrivate.Entry] + WORDS[YggDIDMapPrivate.DocPart] + WORDS[YggDIDMapPrivate.SegmentRep[1]],
initialState: suspended
];
BTree.Open[
tree: tree,
storage: btreeInfo,
pageSize: BtreePageSizeInWords,
initialize: FALSE,
maintainRecomputableState: TRUE
];
};
debugReferencePageNumber: INT ← 0;
debugReferencePageNumberCount: INT ← 0;
BTreeReferencePage:
PUBLIC BTree.ReferencePage = {
PROC [storage: PageStorage, number: PageNumber, type: ReferenceType]
RETURNS [ptr: PagePtr]
addVMPageSet:
ENTRY
PROC = {
treeEntry.currentVMPageSets ← CONS[vMPageSet, treeEntry.currentVMPageSets];
};
debugReferencePageNumberCount: INT ← 0;
treeEntry: TreeEntry;
vMPageSet: YggBuffMan.VMPageSet;
IF number = debugReferencePageNumber THEN debugReferencePageNumberCount ← debugReferencePageNumberCount + 1;
treeEntry ← NARROW[storage];
IF type = new
THEN {
IF YggFile.Info[treeEntry.file, treeEntry.tid].size < (number+2)*FilePagesPerBtreePage THEN YggFile.SetSize[treeEntry.file, (number+25)*FilePagesPerBtreePage, treeEntry.tid];
vMPageSet ← YggBuffMan.UsePages[fileHandle: treeEntry.file, tid: treeEntry.tid, pageRun: [firstPage: number+FilePagesPerBtreePage, count: FilePagesPerBtreePage]];
}
ELSE {
vMPageSet ← YggBuffMan.ReadPages[fileHandle: treeEntry.file, tid: treeEntry.tid, pageRun: [firstPage: number*FilePagesPerBtreePage+FilePagesPerBtreePage, count: FilePagesPerBtreePage]];
};
IF vMPageSet = NIL OR vMPageSet.rest # NIL THEN ERROR;
addVMPageSet[];
RETURN[vMPageSet.first.buffer];
};
BTreeReleasePage:
PUBLIC BTree.ReleasePage = {
PROC [storage: PageStorage, number: PageNumber, update: UpdateState]
removeVMPageSet:
ENTRY
PROC = {
prevlovmps: LIST OF YggBuffMan.VMPageSet ← NIL;
FOR lovmps:
LIST
OF YggBuffMan.VMPageSet ← treeEntry.currentVMPageSets, lovmps.rest
UNTIL lovmps =
NIL
DO
IF lovmps.first.first.pageRun.firstPage = number*FilePagesPerBtreePage+FilePagesPerBtreePage
THEN {
IF prevlovmps = NIL THEN treeEntry.currentVMPageSets ← lovmps.rest
ELSE prevlovmps.rest ← lovmps.rest;
EXIT;
};
prevlovmps ← lovmps;
ENDLOOP;
};
debugReferencePageNumberCount: INT ← 0;
treeEntry: TreeEntry;
releaseState: YggBuffMan.ReleaseState;
vMPageSet: YggBuffMan.VMPageSet;
IF number = debugReferencePageNumber THEN debugReferencePageNumberCount ← debugReferencePageNumberCount + 1;
treeEntry ← NARROW[storage];
SELECT update
FROM
startOfUpdate => {
releaseState ← writeBatchedNoWait;
};
endOfUpdate => {
releaseState ← writeBatchedNoWait;
};
unchanged => {
releaseState ← clean;
};
ENDCASE;
YggBuffMan.ReleaseVMPageSet[vMPageSet: vMPageSet, releaseState: releaseState,
keep: TRUE];
};
Internal red black procs
noteUseInTransaction:
ENTRY
PROC [doc: Document, tid: Camelot.tidT] ~ {
data: RedBlackTree.UserData;
scratchTIDObj.btid ← tid.top;
data ← RedBlackTree.Lookup[ transactionToModifiedDocumentMap, scratchTIDObj];
IF data =
NIL
THEN {
dt: DocTrans ← NEW[DocTransRep];
dt.btidForDoc ← NEW[TIDObjRep ← [tid.top]];
dt.docs ← LIST[doc];
RedBlackTree.Insert[transactionToModifiedDocumentMap, dt, scratchTIDObj];
}
ELSE {
dt: DocTrans;
dt ← NARROW[data];
FOR docList:
LIST
OF Document ← dt.docs, docList.rest
UNTIL docList =
NIL
DO
IF YggDID.EqualDIDs[docList.first.did, doc.did] THEN RETURN;
ENDLOOP;
dt.docs ← CONS[doc, dt.docs];
};
};
GetKeyProc: RedBlackTree.GetKey = {
PROC [data: UserData] RETURNS [Key]
docTrans: DocTrans ← NARROW[data];
RETURN[ docTrans.btidForDoc ];
};
CompareProc: RedBlackTree.Compare = {
PROC [k: Key, data: UserData] RETURNS [Basics.Comparison]
fileTransData: DocTrans ← NARROW[data];
keyData: TIDObj ← NARROW[k];
SELECT keyData.highTicker
FROM
> fileTransData.btidForDoc.highTicker => RETURN [greater];
< fileTransData.btidForDoc.highTicker => RETURN [less];
ENDCASE => {
SELECT keyData.lowTicker
FROM
> fileTransData.btidForDoc.lowTicker => RETURN [greater];
< fileTransData.btidForDoc.lowTicker => RETURN [less];
ENDCASE => RETURN [equal];
};
};
stockTIDObj: TIDObj ← NIL;
CheckoutTIDObj:
ENTRY
PROC [btid: Camelot.btidT]
RETURNS [k: TIDObj] = {
IF stockTIDObj = NIL THEN k ← NEW [TIDObjRep] ELSE { k ← stockTIDObj; stockTIDObj ← NIL };
k.btid ← btid;
};
RecycleTIDObj:
ENTRY
PROC [k: TIDObj] = {
stockTIDObj ← k;
};
Old enumeration proc
doc = NIL starts a new enumeration, and GetNext returns nextDocument = NIL when the enumeration is exhausted. The only guaranteed property of the enumeration is that all documents in existence during the entire enumeration will be visited at least once. Some documents may be seen more than once.
GetNext:
PUBLIC
--ENTRY--
PROCEDURE[doc: Document]
RETURNS [nextDocument: Document] =
BEGIN -- errors defined in FileMap: none.
ENABLE UNWIND => NULL;
RETURN[EnumerateNext[doc]];
END;
Hash table management:
Explanation of client-supplied parameters:
The procedure ClientHashInit is called during hash table initialization, to allow the hash function to precompute values based on the range and to make any small adjustments to the range that are necessary.
HashDocument must:
be a REF type.
contain a field "next" of type HashDocument, under the exclusive control of the hash package.
Key is an arbitrary type.
SetKey sets the "key value" associated with a HashDocument. The key value must not change between the time the doc is Inserted into the table and the time it is deleted from the table.
Hash must be a function of the key value of the parameter "hashDocument".
EqualKeys must be the equality relation on the key values of the parameters "hashDocument1" and "hashDocument2".
Interface description:
InitializeHashTable: INTERNAL PROCEDURE[numHashSlotsDesired: NAT, hashTableZone:
ZONE, hashDocument: HashDocument];
errors: HashPkgCallerProgrammingError (numHashSlotsDesired = 0).
Insert: ENTRY PROCEDURE[hashDocument: HashDocument];
errors: HashPkgDuplicateKey.
LookupInCache: ENTRY PROCEDURE[key: Key] RETURNS [hashDocument: HashDocument];
returns hashDocument = NIL if not found.
Delete: ENTRY PROCEDURE[hashDocument: HashDocument];
errors: HashPkgCallerProgrammingError (not found).
EnumerateNext: INTERNAL PROCEDURE[prevHashDocument: HashDocument] RETURNS
[hashDocument: HashDocument];
errors: none.
prevHashDocument = NIL starts the enumeration, returned hashDocument = NIL is the end
of the enumeration. This procedure guarantees that any hashDocument in existence throughout
the entire enumeration will be seen. Other docs may or not not be seen. HashDocuments
may be seen more than once.
EnumerateWithProc: INTERNAL PROCEDURE[proc: PROCEDURE[hashDocument: HashDocument]
RETURNS[stop: BOOLEAN]];
errors: none.
client-supplied parameters to hash package begin here:
HashDocument: TYPE = Document;
Key: TYPE = YggDID.DID;
ClientHashInit:
INTERNAL
PROCEDURE[numHashSlotsDesired:
NAT]
RETURNS
[numHashSlotsAllowed:
NAT] =
BEGIN
divisor, quotient, remainder: CARDINAL;
DO divisor ← 2;
DO
[quotient, remainder] ← Basics.DivMod[numHashSlotsDesired, divisor];
quotient ← numHashSlotsDesired / divisor;
remainder ← numHashSlotsDesired MOD divisor;
IF remainder = 0 THEN EXIT; -- this number is not prime.
IF quotient < divisor THEN RETURN[numHashSlotsDesired]; -- prime.
divisor ← divisor + 1;
ENDLOOP;
numHashSlotsDesired ← numHashSlotsDesired + 1;
ENDLOOP;
END;
ClientHash:
INTERNAL
PROCEDURE[hashDocument: HashDocument]
RETURNS [index:
NAT
--[0..numHashSlots)--] =
INLINE {
index ← YggDID.HashDID[hashDocument.did] MOD numHashSlots;
ClientEqualKeys:
INTERNAL
PROCEDURE[hashDocument1, hashDocument2: HashDocument]
RETURNS [equal:
BOOLEAN] =
INLINE {
RETURN[YggDID.EqualDIDs[hashDocument1.did, hashDocument2.did]];
};
end of client-supplied parameters, start of invariant hash package code:
The INTERNAL procedures below expect to be called from a client procedure holding the module monitor lock, which protects the following data structures:
hashSlots: REF HashSlots ← NIL;
HashSlots: TYPE = RECORD[SEQUENCE nSlots: NAT OF HashDocument];
numHashSlots: NAT ← 0; -- boy, will they be sorry if they don't init this package.
lookupHashDocument: HashDocument ← NIL; -- for the "package's" use only.
errors:
HashPkgCallerProgrammingError: ERROR = CODE; -- various fatal conditions.
HashPkgDuplicateKey: ERROR = CODE; -- from Insert.
InitializeHashTable:
INTERNAL
PROCEDURE[numHashSlotsDesired:
NAT, hashDocument: HashDocument] =
BEGIN
-- errors: HashPkgCallerProgrammingError (numHashSlotsDesired = 0).
numHashSlots ← ClientHashInit[numHashSlotsDesired];
IF numHashSlots = 0 THEN ERROR HashPkgCallerProgrammingError;
lookupHashDocument ← hashDocument;
hashSlots ← NEW[HashSlots[numHashSlots]];
FOR index:
NAT
IN [0..numHashSlots)
DO hashSlots[index] ← NIL; ENDLOOP;
END;
Insert:
ENTRY
PROC [hashDocument: HashDocument] = {
errors: HashPkgDuplicateKey.
index: NAT ← ClientHash[hashDocument];
FOR newHashDocument: HashDocument ← hashSlots[index], newHashDocument.next
UNTIL newHashDocument =
NIL
DO
IF ClientEqualKeys[newHashDocument, hashDocument]
THEN ERROR HashPkgDuplicateKey;
ENDLOOP;
hashDocument.next ← hashSlots[index];
hashSlots[index] ← hashDocument;
SafeStorage.EnableFinalization[hashDocument];
};
LookupInCache:
ENTRY
PROC [key: Key]
RETURNS [hashDocument: HashDocument] = {
returns hashDocument = NIL if not found.
lookupHashDocument.did ← key;
FOR hashDocument ← hashSlots[ClientHash[lookupHashDocument]], hashDocument.next
UNTIL hashDocument =
NIL
DO
IF ClientEqualKeys[hashDocument, lookupHashDocument] THEN RETURN;
ENDLOOP;
RETURN[NIL];
};
Delete:
ENTRY
PROC [hashDocument: HashDocument] = {
errors: HashPkgCallerProgrammingError (not found).
index: NAT ← ClientHash[hashDocument];
prevHashDocument: HashDocument ← NIL;
FOR newHashDocument: HashDocument ← hashSlots[index], newHashDocument.next
UNTIL newHashDocument =
NIL
DO
IF ClientEqualKeys[newHashDocument, hashDocument] THEN EXIT;
prevHashDocument ← newHashDocument;
REPEAT FINISHED => ERROR HashPkgCallerProgrammingError;
ENDLOOP;
IF prevHashDocument =
NIL
THEN hashSlots[index] ← hashDocument.next
ELSE prevHashDocument.next ← hashDocument.next;
};
prevHashDocument = NIL starts the enumeration, returned hashDocument = NIL is the end of the enumeration. This procedure guarantees that any hashDocument in existence throughout the entire enumeration will be seen. Other docs may or not not be seen. HashDocuments may be seen more than once.
EnumerateNext:
ENTRY
PROCEDURE[prevHashDocument: HashDocument]
RETURNS
[hashDocument: HashDocument] = {
errors: none.
index: NAT;
IF prevHashDocument = NIL THEN index ← 0
ELSE {
index ← ClientHash[prevHashDocument];
FOR hashDocument ← hashSlots[index], hashDocument.next
UNTIL hashDocument =
NIL
DO
IF ClientEqualKeys[hashDocument, prevHashDocument] THEN GOTO found;
REPEAT
found => {
IF hashDocument.next # NIL THEN RETURN[hashDocument.next];
index ← index + 1;
};
ENDLOOP;
};
UNTIL index >= numHashSlots
DO
IF hashSlots[index] # NIL THEN RETURN[hashSlots[index]];
index ← index + 1;
ENDLOOP;
RETURN[NIL];
};
EnumerateWithProc:
INTERNAL
PROCEDURE[proc:
PROCEDURE[hashDocument: HashDocument]
RETURNS[stop:
BOOLEAN]] = {
errors: none.
FOR index:
NAT
IN [0..numHashSlots)
DO
FOR hashDocument: HashDocument ← hashSlots[index], hashDocument.next
UNTIL hashDocument =
NIL
DO
IF proc[hashDocument] THEN RETURN;
ENDLOOP;
ENDLOOP;
};
end of invariant hash package code.
Finalization
FinalizationProcess:
PROCEDURE = {
FinalizeEntry:
ENTRY
PROCEDURE [docm: Document] = {
IF SafeStorage.IsFinalizationEnabled[docm] THEN RETURN;
Delete[docm];
docm ← NIL;
};
DO
doc: Document;
FinalizeEntry[doc ← NARROW[SafeStorage.FQNext[finalizationQ]]];
doc ← NIL;
ENDLOOP;
};
Initialization
Initialize:
PUBLIC
ENTRY
PROCEDURE[numHashSlotsDesired:
NAT, fQLength:
CARDINAL] = {
IF fQLength = 0 THEN RETURN WITH ERROR ZeroIllegal;
finalizationQ ← SafeStorage.NewFQ[fQLength]; -- do this before
SafeStorage.EstablishFinalization[CODE[DocumentRep], 1, finalizationQ]; -- allocating any FileObjects.
TRUSTED BEGIN Process.Detach[FORK FinalizationProcess[]]; END;
secondLookupHashDocument ← NEW[DocumentRep ← [did: NIL, componentFiles: NIL, interlock: FALSE, next: NIL]];
InitializeHashTable[numHashSlotsDesired, secondLookupHashDocument];
set up Btrees and NextDID
TRUSTED {Process.InitializeCondition[LOOPHOLE[@aShortTime], Process.MsecToTicks[50]]; };
transactionToModifiedDocumentMap ← RedBlackTree.Create[getKey: GetKeyProc, compare: CompareProc];
};
Initialize[129, 250];
END.