DIRECTORY Ascii USING [NUL], Basics USING [bytesPerWord, charsPerWord, Comparison, logBytesPerWord, RawBytes, UnsafeBlock], BasicTime USING [Period], BTree USING [DeleteKey, Entry, EntSize, EnumerateEntries, Key, New, Open, PageNumber, PagePtr, PageStorage, ReferencePage, ReleasePage, Tree, UpdateEntry], BTreeVM USING [CacheSize, FlushCache, FreeBuffers, GetStats, Handle, Open, ReferencePage, ReleasePage, Stats], IO USING [CreateStream, CreateStreamProcs, Flush, GetLength, SetIndex, STREAM, StreamProcs, UnsafeGetBlock, UnsafePutBlock], PBasics USING [ByteBlt], Process USING [Detach, MsecToTicks, Pause, SetTimeout, Ticks], RefTab USING [Create, Delete, EachPairAction, EqualProc, Fetch, GetSize, Insert, Pairs, Ref, Val], Rope USING [Fetch, Length, NewText, ROPE, Text, UnsafeMoveChars], SafeStorage USING [CantEstablishFinalization, EnableFinalization, EstablishFinalization, FinalizationQueue, FQNext, NewFQ], YggDID USING [CompareDIDs, DID, FirstDID, LastDID, SizeForDID, StabilizeDID, VolatilizeDID], YggDIDMap USING [GetComponentFiles, OpenDocumentFromDID], YggDIDPrivate USING [DIDRep], YggEnvironment USING [nullTransID, TransID], YggFileStream USING [StreamFromComponentFilesAndTid], YggIndex USING [EntryProc, Index, IndexRep], YggInternal USING [Document, FileHandle], YggInline USING [BytesForWords, WordsForBytes], YggRep USING [AccurateGMT, AccurateGMTRep, BitsRep, date, did, DocType, float, int, rope, shortRope, TypedPrimitiveElement, unknown], YggTransaction USING [EqualTrans, IsNullTrans]; YggIndexImpl: CEDAR MONITOR IMPORTS Basics, BasicTime, BTree, BTreeVM, IO, PBasics, Process, RefTab, Rope, SafeStorage, YggDID, YggDIDMap, YggFileStream, YggInline, YggTransaction EXPORTS YggDID, YggIndex ~ BEGIN ROPE: TYPE = Rope.ROPE; DID: PUBLIC TYPE ~ REF DIDRep; DIDRep: PUBLIC TYPE ~ YggDIDPrivate.DIDRep; Index: TYPE = YggIndex.Index; AllNulls: PACKED ARRAY [0..Basics.charsPerWord) OF CHAR _ ALL[0C]; bytesPerPage: INT _ 4096; MyCondition: CONDITION; Handle: TYPE = REF VMObject; VMObject: PUBLIC TYPE = MONITORED RECORD [ backingStream: IO.STREAM, btreevmHandle: BTreeVM.Handle, btreeHandle: BTree.Tree _ NIL, unused: BOOL _ FALSE ]; Cache: RefTab.Ref _ NIL; desiredSizeOfCache: INT _ 50; CacheEntry: TYPE = RECORD [ backingStream: IO.STREAM, -- stream with null tid fileUse: ATOM, tid: YggEnvironment.TransID _ YggEnvironment.nullTransID, currentWriteStream: IO.STREAM, -- stream with tid myHandle: Index, users: INT _ 0 -- -1 means in the process of being opened ]; scratchKey: REF CacheEntry _ NIL; Key: TYPE = REF KeyRep; KeyRep: TYPE = RECORD [ tpe: YggRep.TypedPrimitiveElement, did: YggDID.DID ]; varOffset: CARD = WORDS[YggDIDPrivate.DIDRep] + WORDS[YggRep.DocType]; EntryPtr: TYPE = LONG BASE POINTER TO Entry; Entry: TYPE = MACHINE DEPENDENT RECORD [ didWithValue(0): YggDIDPrivate.DIDRep, docType(WORDS[YggDIDPrivate.DIDRep]): YggRep.DocType, val(varOffset): SELECT OVERLAID * FROM int => [ int(varOffset): INT32 ], float => [ float(varOffset): REAL32 ], date => [ date(varOffset): YggRep.AccurateGMTRep ], shortRope => [ shortRope(varOffset): PACKED ARRAY [0..0) OF CHAR ], rope => [ ropeSize(varOffset): INT32, rope(varOffset+WORDS[INT32]): PACKED ARRAY [0..0) OF CHAR ], did => [ did(varOffset): YggDIDPrivate.DIDRep ], bits => [ bitsSize(varOffset): INT32, bits(varOffset+WORDS[INT]): PACKED ARRAY [0..0) OF BYTE ] ENDCASE ]; Open: PUBLIC PROC [did: YggDID.DID, fileUse: ATOM, cacheSize: BTreeVM.CacheSize, initialize: BOOL, trans: YggEnvironment.TransID] RETURNS [index: Index _ NIL] ~ { index _ OpenBTreeVM[did: did, fileUse: fileUse, bytesPerPage: bytesPerPage, cacheSize: cacheSize, base: 0, initialize: initialize]; }; Close: PUBLIC PROC [index: Index] = { CloseBTreeVM[index]; }; SimpleEntryProc: YggIndex.EntryProc = { -- for debugging purposes cont: BOOLEAN _ TRUE; RETURN[cont]; }; EnumerateEntries: PUBLIC PROC [index: Index, start: YggRep.TypedPrimitiveElement, end: YggRep.TypedPrimitiveElement, proc: YggIndex.EntryProc] ~ { enumProc: UNSAFE PROC [entry: BTree.Entry] RETURNS [continue: BOOL _ TRUE] = TRUSTED { entryPtr: EntryPtr = LOOPHOLE[entry]; value: YggRep.TypedPrimitiveElement; did: YggDID.DID; IF lastKey # NIL THEN { done: Basics.Comparison; done _ Compare[lastKey, entry]; IF done = less OR done = equal THEN RETURN[FALSE]; }; SELECT entryPtr.docType FROM YggRep.unknown => ERROR; YggRep.int =>{ value _ [YggRep.int, NEW[INT32 _ entryPtr.int]]; did _ YggDID.VolatilizeDID[buffer: @entryPtr.didWithValue]; }; YggRep.float => { value _ [YggRep.float, NEW[REAL32 _ entryPtr.float]]; did _ YggDID.VolatilizeDID[buffer: @entryPtr.didWithValue]; }; YggRep.date => { value _ [YggRep.date, NEW[YggRep.AccurateGMTRep _ entryPtr.date]]; did _ YggDID.VolatilizeDID[buffer: @entryPtr.didWithValue]; }; YggRep.shortRope => { ropeLen: INT; scratch: Rope.Text; FOR ropeLen IN [ 0 .. INT.LAST ) DO IF entryPtr.shortRope[ropeLen] = Ascii.NUL THEN EXIT; ENDLOOP; scratch _ Rope.NewText[ropeLen]; FOR charNo: INT IN [ 0 .. ropeLen ) DO TRUSTED {scratch[charNo] _ entryPtr.shortRope[charNo];}; ENDLOOP; ropeLen _ YggInline.BytesForWords[YggInline.WordsForBytes[ropeLen+1]]; value _ [YggRep.shortRope, NEW[ROPE _ scratch]]; did _ YggDID.VolatilizeDID[buffer: @entryPtr.didWithValue]; }; YggRep.rope => { ropeLen: CARD; scratch: Rope.Text; scratch _ Rope.NewText[entryPtr.ropeSize]; FOR charNo: INT IN [ 0 .. entryPtr.ropeSize ) DO TRUSTED {scratch[charNo] _ entryPtr.rope[charNo];}; ENDLOOP; ropeLen _ YggInline.BytesForWords[YggInline.WordsForBytes[entryPtr.ropeSize]]; value _ [YggRep.rope, NEW[ROPE _ scratch]]; did _ YggDID.VolatilizeDID[buffer: @entryPtr.didWithValue]; }; YggRep.did => { value _ [YggRep.did, YggDID.VolatilizeDID[buffer: @entryPtr.did]]; did _ YggDID.VolatilizeDID[buffer: @entryPtr.didWithValue]; }; ENDCASE => { nBytes: CARD; bitsLen: CARD; bits: REF YggRep.BitsRep; nextByteToRead: INT _ 0; bits _ NEW[YggRep.BitsRep[CARD16[entryPtr.bitsSize]]]; bits.validBytes _ entryPtr.bitsSize; TRUSTED {nBytes _ PBasics.ByteBlt[ from: [blockPointer: @entryPtr.bits, startIndex: 0, stopIndexPlusOne: entryPtr.bitsSize], to: [blockPointer: LOOPHOLE[bits, LONG POINTER] + SIZE[YggRep.BitsRep[0]], startIndex: 0, stopIndexPlusOne: entryPtr.bitsSize]]; }; IF nBytes # CARD[entryPtr.bitsSize] THEN ERROR; bitsLen _ YggInline.BytesForWords[YggInline.WordsForBytes[entryPtr.bitsSize]]; value _ [entryPtr.docType, bits]; did _ YggDID.VolatilizeDID[buffer: @entryPtr.didWithValue]; }; continue _ proc[value, did]; }; keyRef: Key _ NIL; lastKey: Key _ NIL; IF start # [YggRep.unknown, NIL] THEN { keyRef _ CheckoutKey[]; keyRef.tpe _ start; keyRef.did _ YggDID.FirstDID; }; IF end # [YggRep.unknown, NIL] THEN { lastKey _ CheckoutKey[]; lastKey.tpe _ end; lastKey.did _ YggDID.LastDID; }; TRUSTED {[] _ BTree.EnumerateEntries[tree: index.tree, relation: greater, key: keyRef, pathStk: NIL, useExistingPath: FALSE, Proc: enumProc];}; IF keyRef # NIL THEN RecycleKey[keyRef]; IF lastKey # NIL THEN RecycleKey[lastKey]; }; WriteEntry: PUBLIC PROC [index: Index, value: YggRep.TypedPrimitiveElement, did: YggDID.DID, replace: BOOLEAN _ FALSE, trans: YggEnvironment.TransID] RETURNS [indexFound: BOOL _ TRUE] ~ { writeEntryInner: UNSAFE PROCEDURE [entry: BTree.Entry] = UNCHECKED { entryPtr: EntryPtr = LOOPHOLE[entry]; IF firstTime THEN { myStream: MyStream; myStream _ NARROW[index.realStream.streamData]; myStream.tid _ trans; }; firstTime _ FALSE; entryPtr.docType _ value.docType; SELECT value.docType FROM YggRep.unknown => ERROR; YggRep.int =>{ ri: REF INT32; ri _ NARROW[value.bits]; entryPtr.int _ ri^; YggDID.StabilizeDID[did: did, buffer: @entryPtr.didWithValue]; }; YggRep.float => { rReal: REF REAL32; rReal _ NARROW[value.bits]; entryPtr.float _ rReal^; YggDID.StabilizeDID[did: did, buffer: @entryPtr.didWithValue]; }; YggRep.date => { rAccurateGMT: YggRep.AccurateGMT; rAccurateGMT _ NARROW[value.bits]; entryPtr.date _ rAccurateGMT^; YggDID.StabilizeDID[did: did, buffer: @entryPtr.didWithValue]; }; YggRep.shortRope => { rRope: ROPE; size: INT; nBytes: INT; nullsToMove: INT; rRope _ NARROW[value.bits]; size _ Rope.Length[rRope]; TRUSTED {nBytes _ Rope.UnsafeMoveChars[block: [LOOPHOLE[@entryPtr.shortRope], 0, size], rope: rRope, start: 0];}; IF nBytes # size THEN ERROR; nullsToMove _ YggInline.BytesForWords[YggInline.WordsForBytes[size+1]] - size; IF nullsToMove <= 0 THEN ERROR; TRUSTED {nBytes _ PBasics.ByteBlt[ from: [blockPointer: @AllNulls, startIndex: 0, stopIndexPlusOne: nullsToMove], to: [blockPointer: @entryPtr.shortRope, startIndex: size, stopIndexPlusOne: size+nullsToMove]]; }; IF nBytes # nullsToMove THEN ERROR; YggDID.StabilizeDID[did: did, buffer: @entryPtr.didWithValue]; }; YggRep.rope => { rRope: ROPE; nBytes: INT; size: INT; nullsToMove: INT; rRope _ NARROW[value.bits]; size _ Rope.Length[rRope]; entryPtr.ropeSize _ size; TRUSTED {nBytes _ Rope.UnsafeMoveChars[block: [LOOPHOLE[@entryPtr.rope], 0, size], rope: rRope, start: 0];}; IF nBytes # size THEN ERROR; nullsToMove _ YggInline.BytesForWords[YggInline.WordsForBytes[size]] - size; IF nullsToMove > 0 THEN TRUSTED { nBytes _ PBasics.ByteBlt [ from: [blockPointer: @AllNulls, startIndex: 0, stopIndexPlusOne: nullsToMove], to: [blockPointer: @entryPtr.rope, startIndex: size, stopIndexPlusOne: size+nullsToMove]]; IF nBytes # nullsToMove THEN ERROR; }; YggDID.StabilizeDID[did: did, buffer: @entryPtr.didWithValue]; }; YggRep.did => { rDID: DID; rDID _ NARROW[value.bits]; YggDID.StabilizeDID[did: rDID, buffer: @entryPtr.did]; YggDID.StabilizeDID[did: did, buffer: @entryPtr.didWithValue]; }; ENDCASE => { rBits: REF YggRep.BitsRep; nullsToMove: INT; nBytes: INT; rBits _ NARROW[value.bits]; entryPtr.ropeSize _ rBits.validBytes; TRUSTED {nBytes _ PBasics.ByteBlt [ from: [blockPointer: LOOPHOLE[rBits, LONG POINTER TO Basics.RawBytes] + SIZE[YggRep.BitsRep[0]] , startIndex: 0, stopIndexPlusOne: rBits.validBytes], to: [blockPointer: @entryPtr.bits, startIndex: 0, stopIndexPlusOne: rBits.validBytes]];}; IF nBytes # INT[rBits.validBytes] THEN ERROR; nullsToMove _ YggInline.BytesForWords[YggInline.WordsForBytes[rBits.validBytes]] - rBits.validBytes; IF nullsToMove > 0 THEN TRUSTED { nBytes _ PBasics.ByteBlt [ from: [blockPointer: @AllNulls, startIndex: 0, stopIndexPlusOne: nullsToMove], to: [blockPointer: @entryPtr.bits, startIndex: rBits.validBytes, stopIndexPlusOne: rBits.validBytes+nullsToMove]]; IF nBytes # nullsToMove THEN ERROR; }; YggDID.StabilizeDID[did: did, buffer: @entryPtr.didWithValue]; }; }; firstTime: BOOL _ TRUE; keyRef: Key _ NIL; entSize: INT; keyRef _ CheckoutKey[]; keyRef.tpe _ value; keyRef.did _ did; entSize _ SIZE[YggRep.DocType] + ValueSize[value] + YggDID.SizeForDID[did]; TRUSTED {BTree.UpdateEntry[tree: index.tree, key: keyRef, pathStk: NIL, useExistingPath: FALSE, words: entSize, Proc: writeEntryInner, updateType: insertOrReplace];}; RecycleKey[keyRef]; }; DeleteEntry: PUBLIC PROC [index: Index, value: YggRep.TypedPrimitiveElement, did: YggDID.DID, trans: YggEnvironment.TransID] RETURNS [found: BOOLEAN] ~ { myStream: MyStream; keyRef: Key _ NIL; keyRef _ CheckoutKey[]; keyRef.tpe _ value; keyRef.did _ did; myStream _ NARROW[index.realStream.streamData]; myStream.tid _ trans; found _ BTree.DeleteKey[tree: index.tree, key: keyRef, pathStk: NIL, useExistingPath: FALSE]; RecycleKey[keyRef]; }; OpenBTreeVM: PROC [ did: YggDID.DID, fileUse: ATOM, bytesPerPage: INT, cacheSize: BTreeVM.CacheSize, base: INT _ 0, initialize: BOOL, trans: YggEnvironment.TransID _ YggEnvironment.nullTransID ] RETURNS [ h: YggIndex.Index _ NIL] = { btreevmHandle: BTreeVM.Handle _ NIL; ce: REF CacheEntry; innerOpen: ENTRY PROC = { found: BOOL; val: RefTab.Val; scratchKey.backingStream _ backingStream; [found, val] _ RefTab.Fetch[x: Cache, key: scratchKey]; IF found THEN { ce _ NARROW[val]; h _ ce.myHandle; WHILE ce.users = -1 DO WAIT MyCondition; ENDLOOP; ce.users _ ce.users + 1; IF h.unused THEN { h.unused _ FALSE; SafeStorage.EnableFinalization[h]; }; } ELSE { ce _ NEW[CacheEntry _ [backingStream, fileUse, trans, NIL, NIL, -1]]; IF ~RefTab.Insert[x: Cache, key: Cache, val: ce] THEN ERROR; IF ~YggTransaction.IsNullTrans[trans] THEN { ce.currentWriteStream _ YggFileStream.StreamFromComponentFilesAndTid[componentFiles: componentFiles, fileUse: fileUse, tid: trans]; }; }; }; doc: YggInternal.Document _ NIL; componentFiles: LIST OF YggInternal.FileHandle; backingStream: IO.STREAM; doc _ YggDIDMap.OpenDocumentFromDID[did, YggEnvironment.nullTransID]; componentFiles _ YggDIDMap.GetComponentFiles[doc]; backingStream _ YggFileStream.StreamFromComponentFilesAndTid[componentFiles: componentFiles, fileUse: fileUse, tid: YggEnvironment.nullTransID]; innerOpen[]; IF h = NIL THEN { opened: ENTRY PROC = { ce.users _ 1; ce.myHandle _ h; BROADCAST MyCondition; }; tree: BTree.Tree; realStream: IO.STREAM; realStream _ NewStreamForBTreeVM[componentFiles, fileUse]; btreevmHandle _ BTreeVM.Open[realStream, bytesPerPage, cacheSize, base]; h _ NEW[YggIndex.IndexRep]; h.btreevmHandle _ btreevmHandle; h.realStream _ realStream; h.backingStream _ backingStream; tree _ BTree.New[ repPrim: [compare: Compare, entrySize: EntrySize], storPrim: [referencePage: BTreeVM.ReferencePage, releasePage: BTreeVM.ReleasePage], minEntrySize: SIZE[YggRep.DocType] + SIZE[CARD], initialState: suspended ]; BTree.Open[ tree: tree, storage: btreevmHandle, pageSize: bytesPerPage, initialize: initialize, maintainRecomputableState: TRUE ]; h.tree _ tree; opened[]; SafeStorage.EnableFinalization[h]; }; }; CloseBTreeVM: ENTRY PROC [ h: Index ] = { ce: REF CacheEntry; found: BOOL; val: RefTab.Val; scratchKey.backingStream _ h.backingStream; [found, val] _ RefTab.Fetch[x: Cache, key: scratchKey]; IF found THEN { ce _ NARROW[val]; ce.users _ ce.users - 1; }; }; Compare: UNSAFE PROC [key: BTree.Key, entry: BTree.Entry] RETURNS [Basics.Comparison _ equal] = UNCHECKED { keyRef: Key = NARROW[key]; entryPtr: EntryPtr = LOOPHOLE[entry]; IF keyRef.tpe.docType = entryPtr.docType THEN { SELECT keyRef.tpe.docType FROM YggRep.unknown => ERROR; YggRep.int => { ri: REF INT32; ri _ NARROW[keyRef.tpe.bits]; IF ri^ < entryPtr.int THEN RETURN[less]; IF ri^ > entryPtr.int THEN RETURN[greater]; RETURN [YggDID.CompareDIDs[keyRef.did, @entryPtr.didWithValue]]; }; YggRep.float => { rReal: REF REAL32; rReal _ NARROW[keyRef.tpe.bits]; IF rReal^ < entryPtr.float THEN RETURN[less]; IF rReal^ > entryPtr.float THEN RETURN[greater]; RETURN [YggDID.CompareDIDs[keyRef.did, @entryPtr.didWithValue]]; }; YggRep.date => { rAccurateGMT: YggRep.AccurateGMT; period: INT; rAccurateGMT _ NARROW[keyRef.tpe.bits]; period _ BasicTime.Period[rAccurateGMT.gmt, entryPtr.date.gmt]; IF period < 0 THEN RETURN[greater]; IF period > 0 THEN RETURN[less]; IF rAccurateGMT.usecs < entryPtr.date.usecs THEN RETURN[less]; IF rAccurateGMT.usecs > entryPtr.date.usecs THEN RETURN[greater]; RETURN [YggDID.CompareDIDs[keyRef.did, @entryPtr.didWithValue]]; }; YggRep.shortRope => { rRope: ROPE; lenKey: CARDINAL; rRope _ NARROW[keyRef.tpe.bits]; lenKey _ Rope.Length[rRope]; FOR i: CARDINAL IN [ 0 .. lenKey ) DO keyC: CHAR = Rope.Fetch[rRope, i]; entryC: CHAR = entryPtr.shortRope[i]; IF keyC = entryC THEN LOOP; -- most probable case IF entryC = Ascii.NUL THEN RETURN [greater]; SELECT keyC FROM < entryC => RETURN [less]; > entryC => RETURN [greater]; ENDCASE; ENDLOOP; IF entryPtr.shortRope[lenKey] = Ascii.NUL THEN { ropeLen: CARD; ropeLen _ YggInline.BytesForWords[YggInline.WordsForBytes[lenKey+1]]; RETURN [YggDID.CompareDIDs[keyRef.did, @entryPtr.didWithValue]]; } ELSE RETURN[less]; }; YggRep.rope => { rRope: ROPE; lenKey: CARD32; rRope _ NARROW[keyRef.tpe.bits]; lenKey _ Rope.Length[rRope]; FOR i: CARD32 IN [ 0 .. MIN[lenKey, CARD32[entryPtr.ropeSize]] ) DO keyC: CHAR = Rope.Fetch[rRope, i]; entryC: CHAR = entryPtr.rope[i]; IF keyC = entryC THEN LOOP; -- most probable case SELECT keyC FROM < entryC => RETURN [less]; > entryC => RETURN [greater]; ENDCASE; ENDLOOP; SELECT CARD32[entryPtr.ropeSize] FROM < lenKey => RETURN [greater]; > lenKey => RETURN [less]; ENDCASE => { ropeLen: CARD; ropeLen _ YggInline.BytesForWords[YggInline.WordsForBytes[entryPtr.ropeSize]]; RETURN [YggDID.CompareDIDs[keyRef.did, @entryPtr.didWithValue]]; }; }; YggRep.did => { comp: Basics.Comparison _ equal; rDID: DID; rDID _ NARROW[keyRef.tpe.bits]; comp _ YggDID.CompareDIDs[keyRef.did, @entryPtr.did]; IF comp # equal THEN RETURN[comp]; RETURN [YggDID.CompareDIDs[keyRef.did, @entryPtr.didWithValue]]; }; ENDCASE => { rBits: REF YggRep.BitsRep; rBits _ NARROW[keyRef.tpe.bits]; FOR i: CARD32 IN [ 0 .. MIN[rBits.validBytes, CARD32[entryPtr.bitsSize]] ) DO keyC: BYTE = rBits.b[i]; entryC: BYTE = entryPtr.bits[i]; IF keyC = entryC THEN LOOP; -- most probable case SELECT keyC FROM < entryC => RETURN [less]; > entryC => RETURN [greater]; ENDCASE; ENDLOOP; SELECT entryPtr.bitsSize FROM < rBits.length => RETURN [greater]; > rBits.length => RETURN [less]; ENDCASE => { bitsLen: CARD; bitsLen _ YggInline.BytesForWords[YggInline.WordsForBytes[entryPtr.bitsSize]]; RETURN [YggDID.CompareDIDs[keyRef.did, @entryPtr.didWithValue]]; }; }; }; IF keyRef.tpe.docType < entryPtr.docType THEN RETURN [less]; RETURN [greater]; }; EntrySize: UNSAFE PROC [entry: BTree.Entry] RETURNS [words: BTree.EntSize _ 4] = UNCHECKED { entryPtr: EntryPtr = LOOPHOLE[entry]; SELECT entryPtr.docType FROM YggRep.unknown => ERROR; YggRep.int =>{ RETURN [WORDS[Entry.int]]; }; YggRep.float => { RETURN [WORDS[Entry.float]]; }; YggRep.date => { RETURN [WORDS[Entry.date]]; }; YggRep.shortRope => { lenKey: CARD; FOR lenKey IN [ 0 .. CARD.LAST ) DO IF entryPtr.shortRope[lenKey] = Ascii.NUL THEN EXIT; ENDLOOP; RETURN [WORDS[Entry.shortRope] + YggInline.WordsForBytes[lenKey+1]]; }; YggRep.rope => { RETURN [WORDS[Entry.rope] + YggInline.WordsForBytes[entryPtr.ropeSize]]; }; YggRep.date => { RETURN [WORDS[Entry.did]]; }; ENDCASE => { RETURN [WORDS[Entry.bits] + YggInline.WordsForBytes[entryPtr.bitsSize]]; }; }; ValueSize: PROC [value: YggRep.TypedPrimitiveElement] RETURNS [words: BTree.EntSize _ 4] = { wordsForValue: INT; SELECT value.docType FROM YggRep.unknown => ERROR; YggRep.int => wordsForValue _ WORDS[INT32]; YggRep.shortRope, YggRep.rope => { bytesForValue: INT; rRope: ROPE; rRope _ NARROW[value.bits]; bytesForValue _ Rope.Length[rRope]; wordsForValue _ YggInline.WordsForBytes[bytesForValue]; }; YggRep.float => wordsForValue _ WORDS[REAL32]; YggRep.date => wordsForValue _ WORDS[YggRep.AccurateGMTRep]; YggRep.did => wordsForValue _ WORDS[YggDIDPrivate.DIDRep]; ENDCASE => { bytesForValue: INT; rBits: REF YggRep.BitsRep; rBits _ NARROW[value.bits]; bytesForValue _ rBits.length; wordsForValue _ YggInline.WordsForBytes[bytesForValue]; }; words _ WORDS[YggRep.DocType] + wordsForValue; }; GetStats: PROC [ h: Index ] RETURNS [hits, misses, reads, writes, cumChainLength, cumReadWriteTime: LONG CARDINAL] = { stats: REF BTreeVM.Stats; stats _ NEW[BTreeVM.Stats]; BTreeVM.GetStats[h.btreevmHandle, stats]; hits _ stats.hits; misses _ stats.misses; reads _ stats.reads; writes _ stats.writes; cumChainLength _ stats.cumChainLength; cumReadWriteTime _ stats.cumReadWriteTime; }; FlushCache: PROC [ h: Index ] = { BTreeVM.FlushCache[h.btreevmHandle]; }; FreeBuffers: PROC [ h: Index ] = { BTreeVM.FreeBuffers[h.btreevmHandle]; }; SizeOfCache: PROC [ size: INT ] = { desiredSizeOfCache _ size; }; myStreamProcs: REF IO.StreamProcs _ IO.CreateStreamProcs[variety: inputOutput, class: $YggdrasilIndex, unsafeGetBlock: myUnsafeGetBlock, unsafePutBlock: myUnsafePutBlock, flush: myFlush, getLength: myGetLength, setIndex: mySetIndex]; MyStream: TYPE = REF MyStreamRep; MyStreamRep: TYPE = RECORD [ tid: YggEnvironment.TransID, componentFiles: LIST OF YggInternal.FileHandle, fileUse: ATOM, backingStream: IO.STREAM, currentWriteStream: IO.STREAM ]; NewStreamForBTreeVM: PROC [componentFiles: LIST OF YggInternal.FileHandle, fileUse: ATOM] RETURNS [stream: IO.STREAM] = { backingStream: IO.STREAM _ NIL; backingStream _ YggFileStream.StreamFromComponentFilesAndTid[componentFiles: componentFiles, fileUse: fileUse, tid: YggEnvironment.nullTransID]; stream _ IO.CreateStream[streamProcs: myStreamProcs, streamData: NEW[MyStreamRep _ [tid: YggEnvironment.nullTransID, componentFiles: componentFiles, backingStream: backingStream, currentWriteStream: NIL]], backingStream: NIL]; }; NewTransForStream: PROC [trans: YggEnvironment.TransID, backingStream: IO.STREAM] = { myStream: MyStream = NARROW[backingStream.streamData]; IF ~YggTransaction.EqualTrans[trans, myStream.tid] THEN { myStream.tid _ trans; myStream.currentWriteStream _ NIL; }; }; myUnsafeGetBlock: UNSAFE PROC [self: IO.STREAM, block: Basics.UnsafeBlock] RETURNS [nBytesRead: INT] ~ TRUSTED { myStream: MyStream = NARROW[self.streamData]; nBytesRead _ IO.UnsafeGetBlock[myStream.backingStream, block]; }; myUnsafePutBlock: PROC [self: IO.STREAM, block: Basics.UnsafeBlock] ~ { myStream: MyStream = NARROW[self.streamData]; IF myStream.currentWriteStream = NIL THEN { myStream.currentWriteStream _ YggFileStream.StreamFromComponentFilesAndTid[componentFiles: myStream.componentFiles, fileUse: myStream.fileUse, tid: myStream.tid]; }; IO.UnsafePutBlock[myStream.currentWriteStream, block]; }; myFlush: PROC [self: IO.STREAM] ~ { myStream: MyStream = NARROW[self.streamData]; IF myStream.currentWriteStream = NIL THEN { myStream.currentWriteStream _ YggFileStream.StreamFromComponentFilesAndTid[componentFiles: myStream.componentFiles, fileUse: myStream.fileUse, tid: myStream.tid]; }; IO.Flush[myStream.currentWriteStream]; }; myGetLength: PROC [self: IO.STREAM] RETURNS [length: INT] ~ { myStream: MyStream = NARROW[self.streamData]; length _ IO.GetLength[myStream.backingStream]; }; mySetIndex: PROC [self: IO.STREAM, index: INT] ~ { myStream: MyStream = NARROW[self.streamData]; IO.SetIndex[self: myStream.backingStream, index: index]; IF myStream.currentWriteStream # NIL THEN IO.SetIndex[self: myStream.currentWriteStream, index: index]; }; stockKeys: ARRAY [0..numberOfStockKeys] OF Key _ ALL[NIL]; numberOfStockKeys: INT = 10; stockKeyIndex: INT _ 0; CheckoutKey: ENTRY PROC RETURNS [k: Key] = { IF stockKeyIndex = 0 THEN k _ NEW [KeyRep] ELSE { stockKeyIndex _ stockKeyIndex - 1; k _ stockKeys[stockKeyIndex]; }; }; RecycleKey: ENTRY PROC [k: Key] = { IF stockKeyIndex < numberOfStockKeys THEN { stockKeys[stockKeyIndex] _ k; stockKeyIndex _ stockKeyIndex + 1; }; }; equalProc: RefTab.EqualProc = { k1: REF CacheEntry; k2: REF CacheEntry; k1 _ NARROW[key1]; k2 _ NARROW[key2]; RETURN [k1.backingStream = k2.backingStream]; }; IndexCacheTrimProcess: PROC = { ticksToWait: Process.Ticks; ticksToWait _ Process.MsecToTicks[222]; DO innerTrim: ENTRY PROC = { eachPairAction: RefTab.EachPairAction = { ce: REF CacheEntry; ce _ NARROW[val]; IF ce.myHandle.unused THEN [] _ RefTab.Delete[x: Cache, key: ce]; size _ size - 1; IF size <= desiredSizeOfCache THEN quit _ TRUE; }; size: INT; size _ RefTab.GetSize[Cache]; [] _ RefTab.Pairs[x: Cache, action: eachPairAction]; }; Process.Pause[ticksToWait]; IF RefTab.GetSize[Cache] > desiredSizeOfCache THEN innerTrim[]; ENDLOOP; }; FinalizationProcess: PROC = { DO fpInner: ENTRY PROC = { h.unused _ TRUE; }; h: Index _ NARROW [SafeStorage.FQNext[finalizationQueue]]; fpInner[]; h _ NIL; ENDLOOP; }; finalizationQueue: SafeStorage.FinalizationQueue _ SafeStorage.NewFQ[]; SafeStorage.EstablishFinalization[type: YggIndex.IndexRep.CODE, npr: 1, fq: finalizationQueue ! SafeStorage.CantEstablishFinalization => CONTINUE]; TRUSTED { Process.Detach[FORK FinalizationProcess]; Process.Detach[FORK IndexCacheTrimProcess]; Process.SetTimeout[condition: @MyCondition, ticks: Process.MsecToTicks[153]]; }; Cache _ RefTab.Create[mod: 47, equal: equalProc, hash: NIL]; scratchKey _ NEW[CacheEntry]; END. 8YggIndexImpl.mesa Copyright Σ 1988, 1989 by Xerox Corporation. All rights reserved. Bob Hagmann January 6, 1989 3:21:13 pm PST Cache to support the Yggdrasil BTree package for indices. This package caches BTreeVM objects and provides transparent access to BTreeVM. It also provides all the indexing functions. Types and constants An entry in a BTree has the following format (32-bit words assumed): The DID. A one word type (YggRep.DocType) The "value" indexed. This is stored in the following type dependant manner: integer, float: one word with the data date: two words with the data short rope (short and contains no nulls): then null terminated string written into just enough words rope (long and may have nulls) and uninterpreted bytes: size word containing the number of characters/bytes, and enough words to contain the characters/bytes DID: the did. The DID. For the Phase 0 implementation, this is a null terminated file name. Exported procedures Initiates indexing activity. This can be called any number of times to get a new Index handle or reopen an index that has been closed. Terminates indexing activity. [value: YggRep.TypedPrimitiveElement, did: YggDID.DID] RETURNS [continue: BOOL] set breakpoint here to look at entry Calls `proc' for each entry in the specified range of key values. The enumeration is halted when either the range of entries is exhausted or `proc' returns FALSE. An [unknown, NIL] value for `start' represents the least value, while an [unknown, NIL] value for `end' represents the largest value. Thus, the complete index can be enumerated by specifying start=[unknown, NIL] and end=[unknown, NIL]. Adds a new entry to the index. Deletes the entry that contains the given attribute value for the given key. BTreeVM related procedures Done with the current use of the BTree. [hits, misses, reads, writes, cumChainLength, cumReadWriteTime] _ BTreeVM.GetStats[h.btreevmHandle]; My streams Utilities Equals PROC [key1, key2: Key] RETURNS [BOOL]; Initialization, cache trim, and finalization PROC [key: Key, val: Val] RETURNS [quit: BOOL _ FALSE]; Κ$˜codešœ™KšœB™BK™*—K™K™KšœΈ™ΈK™K™šΟk ˜ Kšœœœ˜KšœœR˜^Kšœ œ ˜Kšœœ˜›Kšœœa˜nKš œœ!œœ œ œ-˜|Kšœœ ˜Kšœœ1˜>KšœœV˜bKšœœœ˜BKšœ œj˜{Kšœœœ>˜\Kšœ œ*˜9Kšœœ ˜Kšœœ˜,Kšœœ"˜5Kšœ œ˜,Kšœ œ˜)Kšœ œ ˜/Kšœœy˜…Kšœœ˜/—K˜KšΠbl œœ˜Kšœ˜—Kšœ˜šœ˜K˜—head™Icode0šœœœ˜K˜Kšœ œœ˜Kšœœœ˜+K˜Mšœœ˜M˜Kš œ œœœœœ˜BK˜Mšœœ˜Icode2šœ  œ˜Nšœœœ ˜š œ œœ œœ˜*Nšœ˜Nšœ˜Nšœœ˜Nšœ ˜Nšœ˜—Nšœœ˜Nšœœ˜šœ œœ˜NšœœœΟc˜2Nšœ œ˜Nšœ9˜9NšœœœŸ˜2Nšœ˜NšœœŸ*˜:Nšœ˜—Nšœ œ˜!Nšœœœ˜šœœœ˜Nšœ"˜"Nšœ˜Nšœ˜—™DN™Nšœ ™ ™LNšœ&™&Nšœ™N™dN™N™ —N™N—N˜Nšœ œœœ˜FNš œ œœœœœ˜,š œœœ œœ˜(Kšœ&˜&Kšœœ(˜5šœœœ˜&šœ˜Kšœ˜Kšœ˜—šœ ˜ Kšœ˜Kšœ˜—šœ ˜ Kšœ&˜&Kšœ˜—šœ˜Kšœœœœ˜1Kšœ˜—šœ ˜ Kšœœ˜Kš œœœ œœ˜9Kšœ˜—šœ˜Kšœ$˜$K˜—šœ ˜ Kšœœ˜Kš œœœ œœ˜7K˜—Kš˜—Kšœ˜——šœ™M˜šΟnœœœ œ œ,œœœ˜’Kšœ‡™‡Kšœƒ˜ƒKšœ˜—K˜K˜š œœœ˜%K™Kšœ˜K˜K™—K˜•StartOfExpansion6 -- [entry: IndexedLog.Entry] RETURNS [continue: BOOL]šΟbœŸ˜AKšΠckœ1’™OKšœœœ˜Kšœ$™$Kšœ˜ K˜—K˜š œœœv˜“Kš œœœCœzœœ™‘š‘œœœœ œœœ˜VNšœœ˜%Nšœ$˜$Nšœ œ˜šœ œœ˜Nšœ˜Nšœ˜Nš œ œœœœ˜2N˜—šœ˜Kšœœ˜šœ˜Kšœœœ˜0Kšœ;˜;K˜—šœ˜Kšœœœ˜5Kšœ;˜;K˜—šœ˜Kšœœ)˜BKšœ;˜;K˜—šœ˜Nšœ œ˜ Jšœ˜š œ œœœ˜#Nšœ%œœœ˜5Nšœ˜—Kšœ ˜ šœ œœ˜&Nšœ1˜8Nšœ˜—KšœF˜FKšœœœ ˜0Kšœ;˜;K˜—šœ˜Nšœ œ˜Jšœ˜Kšœ*˜*šœ œœ˜0Nšœ,˜3Nšœ˜—KšœN˜NKšœœœ ˜+Kšœ;˜;K˜—šœ˜KšœB˜BKšœ;˜;K˜—šœ˜ Kšœœ˜ Nšœ œ˜Kšœ˜Kšœœ˜Kšœœœ˜6Kšœ$˜$šœ˜"KšœZ˜ZKšœœ œœJ˜€K˜—Kšœ"œœ˜/KšœN˜NKšœ!˜!Kšœ;˜;K˜——Nšœ˜N˜—Nšœœ˜Nšœœ˜šœœœ˜'Kšœ˜Kšœ˜Kšœ˜K˜—šœœœ˜%Kšœ˜Kšœ˜Kšœ˜K˜—KšœYœœ˜Kšœ œœ˜(Kšœ œœ˜*K˜—K˜K˜š  œœœAœ œœ!œœœ˜»Kšœ ™ š‘œœ œ œ˜DNšœœ˜%šœ œ˜Nšœ˜Nšœ œ˜/Nšœ˜N˜—Nšœ œ˜Nšœ!˜!šœ˜Kšœœ˜šœ˜Kšœœœ˜Kšœœ ˜Kšœ˜Kšœ>˜>K˜—šœ˜Kšœœœ˜Kšœœ ˜Kšœ˜Kšœ>˜>K˜—šœ˜Kšœ!˜!Kšœœ ˜"Kšœ˜Kšœ>˜>K˜—šœ˜Kšœœ˜ Kšœœ˜ Kšœœ˜ Kšœ œ˜Kšœœ ˜Kšœ˜Kšœ(œ:˜qKšœœœ˜KšœN˜NKšœœœ˜šœ˜"KšœO˜OKšœ_˜_K˜—Kšœœœ˜#Kšœ>˜>K˜—šœ˜Kšœœ˜ Kšœœ˜ Kšœœ˜ Kšœ œ˜Kšœœ ˜Kšœ˜Kšœ˜Kšœ(œ5˜lKšœœœ˜KšœL˜Lšœœœ˜!šœ˜KšœO˜OKšœZ˜Z—Kšœœœ˜#K˜—Kšœ>˜>K˜—šœ˜Kšœœ˜ Kšœœ ˜Kšœ6˜6Kšœ>˜>K˜—šœ˜ Kšœœ˜Kšœ œ˜Kšœœ˜ Kšœœ ˜Kšœ%˜%šœ˜#Kš œœœœœœJ˜–KšœY˜Y—Kšœ œœœ˜-Kšœd˜dšœœœ˜!šœ˜KšœO˜OKšœr˜r—Kšœœœ˜#K˜—Kšœ>˜>K˜——N˜—Nšœ œœ˜Nšœœ˜Nšœ œ˜ Kšœ˜Kšœ˜Kšœ˜Kšœ œ=˜KKšœ<œœH˜¦Kšœ˜K˜—K˜š   œœœAœœ œ˜™KšœL™LNšœ˜Nšœœ˜Kšœ˜Kšœ˜Kšœ˜Nšœ œ˜/Nšœ˜Kšœ]˜]Kšœ˜K˜—K˜—™š  œœœ œœ&œœ?œœ˜ιNšœ œ˜$Nšœœ ˜š‘ œœœ˜Nšœœ˜ Nšœ˜Nšœ)˜)Nšœ7˜7šœœ˜Nšœœ˜Nšœ˜Nšœœœœ˜1Nšœ˜šœ œ˜Nšœ œ˜N˜"N˜—N˜—šœœ˜Nšœœ.œœ˜ENšœ/œœ˜<šœ$œ˜,Kšœƒ˜ƒN˜—N˜—N˜—Kšœœ˜ Kšœœœ˜/Kšœœœ˜KšœE˜EKšœ2˜2Kšœ˜Nšœ ˜ šœœœ˜š‘œœœ˜Nšœ ˜ Nšœ˜Nš œ ˜N˜—Nšœ˜Nšœ œœ˜Nšœ:˜:NšœH˜HNšœœ˜Nšœ ˜ Nšœ˜Nšœ ˜ šœ˜K˜2KšœS˜SKšœœœœ˜1K˜Kšœ˜—šœ ˜ Kšœ ˜ Kšœ˜Kšœ˜Kšœ˜Kšœ˜K˜—N˜Nšœ ˜ N˜"N˜—Nšœ˜N˜—š  œ œ˜)K™'Nšœœ ˜Nšœœ˜ Nšœ˜Nšœ+˜+Nšœ7˜7šœœ˜Nšœœ˜Nšœ˜N˜—K˜K˜—š  œœœ&œ œ˜kNšœœ˜Nšœœ˜%šœ'œ˜/šœ˜Kšœœ˜šœ˜Kšœœœ˜Kšœœ˜Kšœœœ˜(Kšœœœ ˜+Kšœ:˜@K˜—šœ˜Kšœœœ˜Kšœœ˜ Kšœœœ˜-Kšœœœ ˜0Kšœ:˜@K˜—šœ˜Kšœ!˜!Kšœœ˜ Kšœœ˜'Kšœ?˜?Kšœ œœ ˜#Kšœ œœ˜ Kšœ*œœ˜>Kšœ*œœ ˜AKšœ:˜@K˜—šœ˜Kšœœ˜ Nšœœ˜Kšœœ˜ Nšœ˜šœœœ˜%Nšœœ˜"Nšœœ˜%NšœœœŸ˜2Nšœœœœ ˜,šœ˜Nšœ œ˜Nšœ œ ˜Nšœ˜—Nšœ˜—šœ$œœ˜0Nšœ œ˜KšœE˜EKšœ:˜@Kšœ˜—Kšœœœ˜K˜—šœ˜Kšœœ˜ Nšœœ˜Kšœœ˜ Nšœ˜š œœœœ&˜CNšœœ˜"Nšœœ˜ NšœœœŸ˜2šœ˜Nšœ œ˜Nšœ œ ˜Nšœ˜—Nšœ˜—šœ˜%Nšœ œ ˜Nšœ œ˜šœ˜ Nšœ œ˜KšœN˜NKšœ:˜@N˜——K˜—šœ˜Kšœ ˜ Kšœ ˜ Kšœœ˜Kšœ5˜5Kšœœœ˜"Kšœ:˜@K˜—šœ˜ Kšœœ˜Kšœœ˜ š œœœœ0˜MNšœœ˜Nšœœ˜ NšœœœŸ˜2šœ˜Nšœ œ˜Nšœ œ ˜Nšœ˜—Nšœ˜—šœ˜Nšœœ ˜#Nšœœ˜ šœ˜ Nšœ œ˜KšœN˜NKšœ:˜@N˜——K˜——N˜—Nšœ'œœ˜N˜—š‘œœœœ ˜GKšœœ˜-šœœœ˜+Kšœ’˜’N˜—Kšœ4˜6N˜—š‘œœ œ˜#Kšœœ˜-šœœœ˜+Kšœ’˜’N˜—Kšœ$˜&N˜—š ‘ œœ œœ œ˜=Kšœœ˜-Nšœœ#˜.N˜—š ‘ œœœœ œ˜2Kšœœ˜-Nšœ6˜8Nšœœœœ;˜gN˜——™ Nš œ œœœœ˜:Nšœœ˜Nšœœ˜š  œœœœ ˜,Nšœœœ ˜*šœœ˜Nšœ"˜"Nšœ˜Nšœ˜—Nšœ˜—š  œœœ ˜#šœ#œ˜,Nšœ˜Nšœ"˜"N˜—Nšœ˜——™š‘ œ˜Nšœœœ™&Nšœœ ˜Nšœœ ˜Nšœœ˜Nšœœ˜Nšœ'˜-Nšœ˜——™,š œœ˜Nšœ˜Nšœ'˜'š˜š‘ œœœ˜šœ)˜)Nšœœœœ™7Nšœœ ˜Nšœœ˜Nšœœ'˜ANšœ˜Nšœœœ˜/Nšœ˜—Nšœœ˜ Nšœ˜Nšœ4˜4Nšœ˜—Nšœ˜Nšœ,œ ˜?Nšœ˜—Nšœ˜—š œœ˜š˜šœ œœ˜Nšœ œ˜N˜—Nšœ œ)˜:Nšœ ˜ Nšœœ˜Nšœ˜—Nšœ˜—NšœG˜GNšœ:œKœ˜“šœ˜ Nšœœ˜)Nšœœ˜+NšœM˜MNšœ˜—Nšœ7œ˜