<> <> <> <> <> <> <> DIRECTORY Basics USING[CompareInt, CardMod], <> YggDummyFile USING[Create, Delete, Error, FindVolumeFromID, OldFP, Handle, Info, LabeledOpen, PageCount, PropertyStorage, RC, SetSize, Volume, VolumeID], YggDummyProcess USING[Detach, GetCurrent, PauseMsec], RedBlackTree, VM USING[AddressForPageNumber, Allocate, Interval, PageCount, PageNumber], YggDummyVM USING [MakeUnchanged, Pin, State], YggEnvironment USING [FileID, PageCount, PageNumber, PageRun, VolumeID, VolOrVolGroupID], YggFileMap USING[GetNext, GetVolumeIDAndFileID, Handle, VerifyFilePageMgrHandle], YggFilePageMgr USING[DirtyNoWaitReleaseState, DirtyWaitReleaseState, ReleaseState, VMPageSet], YggFilePageMgrIO USING[DoIO, GetNext, IORequest, IOType, LogError, RegisterRequest], YggFilePageMgrLru USING[CheckCacheInCleanState, GetOtherChunkFromLruList, GetOurChunkFromLruList, InitializeLruLists, LruListPlace, PutMappedChunkOnLruList, PutUnmappedChunkOnLruList, RelinkChunkAsLruOnLruList, SweepItem, UsingTooMuchOfCache, WaitToSweep], YggFilePageMgrPrivateChunk USING[Chunk, ChunkFilePageCount, ChunkVMPageCount, ChunkType, ClientChunkType, ListChunkType, RefChunk], YggFilePageMgrPrivateFile USING[FPMFileObject, VolumeState], YggInternal USING[FileHandle]; YggFilePageMgrMainImpl: CEDAR MONITOR LOCKS fpmFileHandle USING fpmFileHandle: FPMFileHandle IMPORTS Basics, YggDummyFile, YggFileMap, YggFilePageMgrIO, YggFilePageMgrLru, RedBlackTree, YggDummyProcess, YggDummyVM, VM EXPORTS YggInternal, YggFilePageMgr, YggFilePageMgrPrivateChunk SHARES YggFilePageMgr = BEGIN VolOrVolGroupID: TYPE = YggEnvironment.VolOrVolGroupID; FileID: TYPE = YggEnvironment.FileID; FPMFileHandle: TYPE = REF FPMFileObject; FPMFileObject: PUBLIC TYPE = YggFilePageMgrPrivateFile.FPMFileObject; RefChunk: TYPE = REF Chunk; Chunk: PUBLIC TYPE = YggFilePageMgrPrivateChunk.Chunk; PageKey: TYPE = REF YggEnvironment.PageNumber; <> <> <> <> <> <> InsufficientSpaceOnVolume: PUBLIC -- ABSTRACTION -- ERROR = CODE; NoSuchFile: PUBLIC -- CALLING -- ERROR = CODE; NoSuchVolume: PUBLIC -- CALLING -- ERROR = CODE; PageRunArgIllegal: PUBLIC -- CALLING -- ERROR = CODE; PageRunExtendsPastEof: PUBLIC -- CALLING -- ERROR = CODE; SizeArgIllegal: PUBLIC -- CALLING -- ERROR = CODE; VolumeTooFragmented: PUBLIC -- ABSTRACTION -- ERROR = CODE; VolumeWentOffline: PUBLIC -- ABSTRACTION -- ERROR = CODE; VolumeError: ARRAY YggFilePageMgrPrivateFile.VolumeState OF ERROR _ [online: InternalFilePageMgrLogicError, wentOffline: VolumeWentOffline, nonExist: NoSuchVolume]; CurrentEpoch: NAT _ 0; ExpectedSecondsToNextCheckpoint: NAT _ 300; SweeperControl: ARRAY ListChunkType OF SweeperControlRec; SweeperControlRec: TYPE = RECORD[ newCheckpoint: BOOL _ TRUE ]; InternalFilePageMgrLogicError: PUBLIC -- PROGRAMMING -- ERROR = CODE; Okay: ERROR = CODE; -- for our own use. ListChunkType: TYPE = YggFilePageMgrPrivateChunk.ListChunkType; ReadDone: CONDITION; WriteDone: CONDITION; ReadPages: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle, pageRun: YggEnvironment.PageRun] RETURNS [vMPageSet: YggFilePageMgr.VMPageSet] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, PageRunArgIllegal, PageRunExtendsPastEof, VolumeWentOffline. vMPageSet _ BasicGetPages[fileHandle, pageRun, read, normal].vMPageSet; END; ReadLogPages: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle, pageRun: YggEnvironment.PageRun] RETURNS [vMPageSet: YggFilePageMgr.VMPageSet] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, PageRunArgIllegal, PageRunExtendsPastEof, VolumeWentOffline. vMPageSet _ BasicGetPages[fileHandle, pageRun, read, log].vMPageSet; END; UsePages: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle, pageRun: YggEnvironment.PageRun] RETURNS [vMPageSet: YggFilePageMgr.VMPageSet] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, PageRunArgIllegal, PageRunExtendsPastEof, VolumeWentOffline. vMPageSet _ BasicGetPages[fileHandle, pageRun, use, normal].vMPageSet; END; UseLogPages: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle, pageRun: YggEnvironment.PageRun] RETURNS [vMPageSet: YggFilePageMgr.VMPageSet] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, PageRunArgIllegal, PageRunExtendsPastEof, VolumeWentOffline. vMPageSet _ BasicGetPages[fileHandle, pageRun, use, log].vMPageSet; END; <> ReadAheadPages: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle, pageRun: YggEnvironment.PageRun] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, PageRunArgIllegal, VolumeWentOffline. BasicReadAhead[fileHandle, pageRun, normal]; END; ReadAheadLogPages: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle, pageRun: YggEnvironment.PageRun] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, PageRunArgIllegal, VolumeWentOffline. BasicReadAhead[fileHandle, pageRun, log]; END; <> ShareVMPageSet: PUBLIC PROCEDURE[vMPageSet: YggFilePageMgr.VMPageSet] = BEGIN -- non system fatal errors: none. refChunk: RefChunk _ vMPageSet.refChunk; -- keep the compiler happy. <= 1.>> MonitoredShareVMPageSet[GetFilePageMgrHandle[refChunk.fileHandle, TRUE], refChunk]; END; ReleaseVMPageSet: PUBLIC PROCEDURE[vMPageSet: YggFilePageMgr.VMPageSet, releaseState: YggFilePageMgr.ReleaseState, keep: BOOLEAN] = BEGIN -- non system fatal errors: none. refChunk: RefChunk _ vMPageSet.refChunk; fpmFileHandle: FPMFileHandle _ GetFilePageMgrHandle[refChunk.fileHandle, TRUE]; writeNeeded: BOOLEAN _ FALSE; IF releaseState IN YggFilePageMgr.DirtyWaitReleaseState THEN BEGIN IF (writeNeeded _ MonitoredWaitForWriteToCompleteThenMaybeSetWIP[ fpmFileHandle, refChunk]) THEN CleanAndWriteChunk[fpmFileHandle, refChunk]; END; MonitoredMainReleaseVMPageSet[fpmFileHandle, refChunk, releaseState, keep, writeNeeded]; END; ForceOutVMPageSet: PUBLIC PROCEDURE[vMPageSet: YggFilePageMgr.VMPageSet] = BEGIN -- non system fatal errors: none. refChunk: RefChunk _ vMPageSet.refChunk; -- keep compiler happy. fpmFileHandle: FPMFileHandle _ GetFilePageMgrHandle[refChunk.fileHandle, TRUE]; IF MonitoredWaitForWriteToCompleteThenMaybeSetWIP[fpmFileHandle, refChunk] THEN BEGIN CleanAndWriteChunk[fpmFileHandle, refChunk]; MonitoredSetChunkValidAfterIO[fpmFileHandle, refChunk, writeCompleted]; END; END; ForceOutFile: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, VolumeWentOffline. fpmFileHandle: FPMFileHandle _ GetFilePageMgrHandle[fileHandle, FALSE]; errors: ERROR _ Okay; listOfValidAndDirty: LIST OF RefChunk; listOfWIPAndClean: LIST OF ChunkAndPage; [errors, listOfValidAndDirty, listOfWIPAndClean] _ MonitoredForceOutFileSortChunks[fpmFileHandle]; IF errors # Okay THEN ERROR errors; DoSequentialIO[fpmFileHandle, write, listOfValidAndDirty, 0]; MonitoredForceOutFileSetValidAndWait[fpmFileHandle, listOfValidAndDirty, listOfWIPAndClean]; END; CheckPointOccuring: PUBLIC PROC [checkPointEpoch: NAT _ 1, expectedSecondsToNextCheckpoint: NAT] RETURNS [oldestEpochWithDirtyUnwrittenPages: NAT _ 1] = { CurrentEpoch _ checkPointEpoch; ExpectedSecondsToNextCheckpoint _ MAX [5, expectedSecondsToNextCheckpoint]; FOR chunkType: ListChunkType IN ListChunkType DO SweeperControl[chunkType].newCheckpoint _ TRUE; ENDLOOP; }; ProcessControlList: TYPE = RECORD[s: SEQUENCE size: NAT OF ProcessControlItem]; ProcessControlItem: TYPE = RECORD[ processID: PROCESS, processRunning: BOOL _ FALSE ]; Sweeper: PUBLIC PROCEDURE[chunkType: YggFilePageMgrPrivateChunk.ChunkType] = { needToHurry: BOOLEAN _ FALSE; hurryInARowCount: INT _ 0; sweepList: LIST OF YggFilePageMgrLru.SweepItem; DO listSize: INT _ 0; millisecondsBetweenWrites: INT _ 0; processControlList: REF ProcessControlList; <> [needToHurry, sweepList] _ YggFilePageMgrLru.WaitToSweep[needToHurry, CurrentEpoch, chunkType]; hurryInARowCount _ IF needToHurry THEN hurryInARowCount + 1 ELSE 1; IF needToHurry THEN processControlList _ NEW[ProcessControlList[hurryInARowCount]]; SweeperControl[chunkType].newCheckpoint _ FALSE; FOR sl: LIST OF YggFilePageMgrLru.SweepItem _ sweepList, sl.rest UNTIL sl = NIL DO listSize _ listSize + 1; ENDLOOP; <> millisecondsBetweenWrites _ (ExpectedSecondsToNextCheckpoint*1000)/listSize; UNTIL sweepList = NIL OR SweeperControl[chunkType].newCheckpoint DO fpmFileHandle: FPMFileHandle _ NIL; listOfValidAndDirty: LIST OF RefChunk; fpmFileHandle _ GetFilePageMgrHandle[sweepList.first.fileHandle, FALSE]; <> [listOfValidAndDirty, sweepList] _ MonitoredSweeperSortChunks[fpmFileHandle, sweepList.first.fileHandle, sweepList]; IF needToHurry THEN { <> DO processControlIndex: INT _ 0; processForked: PROCESS; FOR processControlIndex IN [1..hurryInARowCount] DO IF processControlList[processControlIndex].processRunning THEN LOOP ELSE EXIT; REPEAT FINISHED => LOOP; ENDLOOP; processControlList[processControlIndex].processRunning _ TRUE; processForked _ FORK ForkedSweepAFile[processControlList, processControlIndex, fpmFileHandle, listOfValidAndDirty, 0, chunkType, 30]; TRUSTED {YggDummyProcess.Detach[processForked];}; processControlList[processControlIndex].processID _ processForked; ENDLOOP; } ELSE SweepAFile[fpmFileHandle, listOfValidAndDirty, millisecondsBetweenWrites, chunkType, 5]; ENDLOOP; ENDLOOP; }; ForkedSweepAFile: PROC [processControlList: REF ProcessControlList, processControlIndex: INT, fpmFileHandle: FPMFileHandle, listOfValidAndDirty: LIST OF RefChunk, millisecondsBetweenWrites: INT, chunkType: YggFilePageMgrPrivateChunk.ChunkType, maxWriteGroupSize: INT ] = { SweepAFile[fpmFileHandle, listOfValidAndDirty, millisecondsBetweenWrites, chunkType, maxWriteGroupSize]; processControlList[processControlIndex].processRunning _ FALSE; }; SweepAFile: PROC [fpmFileHandle: FPMFileHandle, listOfValidAndDirty: LIST OF RefChunk, millisecondsBetweenWrites: INT, chunkType: YggFilePageMgrPrivateChunk.ChunkType, maxWriteGroupSize: INT ] = { DO sizeInHead: INT _ 0; restOfListOfValidAndDirty: LIST OF RefChunk; FOR sl: LIST OF RefChunk _ listOfValidAndDirty, sl.rest UNTIL sl = NIL OR SweeperControl[chunkType].newCheckpoint DO <> sizeInHead _ sizeInHead + 1; IF sizeInHead >= maxWriteGroupSize OR sl.rest = NIL THEN { <> millisecondsForWrites: INT _ 100; <> restOfListOfValidAndDirty _ sl.rest; sl.rest _ NIL; <> <> DoSequentialIO[fpmFileHandle, write, listOfValidAndDirty, 0]; MonitoredSetListOfChunksValidAfterIO[fpmFileHandle, listOfValidAndDirty, writeCompleted]; <> <> <> IF millisecondsForWrites >= 0 AND millisecondsForWrites < 1000 THEN { millisecondsToDelay: INT _ (sizeInHead * millisecondsBetweenWrites) - millisecondsForWrites; IF millisecondsToDelay > 0 THEN YggDummyProcess.PauseMsec[millisecondsToDelay]; }; listOfValidAndDirty _ restOfListOfValidAndDirty; EXIT; }; ENDLOOP; ENDLOOP; }; ForceOutEverything: PUBLIC PROCEDURE = BEGIN -- non system fatal errors: NoSuchVolume, VolumeWentOffline. NilProc: PROCEDURE RETURNS[FPMFileHandle] ~ { RETURN[NIL] }; FOR fileHandle: YggInternal.FileHandle _ YggFileMap.GetNext[NIL], YggFileMap.GetNext[fileHandle] UNTIL fileHandle = NIL DO IF YggFileMap.VerifyFilePageMgrHandle[fileHandle, NilProc] # NIL THEN ForceOutFile[fileHandle ! NoSuchFile => CONTINUE]; ENDLOOP; END; <> RestoreCacheToCleanState: PUBLIC PROCEDURE = BEGIN -- non system fatal errors: none. FOR fileHandle: YggInternal.FileHandle _ YggFileMap.GetNext[NIL], YggFileMap.GetNext[fileHandle] UNTIL fileHandle = NIL DO errors: ERROR _ MonitoredUnmapFile[GetFilePageMgrHandle[fileHandle, FALSE]]; IF ((errors # Okay) AND (errors # NoSuchFile)) THEN ERROR errors; ENDLOOP; IF (NOT YggFilePageMgrLru.CheckCacheInCleanState[]) THEN ERROR InternalFilePageMgrLogicError; -- maybe not, maybe the client did a number on us. END; MyPageRun: TYPE = RECORD[firstPage: YggEnvironment.PageNumber, count: NAT, chunkStartFilePage: YggEnvironment.PageNumber]; ReadReadAheadOrUse: TYPE = {read, readAhead, use}; <> <> <<(a) each chunk type may have a different size, which affects the conversion of the client's page number, etc. to the chunk page number, etc.>> <<(b) tells which lru list to go to to get a new chunk.>> <> <<(a) read may do a read, use doesn't have to read unless partial chunk, readAhead's caller will do the reads in a bunch.>> BasicGetPages: PROCEDURE[fileHandle: YggInternal.FileHandle, pageRun: YggEnvironment.PageRun, readReadAheadOrUse: ReadReadAheadOrUse, chunkType: YggFilePageMgrPrivateChunk.ClientChunkType] RETURNS [vMPageSet: YggFilePageMgr.VMPageSet, nFilePagesToRead: YggEnvironment.PageCount] = -- the peculiar structure is to avoid monitor conflicts: see discussion below in the DO loop. BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, PageRunArgIllegal, PageRunExtendsPastEof, VolumeWentOffline. fpmFileHandle: FPMFileHandle _ GetFilePageMgrHandle[fileHandle, FALSE]; myPageRun: MyPageRun; otherRefChunk: RefChunk _ NIL; errors: ERROR; done: BOOLEAN; IF ((pageRun.firstPage < 0) OR (pageRun.count = 0)) THEN ERROR PageRunArgIllegal; myPageRun _ [firstPage: pageRun.firstPage, count: pageRun.count, chunkStartFilePage: ChunkStartFilePage[pageRun.firstPage, chunkType]]; DO <> <> <<[...] _ MonitoredFindOldOrGiveNewChunk[...]; -- see if its already mapped>> <> <<-- get a new chunk>> <> <> <<-- see if its already mapped; if not use the otherRefChunk>> <<[...] _ MonitoredFindOldOrGiveNewChunk[...otherRefChunk...];>> <<};>> <<-- code from the IF done below>> <> <<>> [errors, done, nFilePagesToRead, vMPageSet] _ MonitoredFindOldOrGiveNewChunk[fpmFileHandle, fileHandle, otherRefChunk, myPageRun, readReadAheadOrUse, chunkType]; IF errors # Okay THEN ERROR errors; IF done THEN BEGIN IF ((nFilePagesToRead # 0) AND (readReadAheadOrUse # readAhead)) THEN BEGIN <> refChunk: RefChunk _ vMPageSet.refChunk; YggFilePageMgrIO.DoIO[read, fpmFileHandle.lowerHandle, [[refChunk.startFilePageNumber], nFilePagesToRead, VMAddressForPageNumber[refChunk.startVMPageNumber]]]; <> MakeAllPagesInChunkClean[refChunk]; <> MonitoredSetChunkValidAfterIO[fpmFileHandle, vMPageSet.refChunk, readCompleted]; END; RETURN; END; IF otherRefChunk = NIL THEN otherRefChunk _ MonitoredOtherGetNewChunk[chunkType] ELSE ERROR InternalFilePageMgrLogicError; ENDLOOP; END; <> MonitoredFindOldOrGiveNewChunk: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, fileHandle: YggInternal.FileHandle, otherRefChunk: RefChunk, myPageRun: MyPageRun, readReadAheadOrUse: ReadReadAheadOrUse, chunkType: YggFilePageMgrPrivateChunk.ClientChunkType] RETURNS [errors: ERROR, done: BOOLEAN, nFilePagesToRead: YggEnvironment.PageCount, vMPageSet: YggFilePageMgr.VMPageSet] = -- values of errors are {NoSuchFile, NoSuchVolume, PageRunExtendsPastEof, VolumeWentOffline} BEGIN -- non system fatal errors: none. refChunk: RefChunk; new, wholeChunk: BOOLEAN; chunkTrueSize: YggEnvironment.PageCount; [errors, refChunk] _ GetMappedChunk[fpmFileHandle, myPageRun, readReadAheadOrUse]; IF errors # Okay THEN BEGIN <> IF otherRefChunk # NIL THEN YggFilePageMgrLru.PutUnmappedChunkOnLruList[otherRefChunk]; RETURN; END; new _ (refChunk = NIL); SELECT TRUE FROM new => IF otherRefChunk = NIL THEN RETURN[Okay, FALSE, 0, [NIL, [0, 0], NIL]] ELSE BEGIN refChunk _ otherRefChunk; MapChunk[fpmFileHandle, fileHandle, myPageRun.chunkStartFilePage, refChunk]; END; NOT new => IF otherRefChunk # NIL <> THEN YggFilePageMgrLru.PutUnmappedChunkOnLruList[otherRefChunk]; ENDCASE; done _ TRUE; refChunk.useCount _ refChunk.useCount + 1; nFilePagesToRead _ 0; [vMPageSet, wholeChunk, chunkTrueSize] _ SetUpVmPageSet[refChunk, myPageRun.firstPage, myPageRun.count, fpmFileHandle.fileDataSize]; SELECT TRUE FROM new => IF ((readReadAheadOrUse = use) AND (wholeChunk)) THEN BEGIN MakeAllPagesInChunkClean[refChunk]; refChunk.state _ valid; END ELSE BEGIN refChunk.state _ readInProgress; nFilePagesToRead _ chunkTrueSize; END; NOT new => SELECT readReadAheadOrUse FROM read, use => DO IF refChunk.state = readInProgress THEN WAIT ReadDone ELSE EXIT; ENDLOOP; ENDCASE; ENDCASE; END; <> GetMappedChunk: INTERNAL PROCEDURE[fpmFileHandle: FPMFileHandle, myPageRun: MyPageRun, readReadAheadOrUse: ReadReadAheadOrUse] RETURNS [errors: ERROR, refChunk: RefChunk] = -- values of errors are {NoSuchFile, NoSuchVolume, PageRunExtendsPastEof, VolumeWentOffline}. BEGIN -- non system fatal errors: none. IF fpmFileHandle.volumeState # online THEN RETURN[VolumeError[fpmFileHandle.volumeState], NIL]; IF (NOT fpmFileHandle.exists) THEN RETURN[NoSuchFile, NIL]; IF myPageRun.firstPage + myPageRun.count > fpmFileHandle.fileDataSize THEN RETURN[PageRunExtendsPastEof, NIL]; IF ((refChunk _ RBTLookup[fpmFileHandle, myPageRun.chunkStartFilePage]) # NIL) AND (refChunk.useCount = 0) THEN YggFilePageMgrLru.GetOurChunkFromLruList[refChunk, FALSE]; errors _ Okay; END; <> MapChunk: INTERNAL PROCEDURE[fpmFileHandle: FPMFileHandle, fileHandle: YggInternal.FileHandle, startChunkFilePage: YggEnvironment.PageNumber, refChunk: RefChunk] = BEGIN -- non system fatal errors: none. RBTInsert[fpmFileHandle, refChunk, startChunkFilePage]; refChunk.fileHandle _ fileHandle; refChunk.startFilePageNumber _ startChunkFilePage; fpmFileHandle.nMappedChunks _ fpmFileHandle.nMappedChunks + 1; END; <> UnmapChunk: INTERNAL PROCEDURE[fpmFileHandle: FPMFileHandle, refChunk: RefChunk] = BEGIN -- non system fatal errors: none. IF RBTDelete[fpmFileHandle, refChunk.startFilePageNumber] = NIL THEN ERROR InternalFilePageMgrLogicError; IF refChunk.defWritePending THEN BEGIN refChunk.defWritePending _ FALSE; fpmFileHandle.nDefWriteChunks _ fpmFileHandle.nDefWriteChunks - 1; END; refChunk.fileHandle _ NIL; fpmFileHandle.nMappedChunks _ fpmFileHandle.nMappedChunks - 1; refChunk.state _ undefined; END; SetUpVmPageSet: INTERNAL PROCEDURE[refChunk: RefChunk, clientStartFilePage: YggEnvironment.PageNumber, clientFilePageCount: NAT, fileDataSize: YggEnvironment.PageCount] RETURNS [vMPageSet: YggFilePageMgr.VMPageSet, wholeChunk: BOOLEAN, chunkTrueSize: YggEnvironment.PageCount] = BEGIN -- non system fatal errors: none. chunkEndFilePagePlus1: YggEnvironment.PageNumber; clientEndFilePagePlus1: YggEnvironment.PageNumber; chunkEndFilePagePlus1 _ MIN[refChunk.startFilePageNumber + YggFilePageMgrPrivateChunk.ChunkFilePageCount[refChunk.chunkType], fileDataSize]; clientEndFilePagePlus1 _ MIN[clientStartFilePage + clientFilePageCount, chunkEndFilePagePlus1]; vMPageSet.pages _ VMAddressForPageNumber[GetVMIntervalFromFileInterval[refChunk.startFilePageNumber, clientStartFilePage, 0, refChunk.startVMPageNumber].vmPageNumber]; vMPageSet.pageRun.firstPage _ clientStartFilePage; vMPageSet.pageRun.count _ clientEndFilePagePlus1 - clientStartFilePage; vMPageSet.refChunk _ refChunk; chunkTrueSize _ chunkEndFilePagePlus1 - refChunk.startFilePageNumber; wholeChunk _ ((clientStartFilePage = refChunk.startFilePageNumber) AND (vMPageSet.pageRun.count = chunkTrueSize)); END; <> <> MonitoredOtherGetNewChunk: PROCEDURE[chunkType: YggFilePageMgrPrivateChunk.ClientChunkType] RETURNS [otherRefChunk: RefChunk] = BEGIN -- non system fatal errors: none. DO otherFileHandle: YggFileMap.Handle; otherStartFilePageNumber: YggEnvironment.PageNumber; mapped: BOOLEAN; [mapped, otherRefChunk, otherFileHandle, otherStartFilePageNumber] _ YggFilePageMgrLru.GetOtherChunkFromLruList[chunkType]; IF mapped -- we didn't get it yet, try to free it. THEN BEGIN ErrorProc: PROCEDURE RETURNS[fpmFileHandle: FPMFileHandle] = BEGIN ERROR ConsistencyError; END; -- non system fatal errors: none. IF (NOT MonitoredOtherFreeChunkFromFile[YggFileMap.VerifyFilePageMgrHandle[otherFileHandle, ErrorProc], otherFileHandle, otherRefChunk, otherStartFilePageNumber]) THEN LOOP; END; EXIT; ENDLOOP; END; <<"Other" because it gets the monitor lock for a different file than the one we are working on -- namely the file that currently contains the Chunk we want to map into this file.>> MonitoredOtherFreeChunkFromFile: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, fileHandle: YggInternal.FileHandle, refChunk: RefChunk, startFilePageNumber: YggEnvironment.PageNumber] RETURNS [okay: BOOLEAN] = BEGIN -- non system fatal errors: none. hurryUp: BOOLEAN _ FALSE; dirty: BOOLEAN _ FALSE; DO okay _ ((RBTLookup[fpmFileHandle, startFilePageNumber] = refChunk) AND (refChunk.useCount = 0)); IF NOT okay THEN RETURN; IF refChunk.state = readInProgress THEN BEGIN WAIT ReadDone; LOOP; END; IF refChunk.state = writeInProgress THEN BEGIN hurryUp _ TRUE; WAIT WriteDone; LOOP; END; EXIT; ENDLOOP; dirty _ ChunkIsDirty[refChunk]; YggFilePageMgrLru.GetOurChunkFromLruList[refChunk, (hurryUp OR dirty)]; IF dirty THEN BEGIN refChunk.state _ writeInProgress; CleanAndWriteChunk[fpmFileHandle: fpmFileHandle, refChunk: refChunk, lockHeld: TRUE]; END; UnmapChunk[fpmFileHandle, refChunk]; END; BasicReadAhead: PROCEDURE[fileHandle: YggInternal.FileHandle, pageRun: YggEnvironment.PageRun, chunkType: YggFilePageMgrPrivateChunk.ClientChunkType[normal..log]] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, PageRunArgIllegal, VolumeWentOffline. vMPageSet: YggFilePageMgr.VMPageSet; lastValidNFilePagesToRead: YggEnvironment.PageCount _ 0; lastIndex: NAT _ 0; listOfRefChunk: LIST OF RefChunk _ NIL; listOfVMPageSet: LIST OF YggFilePageMgr.VMPageSet _ NIL; DO nFilePagesToRead: YggEnvironment.PageCount; [vMPageSet, nFilePagesToRead] _ BasicGetPages[fileHandle, pageRun, readAhead, chunkType ! PageRunExtendsPastEof => GOTO done]; IF pageRun.count < vMPageSet.pageRun.count THEN ERROR InternalFilePageMgrLogicError; pageRun.firstPage _ pageRun.firstPage + vMPageSet.pageRun.count; pageRun.count _ pageRun.count - vMPageSet.pageRun.count; IF nFilePagesToRead # 0 THEN BEGIN lastIndex _ lastIndex + 1; lastValidNFilePagesToRead _ nFilePagesToRead; listOfRefChunk _ CONS[vMPageSet.refChunk, listOfRefChunk]; listOfVMPageSet _ CONS[vMPageSet, listOfVMPageSet]; END ELSE ReleaseVMPageSet[vMPageSet, clean, TRUE]; IF ((pageRun.count = 0) OR (lastIndex = MaxReadAheadSets)) THEN EXIT; REPEAT done => NULL; ENDLOOP; IF listOfRefChunk # NIL THEN TRUSTED BEGIN YggDummyProcess.Detach[FORK ForkedBasicReader[fileHandle, listOfRefChunk, listOfVMPageSet, lastValidNFilePagesToRead]]; END; END; ForkedBasicReader: PROCEDURE[fileHandle: YggInternal.FileHandle, listOfRefChunk: LIST OF RefChunk, listOfVMPageSet: LIST OF YggFilePageMgr.VMPageSet, nFilePagesToRead: YggEnvironment.PageCount] = BEGIN -- non system fatal errors: none. fpmFileHandle: FPMFileHandle _ GetFilePageMgrHandle[fileHandle, TRUE]; DoSequentialIO[fpmFileHandle, read, listOfRefChunk, nFilePagesToRead]; FOR listOfVMPageSet _ listOfVMPageSet, listOfVMPageSet.rest UNTIL listOfVMPageSet = NIL DO ReleaseVMPageSet[listOfVMPageSet.first, clean, TRUE]; ENDLOOP; MonitoredSetListOfChunksValidAfterIO[fpmFileHandle, listOfRefChunk, readCompleted]; END; MonitoredShareVMPageSet: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, refChunk: RefChunk] = BEGIN -- non system fatal errors: none. IF (refChunk.useCount _ refChunk.useCount + 1) = 1 THEN ERROR ConsistencyError; END; MonitoredMainReleaseVMPageSet: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, refChunk: RefChunk, releaseState: YggFilePageMgr.ReleaseState, keep: BOOLEAN, setValid: BOOLEAN] = BEGIN -- non system fatal errors: none. IF setValid THEN BEGIN refChunk.state _ valid; BROADCAST WriteDone; END; IF refChunk.chunkType = normal AND YggFilePageMgrLru.UsingTooMuchOfCache[fpmFileHandle] AND refChunk.state # readInProgress THEN BEGIN -- we catch after the fact. keep _ FALSE; IF releaseState = writeIndividualNoWait THEN releaseState _ writeBatchedNoWait; END; IF (refChunk.useCount _ refChunk.useCount - 1) = 0 THEN { YggFilePageMgrLru.PutMappedChunkOnLruList[refChunk, IF ((releaseState IN YggFilePageMgr.DirtyNoWaitReleaseState) OR (keep)) THEN mru ELSE lru]; IF ((releaseState = writeBatchedNoWait) AND (NOT refChunk.defWritePending)) THEN { refChunk.defWritePending _ TRUE; IF (fpmFileHandle.nDefWriteChunks _ fpmFileHandle.nDefWriteChunks + 1) >= LimitDefWriteChunks THEN StartSomeDeferredWrites[fpmFileHandle, refChunk, keep]; }; }; END; <> ChunkAndPage: TYPE = RECORD[refChunk: RefChunk, startFilePageNumber: YggEnvironment.PageNumber]; StartSomeDeferredWrites: INTERNAL PROCEDURE[fpmFileHandle: FPMFileHandle, startChunk: RefChunk, keep: BOOLEAN] = BEGIN -- non system fatal errors: none. nextChunk: RefChunk _ startChunk; searchProc: RBTLookupProc _ RBTLookupNextSmaller; listOfChunksAndPages: LIST OF ChunkAndPage _ NIL; AddChunk: PROCEDURE = BEGIN -- non system fatal errors: none. listOfChunksAndPages _ CONS[[nextChunk, nextChunk.startFilePageNumber], listOfChunksAndPages]; END; ForkDemon: PROCEDURE = BEGIN -- non system fatal errors: none. TRUSTED BEGIN YggDummyProcess.Detach[FORK DeferredWriteDemon[fpmFileHandle, listOfChunksAndPages, keep]]; END; listOfChunksAndPages _ NIL; END; DO IF nextChunk.defWritePending THEN BEGIN nextChunk.defWritePending _ FALSE; fpmFileHandle.nDefWriteChunks _ fpmFileHandle.nDefWriteChunks - 1; AddChunk[]; IF fpmFileHandle.nDefWriteChunks = 0 THEN EXIT; END; IF (nextChunk _ searchProc[fpmFileHandle, nextChunk.startFilePageNumber]) = NIL THEN BEGIN IF searchProc = RBTLookupNextLarger THEN ERROR InternalFilePageMgrLogicError; ForkDemon[]; -- keep all the chunks going one way, for performance. searchProc _ RBTLookupNextLarger; IF (nextChunk _ searchProc[fpmFileHandle, startChunk.startFilePageNumber]) = NIL THEN ERROR InternalFilePageMgrLogicError; END; ENDLOOP; ForkDemon[]; END; <> DeferredWriteDemon: PROCEDURE[fpmFileHandle: FPMFileHandle, listOfChunksAndPages: LIST OF ChunkAndPage, keep: BOOLEAN] = BEGIN -- non system fatal errors: none. listOfValidAndDirty: LIST OF RefChunk; listOfWIPAndClean: LIST OF ChunkAndPage; [listOfValidAndDirty, listOfWIPAndClean] _ MonitoredDefWriteSortChunks[fpmFileHandle, listOfChunksAndPages, keep]; DoSequentialIO[fpmFileHandle, write, listOfValidAndDirty, 0]; MonitoredDefWriteSetValidAndWait[fpmFileHandle, listOfValidAndDirty, listOfWIPAndClean]; END; <> <> <> <> <> MonitoredDefWriteSortChunks: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, listOfChunksAndPages: LIST OF ChunkAndPage, keep: BOOLEAN] RETURNS[listOfValidAndDirty: LIST OF RefChunk, listOfWIPAndClean: LIST OF ChunkAndPage] = BEGIN -- non system fatal errors: none. refChunk: RefChunk; listOfValidAndDirty _ NIL; listOfWIPAndClean _ NIL; FOR listOfChunksAndPages _ listOfChunksAndPages, listOfChunksAndPages.rest UNTIL listOfChunksAndPages = NIL DO dirty: BOOLEAN; refChunk _ RBTLookup[fpmFileHandle, listOfChunksAndPages.first.startFilePageNumber]; IF ((refChunk # listOfChunksAndPages.first.refChunk) OR (refChunk.defWritePending)) THEN LOOP; dirty _ ChunkIsDirty[refChunk]; SELECT TRUE FROM ((refChunk.state = valid) AND (dirty)) => BEGIN listOfValidAndDirty _ CONS[listOfChunksAndPages.first.refChunk, listOfValidAndDirty]; refChunk.state _ writeInProgress; END; (keep) => SELECT TRUE FROM ((refChunk.state = valid) AND (refChunk.useCount = 0)) => YggFilePageMgrLru.RelinkChunkAsLruOnLruList[refChunk]; ((refChunk.state = writeInProgress) AND (NOT dirty)) => listOfWIPAndClean _ CONS[listOfChunksAndPages.first, listOfWIPAndClean]; ENDCASE; ENDCASE => NULL; ENDLOOP; END; <> <> <> MonitoredDefWriteSetValidAndWait: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, listOfValidAndDirty: LIST OF RefChunk, listOfWIPAndClean: LIST OF ChunkAndPage] = BEGIN -- non system fatal errors: none. refChunk: RefChunk; InternalSetListOfChunksValidAfterIOThenMaybeReorder[fpmFileHandle, listOfValidAndDirty, writeCompleted, TRUE]; DO IF listOfWIPAndClean = NIL THEN RETURN; refChunk _ RBTLookup[fpmFileHandle, listOfWIPAndClean.first.startFilePageNumber]; IF ((refChunk = listOfWIPAndClean.first.refChunk) AND (NOT refChunk.defWritePending) AND (NOT ChunkIsDirty[refChunk])) THEN SELECT TRUE FROM (refChunk.state = valid) => IF refChunk.useCount = 0 THEN YggFilePageMgrLru.RelinkChunkAsLruOnLruList[refChunk]; (refChunk.state = writeInProgress) => BEGIN WAIT WriteDone; LOOP; END; ENDCASE => NULL; listOfWIPAndClean _ listOfWIPAndClean.rest; ENDLOOP; END; MonitoredForceOutFileSortChunks: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle] RETURNS [errors: ERROR, listOfValidAndDirty: LIST OF RefChunk, listOfWIPAndClean: LIST OF ChunkAndPage] = -- values of errors are {NoSuchFile, NoSuchVolume, VolumeWentOffline}. BEGIN -- non system fatal errors: none. refChunk: RefChunk; dirty: BOOLEAN; currentFilePageNumber: YggEnvironment.PageNumber; listOfValidAndDirty _ NIL; listOfWIPAndClean _ NIL; IF fpmFileHandle.volumeState # online THEN BEGIN errors _ VolumeError[fpmFileHandle.volumeState]; RETURN; END; IF (NOT fpmFileHandle.exists) THEN BEGIN errors _ NoSuchFile; RETURN; END; errors _ Okay; refChunk _ RBTLookupSmallest[fpmFileHandle]; DO IF refChunk = NIL THEN RETURN; currentFilePageNumber _ refChunk.startFilePageNumber; BEGIN DO dirty _ ChunkIsDirty[refChunk]; IF ((refChunk.state # writeInProgress) OR (NOT dirty)) THEN EXIT; WAIT WriteDone; IF (RBTLookup[fpmFileHandle, currentFilePageNumber] # refChunk) THEN GOTO doneWithThisChunk; ENDLOOP; SELECT TRUE FROM ((refChunk.state = valid) AND (dirty)) => BEGIN listOfValidAndDirty _ CONS[refChunk, listOfValidAndDirty]; refChunk.state _ writeInProgress; END; (refChunk.state = writeInProgress) => listOfWIPAndClean _ CONS[[refChunk, refChunk.startFilePageNumber], listOfWIPAndClean]; ENDCASE; EXITS doneWithThisChunk => NULL; END; refChunk _ RBTLookupNextLarger[fpmFileHandle, currentFilePageNumber]; ENDLOOP; END; MonitoredForceOutFileSetValidAndWait: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, listOfValidAndDirty: LIST OF RefChunk, listOfWIPAndClean: LIST OF ChunkAndPage] = BEGIN -- non system fatal errors: none. refChunk: RefChunk; InternalSetListOfChunksValidAfterIOThenMaybeReorder[fpmFileHandle, listOfValidAndDirty, writeCompleted, FALSE]; DO IF listOfWIPAndClean = NIL THEN RETURN; refChunk _ RBTLookup[fpmFileHandle, listOfWIPAndClean.first.startFilePageNumber]; IF ((refChunk = listOfWIPAndClean.first.refChunk) AND (listOfWIPAndClean.first.refChunk.state = writeInProgress)) THEN WAIT WriteDone ELSE listOfWIPAndClean _ listOfWIPAndClean.rest; ENDLOOP; END; MonitoredSweeperSortChunks: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, fileHandle: YggFileMap.Handle, sweepList: LIST OF YggFilePageMgrLru.SweepItem] RETURNS[listOfValidAndDirty: LIST OF RefChunk, newSweepList: LIST OF YggFilePageMgrLru.SweepItem] = BEGIN -- non system fatal errors: none. refChunk: RefChunk; listOfValidAndDirty _ NIL; DO refChunk _ RBTLookup[fpmFileHandle, sweepList.first.startFilePageNumber]; IF ((refChunk # NIL) AND (refChunk.useCount = 0) AND (refChunk.state = valid) AND (ChunkIsDirty[refChunk])) THEN BEGIN listOfValidAndDirty _ CONS[refChunk, listOfValidAndDirty]; refChunk.state _ writeInProgress; END; sweepList _ sweepList.rest; IF ((sweepList = NIL) OR (sweepList.first.fileHandle # fileHandle)) THEN EXIT; ENDLOOP; newSweepList _ sweepList; END; MonitoredUnmapFile: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle] RETURNS[errors: ERROR] = -- values of errors are {NoSuchFile, NoSuchVolume, VolumeWentOffline}. BEGIN -- non system fatal errors: none. IF fpmFileHandle.volumeState # online THEN RETURN[VolumeError[fpmFileHandle.volumeState]]; IF (NOT fpmFileHandle.exists) THEN RETURN[NoSuchFile]; DumpInconvenientlyMappedChunks[fpmFileHandle, TRUE, 0]; RETURN[Okay]; END; Create: PUBLIC PROCEDURE[volumeID: YggEnvironment.VolumeID, initialSize: YggEnvironment.PageCount, proc: PROCEDURE[YggEnvironment.FileID]] = BEGIN -- non system fatal errors: InsufficientSpaceOnVolume, NoSuchVolume, SizeArgIllegal, VolumeTooFragmented, VolumeWentOffline plus any raised by proc. LocalProc: PROCEDURE[fp: YggDummyFile.OldFP, propertyStorage: YggDummyFile.PropertyStorage, pageCount: YggDummyFile.PageCount] = BEGIN proc[fp]; END;-- non system fatal errors: any raised by proc. volume: YggDummyFile.Volume; volOrVolGroupID: VolOrVolGroupID _ volumeID; IF initialSize < 0 THEN ERROR SizeArgIllegal; IF (volume _ YggDummyFile.FindVolumeFromID[volOrVolGroupID]) = NIL THEN ERROR NoSuchVolume; [] _ YggDummyFile.Create[volume: volume, size: initialSize, report: LocalProc ! YggDummyFile.Error => SELECT why FROM wentOffline => GOTO wentOffline; volumeFull => GOTO volumeFull; fragmented => GOTO fragmented; ENDCASE; ]; EXITS fragmented => ERROR VolumeTooFragmented; volumeFull => ERROR InsufficientSpaceOnVolume; wentOffline => ERROR VolumeWentOffline; END; Delete: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, VolumeWentOffline. errors: ERROR; IF (errors _ MonitoredDelete[GetFilePageMgrHandle[fileHandle, FALSE]]) # Okay THEN ERROR errors; END; MonitoredDelete: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle] RETURNS[errors: ERROR] = -- values of errors are {NoSuchFile, NoSuchVolume, VolumeWentOffline}. BEGIN -- non system fatal errors: none. IF fpmFileHandle.volumeState # online THEN RETURN[VolumeError[fpmFileHandle.volumeState]]; IF (NOT fpmFileHandle.exists) THEN RETURN[NoSuchFile]; DumpInconvenientlyMappedChunks[fpmFileHandle, TRUE, 0]; YggDummyFile.Delete[fpmFileHandle.lowerHandle ! YggDummyFile.Error => SELECT why FROM wentOffline => GOTO wentOffline; unknownFile => GOTO horrible; ENDCASE;]; fpmFileHandle.fileDataSize _ 0; fpmFileHandle.exists _ FALSE; RETURN[Okay]; EXITS horrible => ERROR InternalFilePageMgrLogicError; wentOffline => RETURN[VolumeWentOffline]; END; SetSize: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle, size: YggEnvironment.PageCount] = BEGIN -- non system fatal errors: InsufficientSpaceOnVolume, NoSuchFile, NoSuchVolume, SizeArgIllegal, VolumeTooFragmented, VolumeWentOffline. errors: ERROR; IF size < 0 THEN ERROR SizeArgIllegal; IF (errors _ MonitoredSetSize[GetFilePageMgrHandle[fileHandle, FALSE], size]) # Okay THEN ERROR errors; END; MonitoredSetSize: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, newSize: YggEnvironment.PageCount] RETURNS[errors: ERROR] = -- values of errors are {InsufficientSpaceOnVolume, NoSuchFile, NoSuchVolume, VolumeTooFragmented, VolumeWentOffline}. BEGIN -- non system fatal errors: none. IF fpmFileHandle.volumeState # online THEN BEGIN errors _ VolumeError[fpmFileHandle.volumeState]; RETURN; END; IF (NOT fpmFileHandle.exists) THEN RETURN[NoSuchFile]; IF fpmFileHandle.fileDataSize = newSize THEN RETURN[Okay]; IF newSize < fpmFileHandle.fileDataSize THEN -- truncating. DumpInconvenientlyMappedChunks[fpmFileHandle, FALSE, newSize]; YggDummyFile.SetSize[fpmFileHandle.lowerHandle, newSize ! YggDummyFile.Error => SELECT why FROM fragmented => GOTO fragmented; unknownFile => GOTO horrible; volumeFull => GOTO volumeFull; wentOffline => GOTO wentOffline; ENDCASE;]; fpmFileHandle.fileDataSize _ newSize; RETURN[Okay]; EXITS fragmented => RETURN[VolumeTooFragmented]; horrible => ERROR InternalFilePageMgrLogicError; volumeFull => RETURN[InsufficientSpaceOnVolume]; wentOffline => RETURN[VolumeWentOffline]; END; < 0 except ones left around in ReadAhead, which we wait for. Some consistency checking is done.>> DumpInconvenientlyMappedChunks: INTERNAL PROCEDURE[fpmFileHandle: FPMFileHandle, dumpingWholeFile: BOOLEAN, size: YggEnvironment.PageCount] = BEGIN -- non system fatal errors: none. IF fpmFileHandle.nMappedChunks # 0 THEN BEGIN refChunk: RefChunk; firstChunkIsPartial: BOOLEAN; [refChunk, firstChunkIsPartial] _ FirstChunkToCheck[fpmFileHandle, dumpingWholeFile, size]; IF refChunk # NIL THEN CheckTheChunks[fpmFileHandle, refChunk, firstChunkIsPartial]; <> [refChunk, firstChunkIsPartial] _ FirstChunkToCheck[fpmFileHandle, dumpingWholeFile, size]; IF refChunk # NIL THEN BEGIN IF ((dumpingWholeFile) OR (NOT firstChunkIsPartial) OR (RBTLookupNextLarger[fpmFileHandle, refChunk.startFilePageNumber] # NIL)) THEN ERROR; END; END; END; <> FirstChunkToCheck: INTERNAL SAFE PROCEDURE[fpmFileHandle: FPMFileHandle, dumpingWholeFile: BOOLEAN, newSize: YggEnvironment.PageCount] RETURNS [refChunk: RefChunk, firstChunkIsPartial: BOOLEAN] = CHECKED BEGIN -- non system fatal errors: none. firstPageToClip: YggEnvironment.PageNumber; fileChunkType: YggFilePageMgrPrivateChunk.ClientChunkType; IF dumpingWholeFile THEN RETURN [RBTLookupSmallest[fpmFileHandle], FALSE]; IF RBTLookupLargest[fpmFileHandle] = NIL THEN RETURN[NIL, FALSE]; firstPageToClip _ ChunkStartFilePage[newSize, fileChunkType]; IF (refChunk _ RBTLookup[fpmFileHandle, firstPageToClip]) = NIL THEN refChunk _ RBTLookupNextLarger[fpmFileHandle, firstPageToClip]; firstChunkIsPartial _ ((refChunk # NIL) AND (refChunk.startFilePageNumber # newSize)); END; CheckTheChunks: INTERNAL SAFE PROCEDURE[fpmFileHandle: FPMFileHandle, refChunk: RefChunk, firstChunkIsPartial: BOOLEAN] = CHECKED BEGIN -- non system fatal errors: none. startPageOfChunk: YggEnvironment.PageNumber; DO startPageOfChunk _ refChunk.startFilePageNumber; BEGIN DO IF (refChunk _ RBTLookup[fpmFileHandle, startPageOfChunk]) = NIL THEN GOTO doneWithThisChunk; IF refChunk.state = readInProgress THEN BEGIN WAIT ReadDone; LOOP; END; IF refChunk.useCount # 0 THEN ERROR; IF refChunk.state = writeInProgress THEN BEGIN WAIT WriteDone; LOOP; END; EXIT; ENDLOOP; IF ((refChunk.state = valid) AND (ChunkIsDirty[refChunk])) THEN BEGIN refChunk.state _ writeInProgress; CleanAndWriteChunk[fpmFileHandle: fpmFileHandle, refChunk: refChunk, lockHeld: TRUE]; END; IF NOT firstChunkIsPartial THEN BEGIN YggFilePageMgrLru.GetOurChunkFromLruList[refChunk, FALSE]; UnmapChunk[fpmFileHandle, refChunk]; YggFilePageMgrLru.PutUnmappedChunkOnLruList[refChunk]; END ELSE refChunk.state _ valid; EXITS doneWithThisChunk => NULL; END; firstChunkIsPartial _ FALSE; IF (refChunk _ RBTLookupNextLarger[fpmFileHandle, startPageOfChunk]) = NIL THEN EXIT; ENDLOOP; END; FileExists: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle] RETURNS [fileExists: BOOLEAN] = BEGIN -- non system fatal errors: NoSuchVolume, VolumeWentOffline. errors: ERROR; [errors, fileExists] _ MonitoredFileExists[GetFilePageMgrHandle[fileHandle, FALSE]]; IF errors # Okay THEN ERROR errors; END; MonitoredFileExists: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle] RETURNS [errors: ERROR, fileExists: BOOLEAN] = -- values of errors are {NoSuchVolume, VolumeWentOffline}. BEGIN -- non system fatal errors: none. IF fpmFileHandle.volumeState # online THEN BEGIN errors _ VolumeError[fpmFileHandle.volumeState]; RETURN; END; RETURN[Okay, fpmFileHandle.exists]; END; GetSize: PUBLIC PROCEDURE[fileHandle: YggInternal.FileHandle] RETURNS [size: YggEnvironment.PageCount] = BEGIN -- non system fatal errors: NoSuchFile, NoSuchVolume, VolumeWentOffline. errors: ERROR; [errors, size] _ MonitoredGetDataSize[GetFilePageMgrHandle[fileHandle, FALSE]]; IF errors # Okay THEN ERROR errors; END; MonitoredGetDataSize: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle] RETURNS [errors: ERROR, size: YggEnvironment.PageCount] = -- values of errors are {NoSuchFile, NoSuchVolume, VolumeWentOffline}. BEGIN -- non system fatal errors: none. IF fpmFileHandle.volumeState # online THEN BEGIN errors _ VolumeError[fpmFileHandle.volumeState]; RETURN; END; IF (NOT fpmFileHandle.exists) THEN RETURN[NoSuchFile, 0]; RETURN[Okay, fpmFileHandle.fileDataSize]; END; <> GetKeyProc: RedBlackTree.GetKey -- PROC [data: UserData] RETURNS [Key] = { RETURN[ data ]; }; CompareProc: RedBlackTree.Compare -- PROC [k: Key, data: UserData] RETURNS [Basics.Comparison] = { dataRefChunk: RefChunk = NARROW[ data ]; WITH k SELECT FROM pnRef: REF YggEnvironment.PageNumber => RETURN[Basics.CompareInt[pnRef^, dataRefChunk.startFilePageNumber]]; keyRefChunk: RefChunk => RETURN[Basics.CompareInt[keyRefChunk.startFilePageNumber, dataRefChunk.startFilePageNumber]]; ENDCASE => ERROR; }; GetFilePageMgrHandle: PROCEDURE[fileHandle: YggFileMap.Handle, handleMustExist: BOOLEAN] RETURNS [fpmFileHandle: FPMFileHandle] = BEGIN -- non system fatal errors: none. InitFPMFileHandle: PROCEDURE RETURNS[fpmFileHandle: FPMFileHandle] = BEGIN -- non system fatal errors: none. volumeID: YggEnvironment.VolumeID; volOrVolGroupID: VolOrVolGroupID; fileID: YggEnvironment.FileID; volume: YggDummyFile.Volume; volumeState: YggFilePageMgrPrivateFile.VolumeState _ online; fileExists: BOOLEAN _ TRUE; lowerHandle: YggDummyFile.Handle _ NIL; fileDataSize: YggEnvironment.PageCount; IF handleMustExist THEN ERROR ConsistencyError; [volumeID, fileID] _ YggFileMap.GetVolumeIDAndFileID[fileHandle]; volOrVolGroupID _ volumeID; IF (volume _ YggDummyFile.FindVolumeFromID[volOrVolGroupID]) = NIL THEN BEGIN volumeState _ nonExist; fileExists _ FALSE; END ELSE BEGIN ENABLE YggDummyFile.Error => SELECT why FROM unknownFile, wentOffline => BEGIN fileExists _ FALSE; IF why = wentOffline THEN volumeState _ wentOffline; CONTINUE; END; ENDCASE; lowerHandle _ YggDummyFile.LabeledOpen[volume: volume, fp: fileID]; fileDataSize _ YggDummyFile.Info[lowerHandle].size; END; fpmFileHandle _ NEW[FPMFileObject _ [ chunkTable: RedBlackTree.Create[getKey: GetKeyProc, compare: CompareProc], nMappedChunks: 0, fileDataSize: fileDataSize, exists: fileExists, volumeState: volumeState, nDefWriteChunks: 0, nLruListChunks: 0, lowerHandle: lowerHandle, rbKeyRef: NEW[YggEnvironment.PageNumber _ 0]]]; END; fpmFileHandle _ YggFileMap.VerifyFilePageMgrHandle[fileHandle, InitFPMFileHandle]; END; ChunkAllocator: PUBLIC PROCEDURE[chunkType: YggFilePageMgrPrivateChunk.ChunkType, permanent: BOOLEAN] RETURNS [refChunk: RefChunk] = BEGIN -- non system fatal errors: running out of vm is system fatal. refChunk _ NEW[Chunk]; refChunk^ _ [chunkType: chunkType, defWritePending: FALSE, state: undefined, useCount: 0, fileHandle: NIL, startFilePageNumber: 0, startVMPageNumber: 0, nVMPages: 0, prev: NIL, next: NIL -- , rbColor: , rbLLink: NIL, rbRLink: NIL -- ]; IF (chunkType IN YggFilePageMgrPrivateChunk.ClientChunkType) THEN BEGIN refChunk.nVMPages _ YggFilePageMgrPrivateChunk.ChunkVMPageCount[chunkType]; refChunk.startVMPageNumber _ VM.Allocate[count: refChunk.nVMPages, partition: normalVM, subRange: [0, 0],start: 0, alignment: 0, in64K: FALSE].page; refChunk.nVMPages _ YggFilePageMgrPrivateChunk.ChunkVMPageCount[chunkType]; YggDummyVM.Pin[[refChunk.startVMPageNumber, YggFilePageMgrPrivateChunk.ChunkVMPageCount[chunkType]]]; END; END; <> GetVMIntervalFromFileInterval: PROCEDURE[startFilePageNumber, filePageNumber: YggEnvironment.PageNumber, filePageCount: YggEnvironment.PageCount, startVMPageNumber: VM.PageNumber] RETURNS[vmPageNumber: VM.PageNumber, vmPageCount: VM.PageCount] = BEGIN -- non system fatal errors: none. vmPageNumber _ startVMPageNumber + (filePageNumber - startFilePageNumber); vmPageCount _ filePageCount; END; <> GetFileIntervalFromVMInterval: PROCEDURE[startVMPageNumber, vmPageNumber: VM.PageNumber, vmPageCount: VM.PageCount, startFilePageNumber: YggEnvironment.PageNumber] RETURNS[filePageNumber: YggEnvironment.PageNumber, filePageCount: YggEnvironment.PageCount] = BEGIN -- non system fatal errors: none. filePageNumber _ startFilePageNumber + (vmPageNumber - startVMPageNumber); filePageCount _ vmPageCount; END; ChunkStartFilePage: PROCEDURE[clientFilePage: YggEnvironment.PageNumber, chunkType: YggFilePageMgrPrivateChunk.ClientChunkType] RETURNS[chunkStartFilePage: YggEnvironment.PageNumber] = BEGIN -- non system fatal errors: none. RETURN[ (clientFilePage - Basics.CardMod[clientFilePage, YggFilePageMgrPrivateChunk.ChunkFilePageCount[chunkType]])]; END; VMAddressForPageNumber: PROCEDURE[page: VM.PageNumber] RETURNS [address: LONG POINTER] = -- non system fatal errors: none. TRUSTED BEGIN RETURN[VM.AddressForPageNumber[page]]; END; MakePagesClean: PROCEDURE[interval: VM.Interval] = BEGIN -- non system fatal errors: none. YggDummyVM.MakeUnchanged[interval]; END; MakeAllPagesInChunkClean: PROCEDURE[refChunk: RefChunk] = BEGIN -- non system fatal errors: none. YggDummyVM.MakeUnchanged[[page: refChunk.startVMPageNumber, count: refChunk.nVMPages]]; END; ChunkIsDirty: PUBLIC PROCEDURE[refChunk: RefChunk] RETURNS [dirty: BOOLEAN] = BEGIN -- non system fatal errors: none. FOR vMPage: VM.PageNumber IN [refChunk.startVMPageNumber..refChunk.startVMPageNumber + refChunk.nVMPages) DO IF YggDummyVM.State[vMPage].dataState = changed THEN RETURN[TRUE]; ENDLOOP; RETURN[FALSE]; END; <> DoSequentialIO: PROCEDURE[fpmFileHandle: FPMFileHandle, io: YggFilePageMgrIO.IOType, listOfRefChunk: LIST OF RefChunk, nFilePagesHighestChunkForRead: YggEnvironment.PageCount] = BEGIN -- non system fatal errors: none. controllingProcess: PROCESS _ LOOPHOLE[YggDummyProcess.GetCurrent[]]; iORequest: YggFilePageMgrIO.IORequest; workToDo: BOOLEAN; listOfIOReq: LIST OF YggFilePageMgrIO.IORequest _ NIL; error: YggDummyFile.RC; errorDiskPage: INT; errorIORequest: YggFilePageMgrIO.IORequest; first: BOOLEAN _ TRUE; IF listOfRefChunk = NIL THEN RETURN; IF ((listOfRefChunk.rest # NIL) AND (listOfRefChunk.first.startFilePageNumber < listOfRefChunk.rest.first.startFilePageNumber)) THEN BEGIN tempList: LIST OF RefChunk _ NIL; FOR listOfRefChunk _ listOfRefChunk, listOfRefChunk.rest UNTIL listOfRefChunk = NIL DO tempList _ CONS[listOfRefChunk.first, tempList]; ENDLOOP; listOfRefChunk _ tempList; END; FOR tempList: LIST OF RefChunk _ listOfRefChunk, tempList.rest UNTIL tempList = NIL DO SELECT io FROM read => BEGIN listOfIOReq _ CONS[[filePageNumber: [tempList.first.startFilePageNumber], nPages: (IF first THEN nFilePagesHighestChunkForRead ELSE YggFilePageMgrPrivateChunk.ChunkFilePageCount[tempList.first.chunkType]), vM: VMAddressForPageNumber[tempList.first.startVMPageNumber]], listOfIOReq]; first _ FALSE; END; write => BEGIN AddToCmdList: PROCEDURE[filePageNumber: YggEnvironment.PageNumber, filePageCount: YggEnvironment.PageCount, vmPageNumber: VM.PageNumber] RETURNS[stop: BOOLEAN] = BEGIN-- non system fatal errors: none. listOfIOReq _ CONS[[filePageNumber: [filePageNumber], nPages: filePageCount, vM: VMAddressForPageNumber[vmPageNumber]], listOfIOReq]; stop _ FALSE; END; GetAndCleanNextSeqForWrite[tempList.first, descending, AddToCmdList]; END; ENDCASE; ENDLOOP; IF listOfIOReq = NIL THEN RETURN; iORequest _ YggFilePageMgrIO.RegisterRequest[controllingProcess: controllingProcess, io: io, file: fpmFileHandle.lowerHandle, list: listOfIOReq]; DO YggFilePageMgrIO.DoIO[io, fpmFileHandle.lowerHandle, iORequest ! YggDummyFile.Error => BEGIN error _ why; errorDiskPage _ diskPage; errorIORequest _ iORequest; YggFilePageMgrIO.LogError[controllingProcess, controller, why, iORequest]; GOTO errorSeen; END]; [error, errorIORequest, workToDo, iORequest] _ YggFilePageMgrIO.GetNext[controllingProcess: controllingProcess, who: controller]; IF error # ok THEN GOTO errorSeen; IF workToDo = FALSE THEN EXIT; REPEAT errorSeen => ERROR; -- someday we'll do the right thing. ENDLOOP; IF io = read THEN FOR tempList: LIST OF RefChunk _ listOfRefChunk, tempList.rest UNTIL tempList = NIL DO MakeAllPagesInChunkClean[tempList.first]; ENDLOOP; END; <> CleanAndWriteChunk: PROCEDURE[fpmFileHandle: FPMFileHandle, refChunk: RefChunk, lockHeld: BOOLEAN _ FALSE] = BEGIN -- non system fatal errors: none. WriterProc: PROCEDURE[filePageNumber: YggEnvironment.PageNumber, filePageCount: YggEnvironment.PageCount, vmPageNumber: VM.PageNumber] RETURNS[stop: BOOLEAN] = BEGIN -- non system fatal errors: none. YggFilePageMgrIO.DoIO[write, fpmFileHandle.lowerHandle, [[filePageNumber], filePageCount, VMAddressForPageNumber[vmPageNumber]]]; stop _ FALSE; END; GetAndCleanNextSeqForWrite[refChunk, ascending, WriterProc, lockHeld]; END; GetAndCleanNextSeqForWrite: PROCEDURE[refChunk: RefChunk, whichWay: {ascending, descending}, proc: PROCEDURE[filePageNumber: YggEnvironment.PageNumber, filePageCount: YggEnvironment.PageCount, vmPageNumber: VM.PageNumber] RETURNS[stop: BOOLEAN], lockHeld: BOOLEAN _ FALSE] = BEGIN -- non system fatal errors: any non system fatal errors returned by "proc". filePageNumber: YggEnvironment.PageNumber; filePageCount: YggEnvironment.PageCount; vmPageNumber: VM.PageNumber; vmPageCount: VM.PageCount; startVMPage: VM.PageNumber _ refChunk.startVMPageNumber; endVMPage: VM.PageNumber _ refChunk.startVMPageNumber + refChunk.nVMPages - 1; DO BEGIN IF whichWay = ascending THEN BEGIN IF YggDummyVM.State[startVMPage].dataState = changed THEN BEGIN vmPageNumber _ startVMPage; DO startVMPage _ startVMPage + 1; IF ((startVMPage > endVMPage) OR (YggDummyVM.State[startVMPage].dataState # changed)) THEN EXIT; ENDLOOP; vmPageCount _ startVMPage - vmPageNumber; GOTO doInterval; END; END ELSE BEGIN IF YggDummyVM.State[endVMPage].dataState = changed THEN BEGIN saveEndVMPage: VM.PageNumber _ endVMPage; DO endVMPage _ endVMPage - 1; IF ((startVMPage > endVMPage) OR (YggDummyVM.State[endVMPage].dataState # changed)) THEN EXIT; ENDLOOP; vmPageNumber _ endVMPage + 1; vmPageCount _ saveEndVMPage - endVMPage; GOTO doInterval; END; END; EXITS doInterval => BEGIN [filePageNumber, filePageCount] _ GetFileIntervalFromVMInterval[refChunk.startVMPageNumber, vmPageNumber, vmPageCount, refChunk.startFilePageNumber]; MakePagesClean[[vmPageNumber, vmPageCount]]; IF proc[filePageNumber, filePageCount, vmPageNumber] THEN RETURN; END; END; IF whichWay = ascending THEN startVMPage _ startVMPage + 1 ELSE endVMPage _ endVMPage - 1; IF startVMPage > endVMPage THEN EXIT; ENDLOOP; END; MonitoredWaitForWriteToCompleteThenMaybeSetWIP: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, refChunk: RefChunk] RETURNS[writeNeeded: BOOLEAN]= BEGIN -- non system fatal errors: none. DO IF refChunk.state # writeInProgress THEN EXIT; WAIT WriteDone; ENDLOOP; IF (writeNeeded _ ChunkIsDirty[refChunk]) THEN refChunk.state _ writeInProgress; END; MonitoredSetChunkValidAfterIO: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, refChunk: RefChunk, what: {readCompleted, writeCompleted}] = BEGIN -- non system fatal errors: none. refChunk.state _ valid; IF what = readCompleted -- weird form to keep compiler happy. THEN BROADCAST ReadDone ELSE BROADCAST WriteDone; END; MonitoredSetListOfChunksValidAfterIO: ENTRY PROCEDURE[fpmFileHandle: FPMFileHandle, listOfRefChunk: LIST OF RefChunk, what: {readCompleted, writeCompleted}] = BEGIN -- non system fatal errors: none. FOR list: LIST OF RefChunk _ listOfRefChunk, list.rest UNTIL list = NIL DO list.first.state _ valid; ENDLOOP; IF what = readCompleted -- weird form to keep compiler happy. THEN BROADCAST ReadDone ELSE BROADCAST WriteDone; END; InternalSetListOfChunksValidAfterIOThenMaybeReorder: INTERNAL PROCEDURE[fpmFileHandle: FPMFileHandle, listOfRefChunk: LIST OF RefChunk, what: {readCompleted, writeCompleted}, reorder: BOOLEAN] = BEGIN -- non system fatal errors: none. FOR list: LIST OF RefChunk _ listOfRefChunk, list.rest UNTIL list = NIL DO list.first.state _ valid; IF ((reorder) AND (NOT ChunkIsDirty[list.first]) AND (list.first.useCount = 0)) THEN YggFilePageMgrLru.RelinkChunkAsLruOnLruList[list.first]; ENDLOOP; IF what = readCompleted -- weird form to keep compiler happy. THEN BROADCAST ReadDone ELSE BROADCAST WriteDone; END; <<>> <> RBTLookupProc: TYPE = PROCEDURE[ fpmFileHandle: FPMFileHandle, key: YggEnvironment.PageNumber ] RETURNS [RefChunk]; RBTLookup: RBTLookupProc = { fpmFileHandle.rbKeyRef^ _ key; RETURN[ NARROW[RedBlackTree.Lookup[ fpmFileHandle.chunkTable, fpmFileHandle.rbKeyRef]] ]; }; RBTLookupNextLarger: RBTLookupProc = { fpmFileHandle.rbKeyRef^ _ key; RETURN[ NARROW[RedBlackTree.LookupNextLarger[ fpmFileHandle.chunkTable, fpmFileHandle.rbKeyRef]] ]; }; RBTLookupNextSmaller: RBTLookupProc = { fpmFileHandle.rbKeyRef^ _ key; RETURN[ NARROW[RedBlackTree.LookupNextSmaller[ fpmFileHandle.chunkTable, fpmFileHandle.rbKeyRef]] ]; }; RBTLookupLargest: PROCEDURE[ fpmFileHandle: FPMFileHandle ] RETURNS [RefChunk] = { RETURN[ NARROW[RedBlackTree.LookupLargest[ fpmFileHandle.chunkTable]] ]; }; RBTLookupSmallest: PROCEDURE[ fpmFileHandle: FPMFileHandle ] RETURNS [RefChunk] = { RETURN[ NARROW[RedBlackTree.LookupSmallest[ fpmFileHandle.chunkTable ]] ]; }; RBTDelete: PROCEDURE[ fpmFileHandle: FPMFileHandle, key: YggEnvironment.PageNumber ] RETURNS [RefChunk] = { n: RedBlackTree.Node; fpmFileHandle.rbKeyRef^ _ key; n _ RedBlackTree.Delete[ fpmFileHandle.chunkTable, fpmFileHandle.rbKeyRef]; RETURN[ IF n=NIL THEN NIL ELSE NARROW[n.data] ]; }; RBTInsert: PROCEDURE[ fpmFileHandle: FPMFileHandle, refChunk: RefChunk, key: YggEnvironment.PageNumber ] = { fpmFileHandle.rbKeyRef^ _ key; RedBlackTree.Insert[ fpmFileHandle.chunkTable, refChunk, fpmFileHandle.rbKeyRef ]; }; InitializeFilePageMgr: PUBLIC PROCEDURE[nNormalChunksInCache: NAT, nLogChunksInCache: NAT, checkPointEpoch: NAT] = BEGIN -- non system fatal errors: none. IF moduleInitialized THEN ERROR; YggFilePageMgrLru.InitializeLruLists[[normal: nNormalChunksInCache, log: nLogChunksInCache]]; -- operates inside the lru list monitor. CurrentEpoch _ checkPointEpoch; moduleInitialized _ TRUE; END; <> ConsistencyError: -- CALLING or FPM -- ERROR = CODE; -- caller has asserted that a FPMFileHandle exists or a vmpageset has its usecount positive, but it doesn't. <
> MaxReadAheadSets: NAT = 10; -- to avoid swamping the cache, we won't read ahead more <> <> LimitDefWriteChunks: NAT = 10; -- once this many chunks from a given file are <> moduleInitialized: BOOLEAN _ FALSE; END. Edit Log