<> <> <> <<>> << >> DIRECTORY AlpineFile, Ascii, DB, DBCommon, DBDefs, List, Process, Rope, RPC, KeyNoteDatabase; KeyNoteDatabaseImpl: CEDAR PROGRAM IMPORTS AlpineFile, DB, DBCommon, List, Process, Rope, RPC EXPORTS KeyNoteDatabase = { <> ROPE: TYPE = Rope.ROPE; DatabaseHandle: TYPE = REF DatabaseHandleObject; DatabaseHandleObject: PUBLIC TYPE = RECORD [ segment: DB.Segment, fileName: ROPE, readOnly: BOOLEAN, transactionInfo: TransactionInfo ]; TransactionInfo: TYPE = REF TransactionInfoObject; TransactionInfoObject: PRIVATE TYPE = RECORD [ transactionHandle: DB.TransactionHandle, tokenDomain: DB.Domain, fileNameDomain: DB.Domain, tokenFileNameRelation: DB.Relation, fileUniverseRelation: DB.Relation, tokenUniverseRelation: DB.Relation, metaDataRelation: DB.Relation, metaDataRelShip: DB.Relship, tokenFileNameIndex: DB.Index ]; <> tfnrTokenField: CARDINAL = 0; tfnrFileNameField: CARDINAL = 1; tfnrFrequencyField: CARDINAL = 2; turTokenField: CARDINAL = 0; turFrequencyField: CARDINAL = 1; fuFileName: CARDINAL = 0; fuSizeInTokens: CARDINAL = 1; metaDataNumOfTokensField: CARDINAL = 0; metaDataNumOfFilesField: CARDINAL = 1; VersionStatusOfFileToBeChecked: PRIVATE TYPE = {notInDB, inDBasOldVersion, inDBasCurrentVersion}; <> <<>> Error: PUBLIC SIGNAL [ec: KeyNoteDatabase.ErrorCode, explanation: ROPE _ NIL] = CODE; <> <> OpenDatabase: PUBLIC PROC [databaseName: ROPE, readonly: BOOLEAN, oldDatabaseHandle: DatabaseHandle] RETURNS [databaseHandle : DatabaseHandle _ NIL] = { ENABLE { UNWIND => NULL; DBCommon.InternalError, DB.Error, DB.Failure => { GO TO CypressProblems}; }; transHandle: DB.TransactionHandle; schemaInvalid: BOOLEAN; DB.Initialize[nCachePages: 1000]; <> DB.DeclareSegment[filePath: databaseName, segment: $Test, readonly: readonly, createIfNotFound: TRUE, nPagesInitial: 3000, nPagesPerExtent: 256]; [trans: transHandle, schemaInvalid: schemaInvalid] _ DB.OpenTransaction[segment: $Test]; { tokenDomain: DB.Domain _ DB.DeclareDomain["token", $Test, FALSE]; fileNameDomain: DB.Domain _ DB.DeclareDomain["fileName", $Test, FALSE]; TypeProc: PROC[domain: DB.Domain] RETURNS[DB.TypeCode] = { RETURN[DB.TypeForDomain[domain]] }; TokenType: DB.TypeSpec = [direct[TypeProc[tokenDomain]]]; FileNameType: DB.TypeSpec = [direct[TypeProc[fileNameDomain]]]; tokenFileNameType: DB.FieldSpec = DB.L2FS[LIST[ [name: "token", type: TokenType ], [name: "fileName", type: FileNameType ], [name: "frequency", type: DB.Integer] ]]; tokenUniverseType: DB.FieldSpec = DB.L2FS[LIST[ [name: "token", type: TokenType ], [name: "frequency", type: DB.Integer] ]]; fileUniverseType: DB.FieldSpec = DB.L2FS[LIST[ [name: "fileName", type: FileNameType ], [name: "sizeInTokens", type: DB.Integer] ]]; metaDataType: DB.FieldSpec = DB.L2FS[LIST[ [name: "NumOfTokens", type: DB.Integer], [name: "NumOfFiles", type: DB.Integer] ]]; tokenFileNameRelation: DB.Relation _ DB.DeclareRelation[name: "TokenFileName", segment: $Test, fields: tokenFileNameType]; fileUniverseRelation: DB.Relation _ DB.DeclareProperty[name: "FileNameUniverse", segment: $Test, fields: fileUniverseType]; tokenUniverseRelation: DB.Relation _ DB.DeclareProperty[name: "TokenUniverse", segment: $Test, fields: tokenUniverseType]; metaDataRelation: DB.Relation _ DB.DeclareRelation[name: "MetaData", segment: $Test, fields: metaDataType]; valList: LIST OF DBDefs.Value _ LIST[DB.I2V[0], DB.I2V[0]]; valSeq: DB.ValueSequence _ DB.L2VS[vals: valList]; tempRelShip: DB.Relship; metaDataRelShip: DB.Relship _ IF (tempRelShip _ DB.FirstRelship[metaDataRelation])=NIL THEN DB.CreateRelship[r: metaDataRelation, init: valSeq] ELSE tempRelShip; tokenFileNameFieldSequence: DB.FieldSequence = DB.L2F[LIST[tfnrTokenField, tfnrFileNameField]]; tokenFileNameIndex: DB.Index _ DB.DeclareIndex[tokenFileNameRelation, tokenFileNameFieldSequence]; IF oldDatabaseHandle = NIL THEN databaseHandle _ NEW [DatabaseHandleObject _ [segment: $Test, fileName: databaseName, readOnly: readonly, transactionInfo : NEW[ TransactionInfoObject _ [transactionHandle: transHandle, tokenDomain: tokenDomain, fileNameDomain: fileNameDomain, tokenFileNameRelation: tokenFileNameRelation, fileUniverseRelation: fileUniverseRelation, tokenUniverseRelation: tokenUniverseRelation, metaDataRelation: metaDataRelation, metaDataRelShip: metaDataRelShip, tokenFileNameIndex: tokenFileNameIndex]] ] ] ELSE oldDatabaseHandle.transactionInfo _ NEW[ TransactionInfoObject _ [transactionHandle: transHandle, tokenDomain: tokenDomain, fileNameDomain: fileNameDomain, tokenFileNameRelation: tokenFileNameRelation, fileUniverseRelation: fileUniverseRelation, tokenUniverseRelation: tokenUniverseRelation, metaDataRelation: metaDataRelation, metaDataRelShip: metaDataRelShip, tokenFileNameIndex: tokenFileNameIndex]]; }; EXITS CypressProblems => { <> <> SIGNAL Error[$CypressProblems]; }; }; CloseDatabase: PUBLIC PROC [db: DatabaseHandle] = { DB.CloseTransaction[trans: db.transactionInfo.transactionHandle]; }; Abort: PUBLIC PROC [db: DatabaseHandle] = { DB.AbortTransaction[trans: db.transactionInfo.transactionHandle]; }; <> <> AddTokensToDatabase: PUBLIC PROC [db: DatabaseHandle, fileName: ROPE, getTokenProc: KeyNoteDatabase.GetTokenProc, clientData: REF ANY] = { OPEN db.transactionInfo; AddTokensToDatabaseCarefully: PROC = { fne: DB.Entity; fileIncludingVersionNumberAlreadyInDatabase: BOOLEAN; [fne, fileIncludingVersionNumberAlreadyInDatabase] _ GetFileNameEntityAndRemoveOldVersionAsSideEffect[db, fileName]; IF ~ fileIncludingVersionNumberAlreadyInDatabase THEN { fileNameVal: DB.Value _ DB.E2V[fne]; numberOfTokensInFile: INTEGER _ 0; FOR tokenProcReturnData: KeyNoteDatabase.TokenProcReturnData _ getTokenProc[clientData], getTokenProc[tokenProcReturnData.newClientData] WHILE tokenProcReturnData#NIL DO tokenVal: DBDefs.Value _ DB.E2V[GetEntity[db,tokenDomain, tokenProcReturnData.token]]; freqVal: DBDefs.Value _ DB.I2V[tokenProcReturnData.frequency]; valList: LIST OF DBDefs.Value _ LIST[tokenVal, fileNameVal, freqVal]; valSeq: DB.ValueSequence _ DB.L2VS[vals: valList]; tokenUniverseRelShip: DB.Relship _ DB.LookupProperty[tokenUniverseRelation, DB.V2E[tokenVal]]; numberOfTokensInFile _ numberOfTokensInFile +1; <> [] _ DB.CreateRelship[r: tokenFileNameRelation, init: valSeq]; <> <<[must look at old value as we're bumping it up]>> IF tokenUniverseRelShip=NIL THEN { tokenUniverseValList: LIST OF DBDefs.Value _ LIST[tokenVal, DB.I2V[1]]; tokenUniverseValSeq: DB.ValueSequence _ DB.L2VS[vals: tokenUniverseValList]; [] _ DB.CreateRelship[r: tokenUniverseRelation, init: tokenUniverseValSeq]; } ELSE { tokenUniverseFrequency: INT _ DB.V2I[DB.GetF[tokenUniverseRelShip, turFrequencyField]] + 1; DB.SetF[tokenUniverseRelShip, turFrequencyField, DB.I2V[tokenUniverseFrequency] ]; }; ENDLOOP; { valList: LIST OF DBDefs.Value _ LIST[DB.E2V[fne], DB.I2V[numberOfTokensInFile]]; valSeq: DB.ValueSequence _ DB.L2VS[vals: valList]; relShip: DB.Relship _ DB.CreateRelship[r: fileUniverseRelation, init: valSeq]; }; } }; CarefullyApply[AddTokensToDatabaseCarefully, TRUE, db]; }; <> FindFrequencyWordInFile: PUBLIC PROC [db: DatabaseHandle, token: ROPE, fileName: ROPE] RETURNS [frequencyOfWordInFile: INTEGER] = { OPEN db.transactionInfo; FindFrequencyWordInFileCarefully: PROC = { fileNameEntity: DB.Entity _ DB.LookupEntity[fileNameDomain, fileName]; tokenEntity: DB.Entity _ DB.LookupEntity[tokenDomain, token]; constraint: DB.Constraint _ DB.L2C[LIST[ [entity[tokenEntity]], [entity[fileNameEntity]] ] ]; enum: DB.RelshipSet _ DB.RelationSubset[tokenFileNameRelation, tokenFileNameIndex, constraint, First]; rel: DB.Relship = DB.NextRelship[enum]; frequencyOfWordInFile _ DB.V2I[DB.GetF[rel, tfnrFrequencyField]]; }; CarefullyApply[FindFrequencyWordInFileCarefully, FALSE, db]; }; FindFrequencyWordInUniverse: PUBLIC PROC [db: DatabaseHandle, token: ROPE] RETURNS [frequencyOfWordInFile: INTEGER] = { OPEN db.transactionInfo; FindFrequencyWordInUniverseCarefully: PROC = { tokenEntity: DB.Entity _ DB.LookupEntity[tokenDomain, token]; IF tokenEntity=NIL THEN frequencyOfWordInFile _ 0 ELSE { relShip: DB.Relship _ DB.LookupProperty[tokenUniverseRelation, tokenEntity]; frequencyOfWordInFile _ DB.V2I[DB.GetF[relShip, turFrequencyField]]; }; }; CarefullyApply[FindFrequencyWordInUniverseCarefully, FALSE, db]; }; <> FindSizeOfFile: PUBLIC PROC [db: DatabaseHandle, fileName: ROPE] RETURNS [sizeOfFileInPages: INTEGER] = { OPEN db.transactionInfo; FindSizeOfFileCarefully: PROC = { fileEntity: DB.Entity _ DB.LookupEntity[fileNameDomain, fileName]; IF fileEntity=NIL THEN ERROR ELSE { relShip: DB.Relship _ DB.LookupProperty[tokenUniverseRelation, fileEntity]; sizeOfFileInPages _ DB.V2I[DB.GetF[relShip, fuSizeInTokens]]; }; }; CarefullyApply[FindSizeOfFileCarefully, FALSE, db]; }; GetListOfFilesContainingToken: PUBLIC PROC [db: DatabaseHandle, token: ROPE] RETURNS [listOfFilesContainingToken: KeyNoteDatabase.ListOfFilesContainingToken _ NIL] = { OPEN db.transactionInfo; GetListOfFilesContainingTokenCarefully: PROC = { tokenEntity: DB.Entity _ DB.LookupEntity[tokenDomain, token]; relationshipSet: DB.RelshipSet; IF tokenEntity = NIL THEN RETURN; relationshipSet _ DB.RelshipsWithEntityField[r: tokenFileNameRelation, field: tfnrTokenField, val: tokenEntity]; DO frequency: INTEGER; fileSize: INTEGER; fileNameEntity: DB.Entity; fileName: ROPE; relationship: DB.Relship _ DB.NextRelship[relationshipSet]; IF relationship = NIL THEN {DB.ReleaseRelshipSet[relationshipSet]; EXIT}; fileNameEntity _ DB.V2E[DB.GetF[relationship, tfnrFileNameField]]; fileName _ DB.EntityInfo[e: fileNameEntity].name; fileSize _ DB.V2I[DB.GetF[ DB.LookupProperty[fileUniverseRelation, fileNameEntity], fuSizeInTokens]]; frequency _ DB.V2I[DB.GetF[relationship, tfnrFrequencyField]]; listOfFilesContainingToken _ CONS [ NEW[KeyNoteDatabase.FilesContainingTokenObject _ [fileName: fileName, frequency: frequency, fileSize: fileSize]], listOfFilesContainingToken]; ENDLOOP; }; CarefullyApply[GetListOfFilesContainingTokenCarefully, FALSE, db]; }; GetListOfTokensInFile: PUBLIC PROC [db: DatabaseHandle, fileName: ROPE] RETURNS [listOfTokensInFile: KeyNoteDatabase.ListOfTokensInFile _ NIL] = { OPEN db.transactionInfo; GetListOfTokensInFileCarefully: PROC = { fileNameEntity: DB.Entity _ DB.LookupEntity[fileNameDomain, fileName]; relationshipSet: DB.RelshipSet; IF fileNameEntity = NIL THEN RETURN; relationshipSet _ DB.RelshipsWithEntityField[r: tokenFileNameRelation, field: tfnrFileNameField, val: fileNameEntity]; DO frequency: INTEGER; frequencyInUniverse: INTEGER; tokenEntity: DB.Entity; token: ROPE; relationship: DB.Relship _ DB.NextRelship[relationshipSet]; IF relationship = NIL THEN {DB.ReleaseRelshipSet[relationshipSet]; EXIT}; tokenEntity _ DB.V2E[DB.GetF[relationship, tfnrTokenField]]; token _ DB.EntityInfo[e: tokenEntity].name; frequencyInUniverse _ DB.V2I[DB.GetF[ DB.LookupProperty[tokenUniverseRelation, tokenEntity], turFrequencyField]]; frequency _ DB.V2I[DB.GetF[relationship, tfnrFrequencyField]]; listOfTokensInFile _ CONS [ NEW[KeyNoteDatabase.TokensInFileObject _ [token: token, frequency: frequency, frequencyInUniverse: frequencyInUniverse]], listOfTokensInFile]; ENDLOOP; }; CarefullyApply[GetListOfTokensInFileCarefully, FALSE, db]; }; GetNumberOfTokensInUniverse: PUBLIC PROC [db: DatabaseHandle] RETURNS [numberOfTokensInUniverse: INTEGER _ 0] = { OPEN db.transactionInfo; GetNumberOfTokensInUniverseCarefully: PROC = { numberOfTokensInUniverse _ DB.V2I[DB.GetF[metaDataRelShip, metaDataNumOfTokensField]]; }; CarefullyApply[GetNumberOfTokensInUniverseCarefully, FALSE, db]; }; GetNumberOfFilesInUniverse: PUBLIC PROC [db: DatabaseHandle] RETURNS [numberOfFilesInUniverse: INTEGER _ 0] = { OPEN db.transactionInfo; GetNumberOfFilesInUniverseCarefully: PROC = { numberOfFilesInUniverse _ DB.V2I[DB.GetF[metaDataRelShip, metaDataNumOfFilesField]]; }; CarefullyApply[GetNumberOfFilesInUniverseCarefully, FALSE, db]; }; VerifyFileNonExistence: PUBLIC PROC [db: DatabaseHandle, fileName: ROPE] RETURNS [fileNotInDatabase: BOOLEAN] = { OPEN db.transactionInfo; entity: DB.Entity; versionStatusOfFileToBeChecked: VersionStatusOfFileToBeChecked; <> [entity: entity, versionStatusOfFileToBeChecked: versionStatusOfFileToBeChecked] _ VerifyOnenessOrNoneness[db, fileName]; fileNotInDatabase _ IF versionStatusOfFileToBeChecked=inDBasCurrentVersion THEN FALSE ELSE TRUE; }; <> <<>> GetFileNameEntityAndRemoveOldVersionAsSideEffect: PRIVATE PROC [db: DatabaseHandle, fileName: ROPE] RETURNS [entity: DB.Entity, fileIncludingVersionNumberAlreadyInDatabase: BOOLEAN] = { OPEN db.transactionInfo; versionStatusOfFileToBeChecked: VersionStatusOfFileToBeChecked; <> [entity: entity, versionStatusOfFileToBeChecked: versionStatusOfFileToBeChecked] _ VerifyOnenessOrNoneness[db, fileName]; SELECT versionStatusOfFileToBeChecked FROM notInDB => { fileIncludingVersionNumberAlreadyInDatabase _ FALSE; entity _ DeclareEntity[db, fileNameDomain, fileName]; }; inDBasOldVersion => { RetractFileNameFromDatabase[db, DB.EntityInfo[entity].name]; fileIncludingVersionNumberAlreadyInDatabase _ FALSE; entity _ DeclareEntity[db, fileNameDomain, fileName]; }; inDBasCurrentVersion => { fileIncludingVersionNumberAlreadyInDatabase _ TRUE; }; ENDCASE => NULL; }; <> VerifyOnenessOrNoneness: PRIVATE PROC [db: DatabaseHandle, fileName: ROPE] RETURNS [entity: DB.Entity, versionStatusOfFileToBeChecked: VersionStatusOfFileToBeChecked] = { OPEN db.transactionInfo; <> fileNameStrippedOfVersionInfo: ROPE _ Rope.Substr[base: fileName, start: 0, len: Rope.FindBackward[s1: fileName, s2: "!"]]; cursor: DB.EntitySet _ DB.DomainSubset[d: fileNameDomain, lowName: fileNameStrippedOfVersionInfo, highName:Rope.Concat[base: fileNameStrippedOfVersionInfo, rest: Rope.FromChar[Ascii.DEL]]]; oldEntity: DB.Entity _ DB.NextEntity[cursor]; oldFileName: ROPE _ DB.EntityInfo[oldEntity].name; oldFileNameStrippedOfVersionInfo: ROPE _ Rope.Substr[base: oldFileName, start: 0, len: Rope.FindBackward[s1: oldFileName, s2: "!"]]; IF Rope.Equal[s1: fileName, s2: oldFileName] THEN RETURN [oldEntity, inDBasCurrentVersion] ELSE { oldFileNameStrippedOfVersionInfo: ROPE _ Rope.Substr[base: oldFileName, start: 0, len: Rope.FindBackward[s1: oldFileName, s2: "!"]]; IF ~ Rope.Equal[s1: fileNameStrippedOfVersionInfo, s2: oldFileNameStrippedOfVersionInfo] THEN RETURN [NIL, notInDB] ELSE { <> nextFileName: ROPE _ DB.EntityInfo[DB.NextEntity[cursor]].name; nextFileNameStrippedOfVersionInfo: ROPE _ Rope.Substr[base: nextFileName, start: 0, len: Rope.FindBackward[s1: nextFileName, s2: "!"]]; IF Rope.Equal[nextFileNameStrippedOfVersionInfo, oldFileNameStrippedOfVersionInfo] THEN ERROR Error[$MultipleVersions] ELSE RETURN [oldEntity, inDBasOldVersion]; }; }; }; <> RetractFileNameFromDatabase: PRIVATE PROC [db: DatabaseHandle, fileName: ROPE] = { OPEN db.transactionInfo; RetractFromTokenUniverse: PROC [item: REF ANY, list: List.LORA] = { token: ROPE _ NARROW[item, REF KeyNoteDatabase.TokensInFileObject].token; tokenVal: DBDefs.Value _ DB.E2V[DB.LookupEntity[tokenDomain, token]]; tokenUniverseRelShip: DB.Relship _ DB.LookupProperty[tokenUniverseRelation, DB.V2E[tokenVal]]; tokenUniverseFrequency: INT _ DB.V2I[DB.GetF[tokenUniverseRelShip, turFrequencyField]] - 1; DB.SetF[tokenUniverseRelShip, turFrequencyField, DB.I2V[tokenUniverseFrequency] ]; }; listOfTokensInFile: KeyNoteDatabase.ListOfTokensInFile _ GetListOfTokensInFile[db, fileName]; List.Map[list: listOfTokensInFile, proc: RetractFromTokenUniverse]; <> DestroyEntity[db, DB.LookupEntity[fileNameDomain, fileName]]; }; GetEntity: PRIVATE PROC [db: DatabaseHandle, domain: DB.Domain, name: ROPE ] RETURNS [entity: DB.Entity] = { OPEN db.transactionInfo; IF (entity _ DB.LookupEntity[domain, name])#NIL THEN RETURN [entity] ELSE RETURN DeclareEntity[db, domain, name]; }; <> DeclareEntity: PRIVATE PROC [db: DatabaseHandle, domain: DB.Domain, name: ROPE ] RETURNS [entity: DB.Entity] = { OPEN db.transactionInfo; SELECT domain FROM tokenDomain => { DB.SetF[metaDataRelShip, metaDataNumOfTokensField, DB.I2V[DB.V2I[DB.GetF[metaDataRelShip, metaDataNumOfTokensField]] + 1] ]; }; fileNameDomain => { DB.SetF[metaDataRelShip, metaDataNumOfFilesField, DB.I2V[DB.V2I[DB.GetF[metaDataRelShip, metaDataNumOfFilesField]] + 1] ]; }; ENDCASE => NULL; RETURN DB.DeclareEntity[domain, name]; }; <> DestroyEntity: PRIVATE PROC [db: DatabaseHandle, e: DB.Entity ] = { OPEN db.transactionInfo; SELECT DB.EntityInfo[e].domain FROM tokenDomain => { DB.SetF[metaDataRelShip, metaDataNumOfTokensField, DB.I2V[DB.V2I[DB.GetF[metaDataRelShip, metaDataNumOfTokensField]] - 1] ]; }; fileNameDomain => { DB.SetF[metaDataRelShip, metaDataNumOfFilesField, DB.I2V[DB.V2I[DB.GetF[metaDataRelShip, metaDataNumOfFilesField]] - 1] ]; }; ENDCASE => NULL; DB.DestroyEntity[e]; }; <> <<>> <<>> TransRequest: TYPE = { abort, close, continue }; <<>> CarefullyApply: PUBLIC PROC[proc: PROC[], didUpdate: BOOL, db: DatabaseHandle] = { reTryCount: INT _ 10; schemaInvalid: BOOL _ TRUE; DO BEGIN ENABLE DB.Aborted => { IF ( reTryCount _ reTryCount - 1) > 0 THEN GOTO retry; <> REJECT }; < {>> <> <> <<};>> proc[]; IF didUpdate THEN FinishTransaction[db , continue]; RETURN; EXITS retry => NULL; END; FinishTransaction[db , abort]; [] _ OpenDatabase[db.fileName, db.readOnly, db]; <<>> ENDLOOP; }; FinishTransaction: PROC[db: DatabaseHandle, request: TransRequest] = { eCode: ATOM; BEGIN ENABLE BEGIN DB.Aborted => {eCode _ $TransactionAbort; GOTO error }; DB.Error => {eCode _ $DBError; GOTO error }; DB.Failure => {eCode _ $DatabaseInaccessible; GOTO error }; RPC.CallFailed => {eCode _ $RPCCallFailed; GOTO error }; AlpineFile.Unknown => { eCode _ IF what = transID THEN $ServerBusy ELSE $Unknown; GOTO error }; END; SELECT request FROM abort => DB.AbortTransaction[db.transactionInfo.transactionHandle]; continue => { DB.MarkTransaction[db.transactionInfo.transactionHandle]; <> <> }; close => DB.CloseTransaction[db.transactionInfo.transactionHandle]; ENDCASE => Error[$bug, "Bad value passed to FinishTransaction"]; EXITS error => ERROR Error[$db, "Can't access transaction"]; END; }; <<>> Process.CheckForAbort[]; }.