<> <> <> <> DIRECTORY BasicTime, Commander, CommandTool, ComputeServerControllerRpcControl, ComputeServerInternal, ComputeServerRpcControl, ComputeServerControl, ComputeServerServer, ComputeServerStatistics, FSBackdoor, FSFileOps, Idle, IO, LupineRuntime, MessageWindow, PrincOps, Process, ProcessBackdoor, Rope, RPC, SummonerControllerControl, SymTab, SystemVersion, UserCredentials, UserProfile, ViewerClasses, ViewerIO, ViewerOps, VMSideDoor, VMStatistics, WatchStats; ComputeServerMaintenanceImpl: CEDAR MONITOR IMPORTS BasicTime, Commander, CommandTool, ComputeServerControllerRpcControl, ComputeServerInternal, ComputeServerRpcControl, ComputeServerServer, ComputeServerStatistics, FSBackdoor, FSFileOps, Idle, IO, LupineRuntime, MessageWindow, Process, ProcessBackdoor, Rope, RPC, SummonerControllerControl, SymTab, SystemVersion, UserCredentials, UserProfile, ViewerIO, ViewerOps, VMSideDoor, VMStatistics, WatchStats EXPORTS ComputeServerControl, ComputeServerInternal, ComputeServerStatistics SHARES ComputeServerServer = BEGIN <> ROPE: TYPE = Rope.ROPE; TimeUpThisIncarnation: PUBLIC INT _ 0; RequestesThisIncarnation: PUBLIC INT _ 0; TimeBusyThisIncarnation: PUBLIC INT _ 0; TotalTimeUp: PUBLIC INT _ 0; TotalRequestes: PUBLIC INT _ 0; TotalTimeBusy: PUBLIC INT _ 0 ; watchStatsRecord: WatchStats.WatchStatsRecord; LastRMReclamations: INT; LastTime: BasicTime.GMT ; serverMachineName: RPC.ShortROPE; serverMachinePupAddress: RPC.ShortROPE; wasUp: BOOL _ FALSE; PackageEntry: TYPE = ComputeServerInternal.PackageEntry; CmdEntryObject: TYPE = ComputeServerInternal.CmdEntryObject; CmdEntry: TYPE = ComputeServerInternal.CmdEntry; ServiceEnabled: PUBLIC BOOL _ FALSE; IdleTransitionsEnabled: PUBLIC BOOL _ FALSE; <> ServerOn: Commander.CommandProc = { argv: CommandTool.ArgumentVector _ NIL; controllerName, remoteCommandDirectory, localCommandDirectory: Rope.ROPE _ NIL; IF ServiceEnabled THEN RETURN ["Server already enabled"]; argv _ CommandTool.Parse[cmd ! CommandTool.Failed => { msg _ errorMsg; CONTINUE; }]; IF argv = NIL THEN RETURN[$Failure, msg]; IF argv.argc > 1 THEN controllerName _ argv[1]; IF argv.argc > 2 THEN remoteCommandDirectory _ argv[2]; IF argv.argc > 3 THEN localCommandDirectory _ argv[3]; SummonerControllerControl.StartUpController[controllerName]; msg _ StartUpServer[controllerName, remoteCommandDirectory, localCommandDirectory]; IF ~Rope.IsEmpty[msg] THEN SummonerControllerControl.ShutDownController[]; }; StartUpServer: PUBLIC PROC [controllerName: Rope.ROPE _ NIL, remoteCommandDirectory: Rope.ROPE _ NIL, localCommandDirectory: Rope.ROPE _ NIL] RETURNS [msg: ROPE _ NIL] = { name, password: ROPE; [name, password] _ UserCredentials.Get[]; msg _ ComputeServerInternal.InitCommands[remoteCommandDirectory, localCommandDirectory]; IF ~Rope.IsEmpty[msg] THEN RETURN; ServiceEnabled _ TRUE; TRUSTED {Process.Detach[FORK TickStatistics[]]; }; ComputeServerInternal.ControllerGVName _ IF Rope.IsEmpty[controllerName] THEN UserProfile.Token["Summoner.ControllerName"] ELSE controllerName; IF ComputeServerInternal.ControllerGVName = NIL THEN ComputeServerInternal.ControllerGVName _ "PaloAlto.summoner" ; ComputeServerRpcControl.ExportInterface[ interfaceName: [ type: "ComputeServer.summoner", instance: ComputeServerInternal.MyNetAddressRope, version: [1,1]], user: name, password: RPC.MakeKey[password] ! LupineRuntime.BindingError => CONTINUE;]; ComputeServerInternal.ControllerInterface _ ComputeServerControllerRpcControl.ImportNewInterface[ interfaceName: [ type: "ComputeServerController.summoner", instance: ComputeServerInternal.ControllerGVName, version: [1,1]] ! RPC.ImportFailed => { SummonerControllerControl.ControllerCannnotBeImported[why]; CONTINUE; }; ]; IF ComputeServerInternal.ControllerInterface # NIL THEN { SummonerControllerControl.NoticeControllerIsUp[]; TellControllerAboutExtraCommands[]; }; ComputeServerInternal.InitStats[ComputeServerInternal.myHostName, ComputeServerInternal.MyNetAddressRope]; TRUSTED {Process.Detach[FORK ComputeServerInternal.SummonerServerMonitor[]]; }; }; ServerOff: Commander.CommandProc = { ShutDownServer[]; SummonerControllerControl.ShutDownController[]; }; ShutDownServer: PUBLIC PROC [] RETURNS [] = { ComputeServerInternal.ControllerInterface _ NIL ; ServiceEnabled _ FALSE; ComputeServerInternal.PackageTable _ SymTab.Create[mod: 59, case: FALSE]; ComputeServerInternal.ConfigTable _ SymTab.Create[mod: 59, case: FALSE]; }; DoStartup: PROC = { msg: ROPE; SummonerControllerControl.StartUpController[ComputeServerInternal.ControllerGVName]; msg _ StartUpServer[ComputeServerInternal.ControllerGVName, ComputeServerInternal.RemoteCommandDir, ComputeServerInternal.LocalCommandDir]; IF ~Rope.IsEmpty[msg] THEN { SummonerControllerControl.ShutDownController[]; MessageWindow.Append[message: Rope.Concat["Can't Start Summoner Server because ", msg], clearFirst: TRUE]; }; }; IdleTransition: Idle.IdleHandler = { <> IF IdleTransitionsEnabled THEN { SELECT reason FROM becomingBusy => { ShutDownServer[]; SummonerControllerControl.ShutDownController[]; }; becomingIdle => { TRUSTED {Process.Detach[FORK DoStartup[]];}; }; ENDCASE; }; }; EnableAutoIdle: Commander.CommandProc = { IdleTransitionsEnabled _ TRUE; }; DisableAutoIdle: Commander.CommandProc = { IdleTransitionsEnabled _ FALSE; }; <> SendStats: PUBLIC PROC [] RETURNS [success, terminateService, newPackage: BOOL _ FALSE, queueingCommands: LIST OF Rope.ROPE] = { <> controllerInterface: ComputeServerControllerRpcControl.InterfaceRecord; numberCPUs: CARDINAL _ 1; diskPartionSize: INT; freePagesOnDisk: INT; freeboard: INT; oldestLRUFileDate: BasicTime.GMT; freeProcesses: CARDINAL _ 0; NowRMReclamations: INT; ReclamationRate: REAL; NowTime: BasicTime.GMT = BasicTime.Now[]; TimePeriod: REAL; firstCall: BOOL; [diskPartionSize, freePagesOnDisk, freeboard] _ FSBackdoor.VolumePages[]; <> watchStatsRecord _ WatchStats.GetWatchStats[]; NowRMReclamations _ VMStatistics.rmReclamations; TimePeriod _ BasicTime.Period[LastTime, NowTime]; TimePeriod _ MAX[1.0, TimePeriod]; IF LastRMReclamations = 0 THEN ReclamationRate _ -1.0 ELSE ReclamationRate _ (NowRMReclamations - LastRMReclamations) / TimePeriod ; LastRMReclamations _ NowRMReclamations; oldestLRUFileDate _ FSFileOps.OldestLruDate[]; <> freeProcesses _ ProcessBackdoor.CountFreeProcesses[]; <> <> <> <> <> <> <> <> <> <> <> <> <> <<};>> <> firstCall _ ~wasUp AND ServiceEnabled; controllerInterface _ ComputeServerInternal.ControllerInterface; IF controllerInterface # NIL THEN [terminateService, newPackage, queueingCommands] _ controllerInterface.NewStats[serverMachineName: serverMachineName, serverMachinePupAddress: serverMachinePupAddress, serverUP: ServiceEnabled, firstCall: firstCall, machineType: SystemVersion.machineType, mainMemory: VMSideDoor.rmPages, numberCPUs: numberCPUs, diskPartionSize: diskPartionSize, freePagesOnDisk: freePagesOnDisk, freeboard: freeboard, freeGFI: watchStatsRecord.gfiFree, freeMDS: watchStatsRecord.mdsFree, freeVM: watchStatsRecord.vmFree, oldestLRUFileDate: oldestLRUFileDate, CPULoad: watchStatsRecord.cpuLoad, reclamationRate: ReclamationRate, freeProcesses: freeProcesses ]; success _ TRUE; wasUp _ ServiceEnabled; }; <> SummonerServerMonitor: PUBLIC PROC [] RETURNS [] = { delayToNextStat: INT _ 2; WHILE ServiceEnabled DO success: BOOL _ TRUE; terminateService: BOOL _ FALSE; newPackage: BOOL _ FALSE; queueingCommands: LIST OF ROPE; cycleCount: INT _ 0; tryDifferentController: BOOL _ FALSE; WHILE ~tryDifferentController AND ComputeServerInternal.ControllerInterface # NIL AND success AND NOT terminateService AND ServiceEnabled DO Process.Pause[Process.SecondsToTicks[delayToNextStat]]; [success, terminateService, newPackage, queueingCommands] _ ComputeServerInternal.SendStats[ ! RPC.CallFailed => { SummonerControllerControl.ControllerCallFailed[why]; ComputeServerInternal.ControllerInterface _ NIL ; newPackage _ FALSE; queueingCommands _ NIL; CONTINUE; }; ]; IF newPackage THEN { ComputeServerInternal.getNewPackages[]; }; IF queueingCommands # NIL THEN newQueueingCommands[queueingCommands]; newPackage _ FALSE; delayToNextStat _ 9; IF (cycleCount _ cycleCount + 1) > 10 THEN { cycleCount _ 0; ComputeServerInternal.KillOldUnstartedServices[]; }; IF cycleCount = 1 OR cycleCount = 6 THEN { controllerInterface: ComputeServerControllerRpcControl.InterfaceRecord _ ComputeServerInternal.ControllerInterface; IF controllerInterface # NIL THEN { [tryDifferentController: tryDifferentController] _ controllerInterface.GetSomeInfo[ ! RPC.CallFailed => { SummonerControllerControl.ControllerCallFailed[why]; ComputeServerInternal.ControllerInterface _ NIL ; newPackage _ FALSE; queueingCommands _ NIL; CONTINUE; }; ]; }; }; ENDLOOP; <> IF ~tryDifferentController AND ComputeServerInternal.ControllerInterface # NIL AND ~ServiceEnabled THEN [success, terminateService] _ ComputeServerInternal.SendStats[ ! RPC.CallFailed => { SummonerControllerControl.ControllerCallFailed[why]; ComputeServerInternal.ControllerInterface _ NIL ; CONTINUE; }; ]; IF terminateService THEN ServiceEnabled _ FALSE; IF ServiceEnabled AND (tryDifferentController OR ComputeServerInternal.ControllerInterface = NIL) THEN ComputeServerInternal.ControllerInterface _ ComputeServerControllerRpcControl.ImportNewInterface[ interfaceName: [ type: "ComputeServerController.summoner", instance: ComputeServerInternal.ControllerGVName, version: [1,1]] ! RPC.ImportFailed => { SummonerControllerControl.ControllerCannnotBeImported[why]; CONTINUE; };]; IF ComputeServerInternal.ControllerInterface # NIL THEN { SummonerControllerControl.NoticeControllerIsUp[]; TellControllerAboutExtraCommands[]; }; ComputeServerInternal.InitStats[ComputeServerInternal.myHostName, ComputeServerInternal.MyNetAddressRope]; ComputeServerInternal.KillOldUnstartedServices[]; IF ComputeServerInternal.ControllerInterface = NIL THEN Process.Pause[Process.SecondsToTicks[7]]; ENDLOOP; }; <> newQueueingCommands: PROC [queueingCommands: LIST OF ROPE] = { FOR q: LIST OF ROPE _ queueingCommands, q.rest UNTIL q = NIL DO foundInCmdTable: BOOL; cmdVal: REF ANY; cmdEntry: CmdEntry; [found: foundInCmdTable, val: cmdVal] _ SymTab.Fetch[x: ComputeServerInternal.CommandTable, key: q.first]; cmdEntry _ NARROW[cmdVal]; IF cmdEntry # NIL THEN { foundPack: BOOL; valPack: REF ANY; packageEntry: PackageEntry; cmdEntry.doQueueing _ TRUE; [found: foundPack, val: valPack] _ SymTab.Fetch[x: ComputeServerInternal.PackageTable, key: cmdEntry.package]; IF ~foundPack THEN LOOP; packageEntry _ NARROW[valPack, PackageEntry]; IF packageEntry.exclusive AND ComputeServerInternal.ActiveServices > 0 THEN LOOP; IF packageEntry.nowActive < packageEntry.maxCountActive THEN { controllerInterface: ComputeServerControllerRpcControl.InterfaceRecord; controllerInterface _ ComputeServerInternal.ControllerInterface; IF controllerInterface # NIL THEN { controllerInterface.MightAcceptQueuedCommand[serverMachineAddress: ComputeServerInternal.MyNetAddressRope, commandName: q.first ! RPC.CallFailed => { SummonerControllerControl.ControllerCallFailed[why]; ComputeServerInternal.ControllerInterface _ NIL ; CONTINUE; }; ]; cmdEntry.okToQueuePosted _ TRUE; }; }; }; ENDLOOP; }; <> TellControllerAboutExtraCommands: PROC = { procData: LIST OF ComputeServerServer.RegisteredProcHandle; EachPairProc: SymTab.EachPairAction = { loopList: LIST OF ComputeServerInternal.RegisteredProcHandle; procData _ NARROW[val]; FOR loopList _ procData, loopList.rest UNTIL loopList = NIL DO procHandle: ComputeServerInternal.RegisteredProcHandle _ loopList.first; foundCmdTab: BOOL; tempControllerInterface: ComputeServerControllerRpcControl.InterfaceRecord _ NIL; [found: foundCmdTab] _ SymTab.Fetch[x: ComputeServerInternal.CommandTable, key: procHandle.service]; IF ~foundCmdTab AND (tempControllerInterface _ ComputeServerInternal.ControllerInterface) # NIL THEN { tempControllerInterface.ExtraCommandAvailable[serverMachineName: ComputeServerInternal.MyNetAddressRope, commandName: procHandle.service, version: procHandle.version ! RPC.CallFailed => { SummonerControllerControl.ControllerCallFailed[why]; ComputeServerInternal.ControllerInterface _ NIL ; CONTINUE; }; ]; }; ENDLOOP; }; [] _ SymTab.Pairs[x: ComputeServerServer.Registry, action: EachPairProc]; }; <> InitStats: PUBLIC PROC [myName: RPC.ShortROPE, myAddress: RPC.ShortROPE] = { serverMachineName _ myName ; serverMachinePupAddress _ myAddress; LastRMReclamations _ VMStatistics.rmReclamations; LastTime _ BasicTime.Now[]; }; <> TickStatistics: PROC [] = { lastStatTime: BasicTime.GMT _ BasicTime.Now[]; aSecond: Process.Ticks = Process.SecondsToTicks[1]; WHILE ServiceEnabled DO nowTime: BasicTime.GMT _ BasicTime.Now[]; period: INT; period _ BasicTime.Period[lastStatTime, nowTime]; IF period > 0 THEN { TimeUpThisIncarnation _ TimeUpThisIncarnation + period; TotalTimeUp _ TotalTimeUp + period; IF ComputeServerInternal.ActiveServices > 0 THEN { TimeBusyThisIncarnation _ TimeBusyThisIncarnation + period; TotalTimeBusy _ TotalTimeBusy + period; }; }; lastStatTime _ nowTime; Process.Pause[aSecond]; ENDLOOP; TimeUpThisIncarnation _ 0; RequestesThisIncarnation _ 0; TimeBusyThisIncarnation _ 0; }; ServerStatistics: Commander.CommandProc = { IF ServiceEnabled THEN { percentBusy: REAL; realTimeBusyThisIncarnation: REAL _ TimeBusyThisIncarnation; realTimeUpThisIncarnation: REAL _ TimeUpThisIncarnation; percentBusy _ (100.0 * realTimeBusyThisIncarnation) / MAX[1.0, realTimeUpThisIncarnation]; msg _ IO.PutFR["Seconds up %g, requests %g, time busy %g, percent busy %5.2f%%", IO.int[TimeUpThisIncarnation], IO.int[RequestesThisIncarnation], IO.int[TimeBusyThisIncarnation], IO.real[percentBusy] ] } ELSE msg _ "Server not enabled"; }; <> StartServerEventTypescript: Commander.CommandProc = { TRUSTED {Process.Detach[FORK ServerEventTypescript[]];}; }; ServerEventTypescript: PROC = { ENABLE IO.Error => GOTO ViewerDestroyed; currentViewer: ViewerClasses.Viewer _ NIL; currentStream: IO.STREAM _ NIL; currentEvent: REF READONLY ComputeServerStatistics.ServerEvent _ NIL; currentViewer _ ViewerOps.CreateViewer[flavor: $Typescript, info: [iconic: FALSE, name: "Compute Server ""Server"" Event Log"]]; currentStream _ ViewerIO.CreateViewerStreams[name: "", viewer: currentViewer].out; currentStream.PutF["Starting log at %g\n", IO.time[BasicTime.Now[]]]; DO currentEvent _ ComputeServerStatistics.NextServerEvent[currentEvent]; SELECT currentEvent.type FROM startService => { currentStream.PutF["Start %g for %g at %g\n", IO.rope[currentEvent.command], IO.rope[currentEvent.remoteMachineName], IO.time[currentEvent.startTime]]; }; doneService => { currentStream.PutF["End %g for %g at %g duration %g seconds\n", IO.rope[currentEvent.command], IO.rope[currentEvent.remoteMachineName], IO.time[currentEvent.endTime], IO.int[BasicTime.Period[currentEvent.startTime, currentEvent.endTime]]]; }; ENDCASE; ENDLOOP; EXITS ViewerDestroyed => {}; }; <> Commander.Register[key: "///Commands/SummonerServerOn", proc: ServerOn, doc: "SummonerServerOn {controllerName {remoteCommandDirectory {localCommandDirectory}}} Turn on use of the Compute Server Service for cluster controllerName using remoteCommandDirectory on a file server and localCommandDirectory on the workstation"]; Commander.Register[key: "///Commands/SummonerServerOff", proc: ServerOff, doc: "Turn off use of the Compute Server Service"]; Commander.Register[key: "///Commands/SummonerEnableAutoIdle", proc: EnableAutoIdle, doc: "Turn on automatic Summoner server toggle at Idle"]; Commander.Register[key: "///Commands/SummonerDisableAutoIdle", proc: DisableAutoIdle, doc: "Turn off automatic Summoner server toggle at Idle"]; Commander.Register[key: "///Commands/SummonerServerTypescript", proc: StartServerEventTypescript, doc: "Start typescript of server events"]; Commander.Register[key: "///Commands/SummonerServerStatistics", proc: ServerStatistics, doc: "Print statistics"]; [] _ Idle.RegisterIdleHandler[IdleTransition]; END. <> <> <> <> <> <<>>