-- File: SquirrelLoadImpl.mesa
-- Contents: program to do ascii database input to segment. See NutDumpImpl for file format.
-- Created by Rick Cattell on: August 25, 1980
-- Last edited by
-- Cattell on: June 22, 1983 5:41 pm
-- Maxwell on: June 4, 1982 12:09 pm
-- Willie-Sue on: February 22, 1983 3:57 pm
-- Donahue on: July 8, 1983 3:38 pm
-- Widom on: June 17, 1984 9:24:12 pm PDT
-- Butler on: June 27, 1984 9:53:26 am PDT

DIRECTORY
Atom,
Convert,
DB,
FS,
PrincOpsUtils,
IO,
SquirrelDump,
SquirrelTool USING[stopped],
NutOps USING [SetUpSegment],
NutViewer USING [Message, MessageRope],
Process,
Rope,
BasicTime;

SquirrelLoadImpl: CEDAR PROGRAM
IMPORTS
Atom, BasicTime, Convert, DB, FS, PrincOpsUtils, IO, NutOps, NutViewer, Rope, SquirrelTool
EXPORTS SquirrelDump -- LoadFromFile -- =

BEGIN OPEN DB, IO;

ControlZ: CHARACTER = 032C;

startFlag: BOOLEAN← TRUE;

entityCount, relshipCount, lineCount: INT;

MissingField: SIGNAL = CODE;
CommitLimit: INT= 1024; -- commits database every time this many lines read
autoCommitted: BOOL← FALSE; -- set if more than CommitLimit tuples read
autoCreate: BOOL← TRUE; -- whether to automatically create entities ref'd & when don't exist
defaultSegment: Segment← NIL;

-- Main procedure, exported to SquirrelOps:

LoadFromFile: PUBLIC PROC[dumpFile: ROPE, segment: DB.Segment, DBFile: ROPE] =
-- Loads whatever is in the dumpFile file; uses same dump format as DumpToFile.
TRUSTED BEGIN
ENABLE BEGIN
 ABORTED => {NutViewer.Message[NIL, "Aborting input!\n"]; CONTINUE};
 DB.Error, DB.Failure, DB.Aborted => {NutViewer.Message[NIL, "Transaction Failed; Aborting input!\n"]; CONTINUE}
 END;
start: BasicTime.GMT = BasicTime.Now[];
elapsedTime: INT;
readOnly: BOOLEAN;
BracketProc: IO.BreakProc =
 CHECKED { RETURN[ IF char='] THEN CharClass[break] ELSE CharClass[other] ] };
timeRope, segmentNameInFile: ROPE;
segmentName: ROPE = Atom.GetPName[segment];
lineCount ← entityCount ← relshipCount← 0;
in← FS.StreamOpen[dumpFile];
IF in=NIL THEN {NutViewer.Message[NIL, "Can't open ", dumpFile]; RETURN};
segmentNameInFile ← in.GetTokenRope[].token;
[] ← in.GetTokenRope[]; -- throw away the leading '[
timeRope ← in.GetTokenRope[BracketProc].token;
[] ← in.GetTokenRope[]; -- throw away the trailing ']
[] ← in.GetChar[]; -- and the CR following it
IF NOT Rope.Equal[segmentNameInFile, segmentName] THEN
{ NutViewer.Message[NIL, "You don't want to do this; file was dump of ", segmentNameInFile,
     " segment; you are loading ", segmentName]; RETURN };
NutViewer.Message[NIL, Rope.Cat["Reading segment ", segmentName, " from "],
     dumpFile, "...\n"];
readOnly ← NutOps.SetUpSegment[ DBFile, segment];
IF readOnly THEN GO TO BadSegment;
InputMR[];
elapsedTime ← BasicTime.Period[from: start, to: BasicTime.Now[]];
NutViewer.Message[NIL, PutFR["Load complete: %g entities and %g tuples, %g elapsed seconds.\nSave or Reset updates.\n",
int[entityCount], int[relshipCount], int[elapsedTime]] ];
EXITS BadSegment =>
NutViewer.Message[NIL, "Can't open segment for writing. Aborting..."];
END;

Declared: PROC[ seg: Segment ] RETURNS[ is: BOOLEAN ] = {
FOR sl: LIST OF Segment ← GetSegments[], sl.rest UNTIL sl = NIL DO
IF sl.first = seg THEN RETURN[ TRUE ] ENDLOOP;
RETURN[ FALSE ] };

-- Global variables

in: IO.STREAM;

InputMR: PROC =
TRUSTED BEGIN
ENABLE BEGIN
EndOfStream =>
{NutViewer.MessageRope[NIL, "Unexpected EOF; aborting input...\n"]; GOTO Return};
IO.Error =>
{IllegalFormat["Unrecoverable input error; aborting..."]; GOTO Return};
DB.Failure =>
{IllegalFormat["Alpine problem; aborting input..."]; GOTO Return};
DB.Aborted =>
{IllegalFormat["Transaction abort; aborting input..."]; GOTO Return} END;

UNTIL in.EndOf[] OR SquirrelTool.stopped DO
BEGIN
ENABLE MissingField =>
{IllegalFormat["missing field in dictionary tuple"]; LOOP};
SELECT in.GetChar[] FROM
'/ =>
BEGIN d: ROPE← ReadString[FALSE];
  IF d.Length[]=0 THEN {IllegalFormat["missing domain name"]; LOOP};
  SELECT TRUE FROM -- check for system entity type
  d.Equal["Domain"] =>
  BEGIN
  name: ROPE = ReadString[];
  []← DeclareDomain[name, defaultSegment, NewOrOld, EstimateRefs[name]];
  IF in.GetChar[]#CR THEN IllegalFormat["Missing CR"];
END;
  d.Equal["Relation"] =>
  BEGIN
  name: ROPE = ReadString[];
  []← DeclareRelation[name: name, segment: defaultSegment, version: NewOrOld];
  IF in.GetChar[]#CR THEN IllegalFormat["Missing CR"];
  END;
  d.Equal["Attribute"] =>
  BEGIN
  relationName: ROPE = ReadString[];
  name, typeName: ROPE;
  type: DataType;
  unique: Uniqueness;
  length: INT;
  link: LinkType;
  relation: Relation = FetchEntity[RelationDomain, relationName, defaultSegment];
  IF relation = NIL THEN {IllegalFormat[relationName, " not a relation"]; LOOP};
  name ← ReadString[];
  typeName← ReadString[];
  type← LookupDataType[typeName];
  IF type = NIL THEN {IllegalFormat[typeName, " not a type"]; LOOP};
  unique← ReadEnum[];
  length← ReadNum[];
  link ← ReadEnum[];
  []← DeclareAttribute[relation, name, type, unique, length, link];
  IF in.GetChar[]#CR THEN IllegalFormat["Missing CR"];
  END;
  d.Equal["Index"] =>
  BEGIN rName: ROPE = ReadString[];
  rel: Relation = DeclareRelation[rName, defaultSegment];
  attrList: LIST OF Attribute;
  FOR nextName: ROPE ← ReadString[FALSE], ReadString[FALSE]
  UNTIL Rope.Equal[nextName, ""] DO
  attrList ← CONS[ DeclareAttribute[rel, nextName], attrList ]
  ENDLOOP;
  IF attrList # NIL THEN [] ← DeclareIndex[ rel, attrList, Version[NewOrOld] ];
  IF in.GetChar[]#CR THEN IllegalFormat["Missing CR"];
  END;
  ENDCASE => -- just a normal entity type
  ReadEntity[d];
  NutViewer.MessageRope[NIL, "."];
  END;
'| =>
BEGIN r: ROPE← ReadString[FALSE];
  IF r.Length[]=0 THEN {IllegalFormat["missing relation name"]; LOOP};
  IF r.Equal["SubType"] THEN
  BEGIN
ENABLE DB.Error =>
TRUSTED {IF code=NotFound THEN
    { IllegalFormat["bad subtype: domain does not exist"]; LOOP } };
  ofS: ROPE← ReadName[];
  of: ROPE← ReadString[];
  isS: ROPE← ReadName[];
  is: ROPE← ReadString[];
  IF NOT (ofS.Equal["of"] AND isS.Equal["is"]) THEN
  {IllegalFormat["illegal subtype syntax"]; LOOP};
  DeclareSubType[
  of: DeclareDomain[name: of, segment: defaultSegment, version: OldOnly],
  is: DeclareDomain[name: is, segment: defaultSegment, version: OldOnly]];
  IF in.GetChar[]#CR THEN IllegalFormat["Missing CR"];
  END
  ELSE
  ReadRelship[r];
  NutViewer.MessageRope[NIL, "."];
  END;
ENDCASE =>
{NutViewer.MessageRope[NIL, "-"]; SkipThruCR[]}; -- treat as comment
END ENDLOOP -- of UNTIL in.EndOf[] --;
IF autoCommitted THEN MarkTransaction[TransactionOf[defaultSegment]];
EXITS Return => NULL;
END;

ReadEntity: PROC [dName: ROPE] =
BEGIN
ENABLE MissingField =>
{IllegalFormat["missing name field"]; GOTO Return};
eName: ROPE;
d: Domain← DeclareDomain[dName, defaultSegment, OldOnly ! Error =>
{IllegalFormat[dName, " not a domain"]; GOTO Return}];
e: Entity;
eName← ReadString[];
e← DeclareEntity[d, eName];
IF in.GetChar[]#CR THEN IllegalFormat["Missing CR"];
NextLine[Entity];
EXITS Return => NULL
END;

ReadRelship: PROC [rName: ROPE] =
-- Reads relship of form attr1:val1|attr2:val2|...|attrN:valN| followed by CR.
-- The vals may contain special characters, in the form \nnn where nnn is octal code.
TRUSTED BEGIN
ENABLE MissingField => {IllegalFormat[rName, " tuple has missing field"]; GOTO Return};
r: Relation = DeclareRelation[rName, defaultSegment, OldOnly ];
rel: Relship;
val, fieldName: ROPE;
field: Attribute;
IF r = NIL THEN {IllegalFormat[rName, " not a relation"]; GOTO Return};
rel ← CreateRelship[r];
UNTIL (fieldName ← ReadName[])=NIL DO BEGIN
field← DeclareAttribute[r: r, name: fieldName, version: OldOnly];
IF field=NIL THEN
{IllegalFormat[fieldName, " not an attribute of this relation"]; GOTO Return};
val← ReadString[FALSE];
SetFS[rel, field, val ! DB.Error => TRUSTED {
SELECT code FROM
NotFound => GOTO MissingEntity;
MustSetKeyAttributeFirst =>
{IllegalFormat[fieldName, " attribute out of order"]; GOTO Return};
MultipleMatch, NonUniqueKeyValue =>
{IllegalFormat[val, " value is key stored more than once"]; GOTO Return};
ENDCASE => NULL }];
EXITS MissingEntity =>
IF autoCreate THEN
BEGIN dom: Domain← V2E[GetP[field, aTypeIs]];
NutViewer.Message[NIL, "Automatically creating ref'd entity ", GetName[dom], ": ", val];
IF Eq[dom, AnyDomainType] THEN -- must dissect val into REAL domain and val
BEGIN -- it's of form "<domain>: <name>", set dom and val accordingly
pos: INT← val.Find[":"];
IF pos=-1 THEN
{IllegalFormat["Missing domain in AnyDomainType field"]; GOTO Return};
dom ← DeclareDomain[val.Substr[0, pos], defaultSegment, OldOnly];
IF dom = NIL THEN {IllegalFormat["No such domain"]; GOTO Return};
val← val.Substr[pos+2];
END;
SetF[rel, field, CreateEntity[dom, val]];
END
ELSE
{IllegalFormat[val, " does not exist"]; GOTO Return};
END ENDLOOP;
NextLine[Relship];
EXITS Return => NULL
END;


-- Input support routines

NextLine: PROC [kind: {Entity, Relship}] =
TRUSTED BEGIN ePos: INT;
IF kind=Entity THEN entityCount← entityCount+1 ELSE relshipCount← relshipCount+1;
IF (lineCount← lineCount+1) MOD CommitLimit # 0 THEN RETURN;
autoCommitted← TRUE;
ePos← in.GetIndex[];
NutViewer.MessageRope[NIL, PutFR["\nAutomatic commit at %g: ", int[ePos]]];
DB.MarkTransaction[TransactionOf[defaultSegment]];
END;

IllegalFormat: PROC[msg, arg: ROPE← NIL] =
TRUSTED BEGIN ePos: INT← in.GetIndex[];
NutViewer.MessageRope[NIL, "\nInput error at "];
NutViewer.MessageRope[NIL, PutFR[,int[ePos]]];
IF msg#NIL THEN {NutViewer.MessageRope[NIL, ": "]; NutViewer.MessageRope[NIL, msg]};
IF arg#NIL THEN NutViewer.MessageRope[NIL, arg];
NutViewer.Message[NIL, ];
SkipThruCR[];
END;

SkipThruCR: SAFE PROC = CHECKED
BEGIN
c: CHARACTER;
WHILE (c← in.GetChar[]) # CR DO ENDLOOP;
END;

ReadName: PROC RETURNS[ROPE] =
-- Terminates on reading a ":" or a CR; returns NIL in latter case.
-- Used for reading the attribute names in tuples.
BEGIN
lastBreak: CHAR;
skipping: BOOL ← FALSE;
s: ROPE← in.GetTokenRope[NameBreak].token;
NameBreak: BreakProc = CHECKED {
lastBreak ← char;
IF skipping THEN RETURN[IF char = CR THEN CharClass[break] ELSE CharClass[other]];
IF char=ControlZ THEN {skipping ← TRUE; RETURN[CharClass[other]]};
IF char='| OR char=CR OR char=': THEN RETURN[CharClass[break]];
RETURN[CharClass[other]] };
IF Rope.Equal[s, "|"] OR Rope.Equal[s, "\n"] THEN s← ""
ELSE [] ← in.GetChar[]; -- throw away the break character
IF s.Length[]=0 THEN
IF lastBreak=': THEN ERROR MissingField ELSE RETURN[NIL]
ELSE RETURN[s]
END;

ReadString: PROC[nonNull: BOOLEAN← TRUE] RETURNS[ROPE] =
-- Terminates on reading a "|"; may return a null string if find immediately.
-- Null string generates error if nonNull=TRUE.
-- Used to read attribute values and entity names.
BEGIN
s: ROPE ← in.GetTokenRope[NameBreak].token;
NameBreak: BreakProc = CHECKED {
IF char='| THEN RETURN[CharClass[break]]
ELSE RETURN[CharClass[other]] };
IF Rope.Equal[s, "|"] THEN s← ""
ELSE [] ← in.GetChar[]; -- if didn't hit '| immediately, throw it away
IF s.Find["\\"]#-1 THEN s← Convert.RopeFromLiteral[Rope.Cat["\"", s, "\""]]; -- backslash codes
IF s.Length[]=0 AND nonNull THEN ERROR MissingField
ELSE RETURN[s]
END;

ReadEnum: PROC RETURNS[UNSPECIFIED] =
-- Read a num, return as UNSPECIFIED so can store in enum type
-- currently doesn't check that terminates with '| or that present at all
TRUSTED BEGIN RETURN[PrincOpsUtils.LowHalf[LOOPHOLE[ReadNum[]]]] END;

ReadNum: PROC RETURNS[INT] =
-- currently doesn't check that terminates with '| or that present at all
BEGIN
SlashBreak: BreakProc = CHECKED
 {IF char='| THEN RETURN[ CharClass[break] ] ELSE RETURN[ CharClass[other] ] };
is: ROPE ← in.GetTokenRope[SlashBreak].token;
[] ← in.GetChar[];
RETURN[GetInt[RIS[is]]];
END;

ReadBool: PROC RETURNS[BOOL] =
BEGIN
SlashBreak: BreakProc = CHECKED
 {IF char='| THEN RETURN[ CharClass[break] ] ELSE RETURN[ CharClass[other] ] };
is: ROPE← in.GetTokenRope[SlashBreak].token;
[] ← in.GetChar[];
RETURN[GetBool[RIS[is]]];
END;

LookupDataType: PROC [name: ROPE] RETURNS [DataType] =
BEGIN
SELECT TRUE FROM
name.Equal["IntType"] => RETURN[IntType];
name.Equal["BoolType"] => RETURN[BoolType];
name.Equal["RecordType"] => RETURN[RecordType];
name.Equal["RopeType"] => RETURN[RopeType];
name.Equal["TimeType"] => RETURN[TimeType];
name.Equal["AnyDomainType"] => RETURN[AnyDomainType];
ENDCASE => RETURN[DeclareDomain[name, defaultSegment, OldOnly]];
END;

EstimateRefs: PROC[domain: ROPE] RETURNS [CARDINAL] =
-- What a hack!
BEGIN RETURN[IF Rope.Equal[domain, "Person"] OR Rope.Equal[domain, "Msg"] THEN 10 ELSE 5] END;



END.

Change Log:

March 25, 1982 5:20 pm by Cattell: converted from DBLoad.

October 13, 1982 12:14 pm by Cattell: autoCreate, autoCommit enabled. Added timing of load.

December 1, 1982 6:56 pm by Cattell: ReadRelship now works properly with autoCreate=TRUE when multiple or AnyDomainType undefined entity fields.

Willie-Sue December 13, 1982: aFooProp => aFooIs, for new system properties

Cattell April 19, 1983 1:15 pm: put back in autoCommit someone took out; it is necessary for large file DBLoads!