Atom USING [MakeAtom],
NameSymbolTable USING [MakeName],
NodeProps USING [DoSpecs, GetProp, PutProp],
CreateNode,
Rope USING [ROPE],
RopeEdit USING [Substr],
RopeReader USING [Backwards, BumpIndex, FreeRopeReader, Get, GetIndex, GetRopeReader, GetString, Peek, Position, Ref, SetIndex, SetPosition],
T2FileOps,
TiogaLooks USING [BaseRuns, Look, Looks, noLooks, Runs],
TiogaLooksOps USING [Concat],
TiogaLooksSupport USING [Short, NewBase],
TiogaBasicClass USING [BasicClass],
TiogaBranchClass USING [BranchClass, branchChildrenAtom, branchContentsAtom, defaultBranchClass, Lookup],
TiogaItemClass USING [ItemClass],
TiogaNode USING [BasicClassID, ItemClassID, Name, Offset, Ref, RefBranchNode, RefItemNode, RefTextNode, RefBoxNode, RefBasicNode, RefListNode],
TiogaNodeOps USING [FetchBasicClass, FetchItemClass, LookupBasicID, LookupItemID, NewBasicNode, NewBranchNode, NewBoxNode, NewListNode, NewTextNode],
TiogaTreeOps USING [LastSibling, Parent];
CreateInternalRep:
PUBLIC PROC [br: TiogaNode.RefBranchNode] = {
The primary client interface to the tree parser
brInfo: SpanInfo ← NARROW[NodeProps.GetProp[br, $BranchInfo]];
copy: SpanInfo;
textBuffer: REF TEXT;
control: RopeReader.Ref ← RopeReader.GetRopeReader[];
text: RopeReader.Ref ← RopeReader.GetRopeReader[];
wholeTree: BOOL ← TRUE;
parent: TiogaNode.Ref;
IF brInfo=NIL THEN ERROR noBranchParseInfo;
IF (parent ← TiogaTreeOps.Parent[br])=
NIL
OR (parent ← TiogaTreeOps.Parent[parent])=
NIL
OR (parent ← TiogaTreeOps.Parent[parent])=NIL THEN wholeTree ← FALSE;
If at least 3 levels deep, do the whole thing. Else keep on doing incremental parsing.
Needless to say, this heuristic can be changed if a better one comes along.
[textBuffer, copy] ← GetStuff[wholeTree];
IF wholeTree
THEN { copy^ ← brInfo^; brInfo ← copy };
If the whole tree is to be cracked then there is no need to plant information blocks on each branch. Instead, a copy of the initial branch info is created and passed around the tree, the information being modified to suit the branch about to be cracked.
[] ← CIR[br, brInfo, wholeTree, textBuffer, control, text];
RopeReader.FreeRopeReader[control];
RopeReader.FreeRopeReader[text];
FreeStuff[textBuffer, copy] };
CIR:
PROC [br: TiogaNode.RefBranchNode, brInfo: SpanInfo,
wholeTree: BOOL, textBuffer: REF TEXT, control, text: RopeReader.Ref]
RETURNS [node: TiogaNode.Ref] = {
NextOp:
PROC
RETURNS [T2FileOps.Op] =
INLINE {
RETURN[RopeReader.Get[control]] };
ReadB32:
PROCEDURE []
RETURNS [n:
INT] = {
Read a 24 bit number packed as 1..4 bytes using the given character reader. Each character provides 7 data bits and a boolean indicating whether or not it is the last in the sequence.
first, second, fourth: T2FileOps.LengthByte;
third: T2FileOps.ThirdByte;
card: T2FileOps.IntBytes;
first ← LOOPHOLE[RopeReader.Backwards[control]];
card.first ← first.data;
IF ~first.others THEN RETURN [LOOPHOLE[card]]; -- value within 7 bits so terminate
second ← LOOPHOLE[RopeReader.Backwards[control]];
card.second ← second.data;
IF ~second.others THEN RETURN [LOOPHOLE[card]]; -- value within 14 bits so terminate
third ← LOOPHOLE[RopeReader.Backwards[control]];
card.thirdBottom ← third.dataBottom;
card.thirdTop ← third.dataTop;
IF ~third.others THEN RETURN [LOOPHOLE[card]]; -- value within 21 bits so terminate
fourth ← LOOPHOLE[RopeReader.Backwards[control]];
card.fourth ← fourth.data;
RETURN [LOOPHOLE[card]] }; -- full 28 bit value
ReadF32:
PROCEDURE []
RETURNS [n:
INT] = {
Read a 24 bit number packed as 1..4 bytes using the given character reader. Each character provides 7 data bits and a boolean indicating whether or not it is the last in the sequence.
first, second, fourth: T2FileOps.LengthByte;
third: T2FileOps.ThirdByte;
card: T2FileOps.IntBytes;
first ← LOOPHOLE[RopeReader.Get[control]];
card.first ← first.data;
IF ~first.others THEN RETURN [LOOPHOLE[card]]; -- value within 7 bits so terminate
second ← LOOPHOLE[RopeReader.Get[control]];
card.second ← second.data;
IF ~second.others THEN RETURN [LOOPHOLE[card]]; -- value within 14 bits so terminate
third ← LOOPHOLE[RopeReader.Get[control]];
card.thirdBottom ← third.dataBottom;
card.thirdTop ← third.dataTop;
IF ~third.others THEN RETURN [LOOPHOLE[card]]; -- value within 21 bits so terminate
fourth ← LOOPHOLE[RopeReader.Get[control]];
card.fourth ← fourth.data;
RETURN [LOOPHOLE[card]] }; -- full 28 bit value
ReadAtom:
PROCEDURE []
RETURNS [
ATOM] =
INLINE {
Change this to be like ReadFormat. Want to avoid the NEW ROPE unless it is a new atom.
RETURN[Atom.MakeAtom[ReadRope[ReadF32[], control]]] };
ReadFormat:
PROC
RETURNS [n: TiogaNode.Name] = {
len: TiogaNode.Offset ← ReadF32[];
IF len>LAST[NAT] THEN ERROR; -- too big for a format name
IF len > textBuffer.maxLength THEN textBuffer ← NEW[TEXT[len]];
textBuffer.length ← 0;
textBuffer.length ← RopeReader.GetString[control, textBuffer, len];
n ← NameSymbolTable.MakeName[textBuffer] };
ReadContents:
PROCEDURE [parent: TiogaNode.Ref, itemClassInfo: TiogaItemClass.ItemClass ←
NIL]
RETURNS [head: TiogaNode.Ref] = {
previous, node: TiogaNode.Ref;
IF op=T2FileOps.
STARTBR
THEN
DO
-- branches
current: TiogaNode.RefBranchNode ← TiogaNodeOps.NewBranchNode[];
IF previous=NIL THEN head ← current ELSE { previous.next ← current; previous.last ← FALSE };
previous ← current;
[] ← CIR[current, NIL, TRUE, textBuffer, control, text];
SELECT RopeReader.Peek[control]
FROM
T2FileOps.
NODATA, T2FileOps.
CLASSDATA, T2FileOps.
ENDOFFILE => {
[] ← NextOp[]; current.next ← parent; RETURN };
ENDCASE;
ENDLOOP;
IF op=T2FileOps.
ITEMCONTENTSPECS
THEN {
-- itemClassInfo supplied by caller
Insert: PROC [n: TiogaNode.Ref, previous: TiogaNode.Ref ← NIL] = {
IF head=NIL THEN head ← n
ELSE {
IF previous=NIL THEN previous ← TiogaTreeOps.LastSibling[head];
previous.next ← n; previous.last ← FALSE };
node ← n; n.last ← TRUE };
rope: Rope.ROPE;
start, len: INT;
[rope, start, len] ← ReadSpecs[];
op ← NextOp[]; -- the op after the specs
IF itemClassInfo.getContents=NIL THEN -- trouble. save the specs
NodeProps.PutProp[parent, TiogaItemClass.itemContentsAtom, RopeEdit.Substr[rope, start, len]]
ELSE itemClassInfo.getContents[TiogaNodeOps.NarrowToItemNode[parent], rope, start, len, Insert] }
ERROR } -- put in the real thing once add getContents to item class
ELSE
DO
SELECT op
FROM
>= T2FileOps.text =>
SELECT op
FROM
-- text, list, box, or basic
<= T2FileOps.lastText => node ← ReadTextNode[op-T2FileOps.text];
<= T2FileOps.lastList => node ← ReadNonTextNode[op-T2FileOps.list, list];
<= T2FileOps.lastBox => node ← ReadNonTextNode[op-T2FileOps.box, box];
<= T2FileOps.lastBasic => node ← ReadNonTextNode[op-T2FileOps.basic, basic];
ENDCASE => EXIT;
ENDCASE => EXIT;
IF previous=NIL THEN head ← node
ELSE { previous.next ← node; previous.last ← FALSE };
previous ← node;
op ← NextOp[];
ENDLOOP;
IF node#NIL THEN node.next ← parent };
KindOfNonTextNode:
TYPE = { box, list, basic };
ReadNonTextNode:
PROCEDURE [offset:
CARDINAL, kind: KindOfNonTextNode]
RETURNS [new: TiogaNode.Ref] = {
isComment, doFmt, doClass: BOOL ← FALSE;
itemClassID: TiogaNode.ItemClassID;
itemClassInfo: TiogaItemClass.ItemClass;
basicClassID: TiogaNode.BasicClassID;
classData: Rope.ROPE;
ls: TiogaNode.RefListNode;
bx: TiogaNode.RefBoxNode;
bs: TiogaNode.RefBasicNode;
{ OPEN T2FileOps;
IF offset >= commentBit*2 THEN ERROR fileFormatError;
IF offset >= commentBit THEN { isComment ← TRUE; offset ← offset-commentBit };
IF offset >= formatBit THEN { doFmt ← TRUE; offset ← offset-formatBit };
IF offset > 0 THEN doClass ← TRUE };
SELECT kind
FROM
box => {
new ← bx ← TiogaNodeOps.NewBoxNode[
itemClassID ← TiogaNodeOps.LookupItemID[IF doClass THEN ReadAtom[] ELSE $Box, TRUE]];
itemClassInfo ← TiogaNodeOps.FetchItemClass[itemClassID] };
list => {
new ← ls ← TiogaNodeOps.NewListNode[
itemClassID ← TiogaNodeOps.LookupItemID[IF doClass THEN ReadAtom[] ELSE $List, TRUE]];
itemClassInfo ← TiogaNodeOps.FetchItemClass[itemClassID] };
basic => new ← bs ← TiogaNodeOps.NewBasicNode[
basicClassID ← TiogaNodeOps.LookupBasicID[IF doClass THEN ReadAtom[] ELSE $Basic, TRUE]];
ENDCASE => ERROR;
new.comment ← isComment;
IF doFmt THEN new.format ← ReadFormat[];
op ← NextOp[];
ReadProperties[new];
SELECT kind
FROM
box => bx.contents ← ReadContents[bx, itemClassInfo];
list => ls.contents ← ReadContents[ls, itemClassInfo];
basic => NULL;
ENDCASE => ERROR;
SELECT op
FROM
T2FileOps.NODATA => NULL;
T2FileOps.CLASSDATA => classData ← ReadRope[ReadF32[], control];
ENDCASE => ERROR;
SELECT kind
FROM
box => {
IF itemClassInfo.set=
NIL
THEN bx.data ← classData
ELSE
itemClassInfo.set[bx, $Restore, classData, FALSE] };
list => {
IF itemClassInfo.set=
NIL
THEN ls.data ← classData
ELSE
itemClassInfo.set[ls, $Restore, classData, FALSE] };
basic => {
basicClassInfo: TiogaBasicClass.BasicClass ← TiogaNodeOps.FetchBasicClass[basicClassID];
IF basicClassInfo.set=
NIL
THEN bs.data ← classData
ELSE
basicClassInfo.set[bs, $Restore, classData, FALSE] };
ENDCASE => ERROR };
ReadSpecs:
PROCEDURE
RETURNS [rope: Rope.
ROPE, pos, len:
INT] = {
len ← ReadF32[];
[rope, pos] ← RopeReader.Position[control];
RopeReader.SetIndex[control, pos+len] };
ReadProp:
PROCEDURE [n: TiogaNode.Ref] = {
key: ATOM ← ReadAtom[];
rope: Rope.ROPE;
len, pos: INT;
val: REF;
[rope, pos, len] ← ReadSpecs[];
val ← NodeProps.DoSpecs[key, rope, pos, len, n];
IF val # NIL THEN NodeProps.PutProp[n, key, val] };
ReadRope:
PROCEDURE [len:
INT, rdr: RopeReader.Ref]
RETURNS [Rope.
ROPE] = {
Read a rope using given reader - reader is assumed to be positioned ready to read first character
rope: Rope.ROPE;
pos: TiogaNode.Offset;
[rope, pos] ← RopeReader.Position[rdr]; -- find out where the start is
RopeReader.SetIndex[rdr, pos+len]; -- step reader over it
RETURN [RopeEdit.Substr[rope, pos, len]] -- return it
};
ReadRuns:
PROCEDURE []
RETURNS [lookRuns: TiogaLooks.Runs] = {
pos: TiogaNode.Offset ← 0;
numRuns: TiogaNode.Offset ← ReadF32[];
WHILE numRuns > 0
DO
num: NAT ← TiogaLooksSupport.Short[MIN[numRuns,LAST[NAT]]];
baseRuns: TiogaLooks.BaseRuns ← TiogaLooksSupport.NewBase[num];
len: TiogaNode.Offset ← pos;
numRuns ← numRuns-num;
FOR i:
NAT
IN [0..num)
DO
-- run
looks: TiogaLooks.Looks ← TiogaLooks.noLooks;
lookLen: TiogaNode.Offset;
op: CHAR;
IF (op ← NextOp[]) # T2FileOps.
NOLOOKS
THEN
DO
SELECT op
FROM
>= T2FileOps.lastLooksFirst => {
c: CHAR ← 'a+(op - T2FileOps.lastLooksFirst);
IF op > T2FileOps.lastLooksLast THEN SIGNAL fileFormatError;
looks[c] ← TRUE; EXIT };
>= T2FileOps.addLooksFirst => {
c: CHAR ← 'a+(op - T2FileOps.addLooksFirst);
looks[c] ← TRUE };
ENDCASE => SIGNAL fileFormatError;
op ← NextOp[]; ENDLOOP;
lookLen ← ReadF32[];
baseRuns[i] ← [pos←pos+lookLen, looks];
ENDLOOP;
lookRuns ←
IF lookRuns=
NIL
THEN baseRuns
ELSE
TiogaLooksOps.Concat[lookRuns,baseRuns,len,pos-len];
ENDLOOP };
ReadTextNode:
PROCEDURE [offset:
CARDINAL]
RETURNS [item: TiogaNode.RefTextNode] = {
isComment, doRuns, doFmt, doClass, doProps: BOOLEAN ← FALSE;
rdr: RopeReader.Ref;
{ OPEN T2FileOps;
IF offset >= propsBit*2 THEN ERROR fileFormatError;
IF offset >= propsBit THEN { doProps ← TRUE; offset ← offset-propsBit };
IF offset >= runsBit THEN { doRuns ← TRUE; offset ← offset-runsBit };
IF offset >= commentBit THEN { isComment ← TRUE; offset ← offset-commentBit };
IF offset >= formatBit THEN { doFmt ← TRUE; offset ← offset-formatBit };
IF offset > 0 THEN doClass ← TRUE };
item ← TiogaNodeOps.NewTextNode[
TiogaNodeOps.LookupItemID[IF doClass THEN ReadAtom[] ELSE $Text, TRUE]];
item.comment ← isComment;
IF doFmt THEN item.format ← ReadFormat[];
IF doRuns THEN item.runs ← ReadRuns[];
rdr ← IF isComment THEN control ELSE text;
item.rope ← ReadRope[ReadF32[], rdr];
RopeReader.BumpIndex[rdr, 1]; -- skip over CR
IF doProps
THEN {
ReadProperties[item];
IF op # T2FileOps.ENDPROPS THEN ERROR fileFormatError;
op ← NextOp[] }};
ReadProperties:
PROC [n: TiogaNode.Ref] = {
DO
SELECT op
FROM
T2FileOps.PROPERTY => { ReadProp[n]; op ← NextOp[] };
T2FileOps.APPLICATION => { n.templateInfo ← application; op ← NextOp[] };
T2FileOps.FORMAL => { n.templateInfo ← formal; op ← NextOp[] };
ENDCASE => RETURN;
ENDLOOP };
offset: CARDINAL;
isComment, doFmt, doClass: BOOLEAN ← FALSE;
branchClassInfo: TiogaBranchClass.BranchClass;
nKids: INT ← 0;
prev, current: TiogaNode.RefBranchNode;
kidCtrlStart, kidTextStart: TiogaNode.Offset;
op, childOp: T2FileOps.Op;
IF brInfo #
NIL
THEN {
-- otherwise, assume readers already set at correct positions
RopeReader.SetPosition[text, brInfo.externalRepRope, brInfo.charsStart];
RopeReader.SetPosition[control, brInfo.externalRepRope, brInfo.ctrlStart] };
IF br=NIL THEN RETURN [ReadContents[NIL]]; -- we're being called to read a non-branch node
op ← NextOp[]; -- start of the branch info
IF op
NOT
IN [T2FileOps.branch..T2FileOps.lastBranch]
THEN
ERROR fileFormatError; -- supposed to be a branch
offset ← op-T2FileOps.branch;
{ OPEN T2FileOps;
IF offset >= commentBit THEN { isComment ← TRUE; offset ← offset-commentBit };
IF offset >= formatBit THEN { doFmt ← TRUE; offset ← offset-formatBit };
IF offset > 0 THEN doClass ← TRUE };
IF doClass
THEN
NodeProps.PutProp[br, $BranchClass, branchClassInfo ← TiogaBranchClass.Lookup[ReadAtom[]]]
ELSE branchClassInfo ← TiogaBranchClass.defaultBranchClass;
IF doFmt THEN br.format ← ReadFormat[];
br.comment ← isComment;
op ← NextOp[];
ReadProperties[br];
IF op=T2FileOps.
BRANCHCONTENTSPECS
THEN {
-- contents saved in class dependent manner
Insert:
PROC [item: TiogaNode.RefItemNode, previous: TiogaNode.RefItemNode ←
NIL] = {
IF br.contents=NIL THEN br.contents ← item
ELSE {
IF previous=NIL THEN previous ← NARROW[TiogaTreeOps.LastSibling[br.contents]];
previous.next ← item; previous.last ← FALSE };
item.next ← br; item.last ← TRUE };
rope: Rope.ROPE;
start, len: INT;
[rope, start, len] ← ReadSpecs[];
op ← NextOp[]; -- the op after the specs
IF branchClassInfo.setContents=
NIL
THEN
-- trouble. save the specs
NodeProps.PutProp[br, TiogaBranchClass.branchContentsAtom, RopeEdit.Substr[rope, start, len]]
ELSE branchClassInfo.setContents[br, rope, start, len, Insert] }
ELSE br.contents ← NARROW[ReadContents[br]]; -- narrow to item node
IF (childOp ← op)=T2FileOps.NOCHILD THEN { br.internalRepCreated ← TRUE; RETURN };
IF childOp=T2FileOps.
CHILDSPECS
THEN {
-- children saved in class dependent manner
Insert:
PROC [child: TiogaNode.RefBranchNode, previous: TiogaNode.RefBranchNode ←
NIL] = {
IF br.child=NIL THEN br.child ← child
ELSE {
IF previous=NIL THEN previous ← NARROW[TiogaTreeOps.LastSibling[br.child]];
previous.next ← child; previous.last ← FALSE };
child.next ← br; child.last ← TRUE };
rope: Rope.ROPE;
start, len: INT;
[rope, start, len] ← ReadSpecs[];
IF branchClassInfo.setChildren=
NIL
THEN
-- trouble. save the specs
NodeProps.PutProp[br, TiogaBranchClass.branchChildrenAtom, RopeEdit.Substr[rope, start, len]]
ELSE branchClassInfo.setChildren[br, rope, start, len, Insert];
br.internalRepCreated ← TRUE;
RETURN };
IF childOp=T2FileOps.
STARTC
OR
((brInfo=NIL OR wholeTree) AND childOp=T2FileOps.ENDITEMS) THEN {
DO
-- parse a child each time through loop
current ← TiogaNodeOps.NewBranchNode[];
IF prev=NIL THEN br.child ← current ELSE { prev.next ← current; prev.last ← FALSE };
current.next ← br; prev ← current;
[] ← CIR[current, NIL, TRUE, textBuffer, control, text];
nKids ← nKids+1;
IF RopeReader.Peek[control]=T2FileOps.ENDC THEN { [] ← NextOp[]; EXIT };
ENDLOOP;
IF childOp=T2FileOps.
ENDITEMS
THEN {
--read and discard ctrl&data lengths and num children
op ← NextOp[]; -- actually is the first byte of number info
THROUGH [0..nKids*2)
DO
-- tricky because numbers written backwards on the file
read a single byte with top bit 0; then read sequence of bytes with top bit 1
can read until see byte with top bit 0 because quaranteed to have another number following
byte: T2FileOps.LengthByte ← LOOPHOLE[op];
IF byte.others THEN ERROR;
DO
byte ← LOOPHOLE[op ← NextOp[]];
IF ~byte.others THEN EXIT; -- this is the last byte of the next number
ENDLOOP;
ENDLOOP;
Now read as many bytes as needed to store num children (have already read first one)
UNTIL nKids < 128 DO [] ← NextOp[]; nKids ← nKids/128; ENDLOOP };
br.internalRepCreated ← TRUE; RETURN };
IF childOp # T2FileOps.ENDITEMS THEN ERROR fileFormatError;
kidCtrlStart ← RopeReader.GetIndex[control]; -- remember where the kids start
kidTextStart ← RopeReader.GetIndex[text];
RopeReader.SetPosition[control, brInfo.externalRepRope, brInfo.ctrlStart+brInfo.ctrlLen];
nKids ← ReadB32[];
THROUGH [1..nKids]
DO
-- create children branch nodes to be converted later
crntInfo: SpanInfo ← NEW[SpanInfoRec];
current ← TiogaNodeOps.NewBranchNode[];
IF prev=NIL THEN br.child ← current ELSE { prev.next ← current; prev.last ← FALSE };
current.next ← br; prev ← current;
crntInfo.externalRepRope ← brInfo.externalRepRope;
crntInfo.charsLen ← ReadB32[]; crntInfo.ctrlLen ← ReadB32[];
crntInfo.ctrlStart ← kidCtrlStart ; crntInfo.charsStart ← kidTextStart ;
kidCtrlStart ← kidCtrlStart + crntInfo.ctrlLen;
kidTextStart ← kidTextStart + crntInfo.charsLen;
NodeProps.PutProp[current, $BranchInfo, crntInfo];
current.externalRepValid ← TRUE;
current.internalRepCreated ← FALSE;
ENDLOOP;
br.internalRepCreated ← TRUE };