Coverage Report

 // This file was automatically generated with the command: 
 //     java org.galagosearch.tupleflow.typebuilder.TypeBuilderMojo ...
 package org.galagosearch.core.types;
 
 import org.galagosearch.tupleflow.Utility;
 import org.galagosearch.tupleflow.ArrayInput;
 import org.galagosearch.tupleflow.ArrayOutput;
 import org.galagosearch.tupleflow.Order;   
 import org.galagosearch.tupleflow.OrderedWriter;
 import org.galagosearch.tupleflow.Type; 
 import org.galagosearch.tupleflow.TypeReader;
 import org.galagosearch.tupleflow.Step; 
 import org.galagosearch.tupleflow.IncompatibleProcessorException;
 import org.galagosearch.tupleflow.ReaderSource;
 import java.io.IOException;             
 import java.io.EOFException;
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Arrays;   
 import java.util.Comparator;
 import java.util.PriorityQueue;
 import java.util.Collection;
 
 public class DocumentExtent implements Type<DocumentExtent> {
     public String extentName;
     public String identifier;
     public int begin;
     public int end; 
     
     public DocumentExtent() {}
     public DocumentExtent(String extentName, String identifier, int begin, int end) {
         this.extentName = extentName;
         this.identifier = identifier;
         this.begin = begin;
         this.end = end;
     }  
     
     public String toString() {
             return String.format("%s,%s,%d,%d",
                                    extentName, identifier, begin, end);
     } 
 
     public Order<DocumentExtent> getOrder(String... spec) {
         if (Arrays.equals(spec, new String[] { "+identifier" })) {
             return new IdentifierOrder();
         }
         return null;
     } 
       
     public interface Processor extends Step, org.galagosearch.tupleflow.Processor<DocumentExtent> {
         public void process(DocumentExtent object) throws IOException;
         public void close() throws IOException;
     }                        
     public interface Source extends Step {
     }
     public static class IdentifierOrder implements Order<DocumentExtent> {
         public int hash(DocumentExtent object) {
             int h = 0;
             h += Utility.hash(object.identifier);
             return h;
         } 
         public Comparator<DocumentExtent> greaterThan() {
             return new Comparator<DocumentExtent>() {
                 public int compare(DocumentExtent one, DocumentExtent two) {
                     int result = 0;
                     do {
                         result = + Utility.compare(one.identifier, two.identifier);
                         if(result != 0) break;
                     } while (false);
                     return -result;
                 }
             };
         }     
         public Comparator<DocumentExtent> lessThan() {
             return new Comparator<DocumentExtent>() {
                 public int compare(DocumentExtent one, DocumentExtent two) {
                     int result = 0;
                     do {
                         result = + Utility.compare(one.identifier, two.identifier);
                         if(result != 0) break;
                     } while (false);
                     return result;
                 }
             };
         }     
         public TypeReader<DocumentExtent> orderedReader(ArrayInput _input) {
             return new ShreddedReader(_input);
         }    
 
         public TypeReader<DocumentExtent> orderedReader(ArrayInput _input, int bufferSize) {
             return new ShreddedReader(_input, bufferSize);
         }    
         public OrderedWriter<DocumentExtent> orderedWriter(ArrayOutput _output) {
             ShreddedWriter w = new ShreddedWriter(_output);
             return new OrderedWriterClass(w); 
         }                                    
         public static class OrderedWriterClass extends OrderedWriter< DocumentExtent > {
             DocumentExtent last = null;
             ShreddedWriter shreddedWriter = null; 
             
             public OrderedWriterClass(ShreddedWriter s) {
                 this.shreddedWriter = s;
             }
             
             public void process(DocumentExtent object) throws IOException {
                boolean processAll = false;
                if (processAll || last == null || 0 != Utility.compare(object.identifier, last.identifier)) { processAll = true; shreddedWriter.processIdentifier(object.identifier); }
                shreddedWriter.processTuple(object.extentName, object.begin, object.end);
                last = object;
             }           
                  
             public void close() throws IOException {
                 shreddedWriter.close();
             }
             
             public Class<DocumentExtent> getInputClass() {
                 return DocumentExtent.class;
             }
         } 
         public ReaderSource<DocumentExtent> orderedCombiner(Collection<TypeReader<DocumentExtent>> readers, boolean closeOnExit) {
             ArrayList<ShreddedReader> shreddedReaders = new ArrayList();
             
             for (TypeReader<DocumentExtent> reader : readers) {
                 shreddedReaders.add((ShreddedReader)reader);
             }
             
             return new ShreddedCombiner(shreddedReaders, closeOnExit);
         }                  
         public DocumentExtent clone(DocumentExtent object) {
             DocumentExtent result = new DocumentExtent();
             if (object == null) return result;
             result.extentName = object.extentName; 
             result.identifier = object.identifier; 
             result.begin = object.begin; 
             result.end = object.end; 
             return result;
         }                 
         public Class<DocumentExtent> getOrderedClass() {
             return DocumentExtent.class;
         }                           
         public String[] getOrderSpec() {
             return new String[] {"+identifier"};
         }
 
         public static String getSpecString() {
             return "+identifier";
         }
                            
         public interface ShreddedProcessor extends Step {
             public void processIdentifier(String identifier) throws IOException;
             public void processTuple(String extentName, int begin, int end) throws IOException;
             public void close() throws IOException;
         }    
         public interface ShreddedSource extends Step {
         }                                              
         
         public static class ShreddedWriter implements ShreddedProcessor {
             ArrayOutput output;
             ShreddedBuffer buffer = new ShreddedBuffer();
             String lastIdentifier;
             boolean lastFlush = false;
             
             public ShreddedWriter(ArrayOutput output) {
                 this.output = output;
             }                        
             
             public void close() throws IOException {
                 flush();
             }
             
             public void processIdentifier(String identifier) {
                 lastIdentifier = identifier;
                 buffer.processIdentifier(identifier);
             }
             public final void processTuple(String extentName, int begin, int end) throws IOException {
                 if (lastFlush) {
                     if(buffer.identifiers.size() == 0) buffer.processIdentifier(lastIdentifier);
                     lastFlush = false;
                 }
                 buffer.processTuple(extentName, begin, end);
                 if (buffer.isFull())
                     flush();
             }
             public final void flushTuples(int pauseIndex) throws IOException {
                 
                 while (buffer.getReadIndex() < pauseIndex) {
                            
                     output.writeString(buffer.getExtentName());
                     output.writeInt(buffer.getBegin());
                     output.writeInt(buffer.getEnd());
                     buffer.incrementTuple();
                 }
             }  
             public final void flushIdentifier(int pauseIndex) throws IOException {
                 while (buffer.getReadIndex() < pauseIndex) {
                     int nextPause = buffer.getIdentifierEndIndex();
                     int count = nextPause - buffer.getReadIndex();
                     
                     output.writeString(buffer.getIdentifier());
                     output.writeInt(count);
                     buffer.incrementIdentifier();
                       
                     flushTuples(nextPause);
                     assert nextPause == buffer.getReadIndex();
                 }
             }
             public void flush() throws IOException { 
                 flushIdentifier(buffer.getWriteIndex());
                 buffer.reset(); 
                 lastFlush = true;
             }                           
         }
         public static class ShreddedBuffer {
             ArrayList<String> identifiers = new ArrayList();
             ArrayList<Integer> identifierTupleIdx = new ArrayList();
             int identifierReadIdx = 0;
                             
             String[] extentNames;
             int[] begins;
             int[] ends;
             int writeTupleIndex = 0;
             int readTupleIndex = 0;
             int batchSize;
 
             public ShreddedBuffer(int batchSize) {
                 this.batchSize = batchSize;
 
                 extentNames = new String[batchSize];
                 begins = new int[batchSize];
                 ends = new int[batchSize];
             }                              
 
             public ShreddedBuffer() {    
                 this(10000);
             }                                                                                                                    
             
             public void processIdentifier(String identifier) {
                 identifiers.add(identifier);
                 identifierTupleIdx.add(writeTupleIndex);
             }                                      
             public void processTuple(String extentName, int begin, int end) {
                 assert identifiers.size() > 0;
                 extentNames[writeTupleIndex] = extentName;
                 begins[writeTupleIndex] = begin;
                 ends[writeTupleIndex] = end;
                 writeTupleIndex++;
             }
             public void resetData() {
                 identifiers.clear();
                 identifierTupleIdx.clear();
                 writeTupleIndex = 0;
             }                  
                                  
             public void resetRead() {
                 readTupleIndex = 0;
                 identifierReadIdx = 0;
             } 
 
             public void reset() {
                 resetData();
                 resetRead();
             } 
             public boolean isFull() {
                 return writeTupleIndex >= batchSize;
             }
 
             public boolean isEmpty() {
                 return writeTupleIndex == 0;
             }                          
 
             public boolean isAtEnd() {
                 return readTupleIndex >= writeTupleIndex;
             }           
             public void incrementIdentifier() {
                 identifierReadIdx++;  
             }                                                                                              
 
             public void autoIncrementIdentifier() {
                 while (readTupleIndex >= getIdentifierEndIndex() && readTupleIndex < writeTupleIndex)
                     identifierReadIdx++;
             }                 
             public void incrementTuple() {
                 readTupleIndex++;
             }                    
             public int getIdentifierEndIndex() {
                 if ((identifierReadIdx+1) >= identifierTupleIdx.size())
                     return writeTupleIndex;
                 return identifierTupleIdx.get(identifierReadIdx+1);
             }
             public int getReadIndex() {
                 return readTupleIndex;
             }   
 
             public int getWriteIndex() {
                 return writeTupleIndex;
             } 
             public String getIdentifier() {
                 assert readTupleIndex < writeTupleIndex;
                 assert identifierReadIdx < identifiers.size();
                 
                 return identifiers.get(identifierReadIdx);
             }
             public String getExtentName() {
                 assert readTupleIndex < writeTupleIndex;
                 return extentNames[readTupleIndex];
             }                                         
             public int getBegin() {
                 assert readTupleIndex < writeTupleIndex;
                 return begins[readTupleIndex];
             }                                         
             public int getEnd() {
                 assert readTupleIndex < writeTupleIndex;
                 return ends[readTupleIndex];
             }                                         
             public void copyTuples(int endIndex, ShreddedProcessor output) throws IOException {
                 while (getReadIndex() < endIndex) {
                    output.processTuple(getExtentName(), getBegin(), getEnd());
                    incrementTuple();
                 }
             }                                                                           
             public void copyUntilIndexIdentifier(int endIndex, ShreddedProcessor output) throws IOException {
                 while (getReadIndex() < endIndex) {
                     output.processIdentifier(getIdentifier());
                     assert getIdentifierEndIndex() <= endIndex;
                     copyTuples(getIdentifierEndIndex(), output);
                     incrementIdentifier();
                 }
             }  
             public void copyUntilIdentifier(ShreddedBuffer other, ShreddedProcessor output) throws IOException {
                 while (!isAtEnd()) {
                     if (other != null) {   
                         assert !other.isAtEnd();
                         int c = + Utility.compare(getIdentifier(), other.getIdentifier());
                     
                         if (c > 0) {
                             break;   
                         }
                         
                         output.processIdentifier(getIdentifier());
                                       
                         copyTuples(getIdentifierEndIndex(), output);
                     } else {
                         output.processIdentifier(getIdentifier());
                         copyTuples(getIdentifierEndIndex(), output);
                     }
                     incrementIdentifier();  
                     
                
                 }
             }
             public void copyUntil(ShreddedBuffer other, ShreddedProcessor output) throws IOException {
                 copyUntilIdentifier(other, output);
             }
             
         }                         
         public static class ShreddedCombiner implements ReaderSource<DocumentExtent>, ShreddedSource {   
             public ShreddedProcessor processor;
             Collection<ShreddedReader> readers;       
             boolean closeOnExit = false;
             boolean uninitialized = true;
             PriorityQueue<ShreddedReader> queue = new PriorityQueue<ShreddedReader>();
             
             public ShreddedCombiner(Collection<ShreddedReader> readers, boolean closeOnExit) {
                 this.readers = readers;                                                       
                 this.closeOnExit = closeOnExit;
             }
                                   
             public void setProcessor(Step processor) throws IncompatibleProcessorException {  
                 if (processor instanceof ShreddedProcessor) {
                     this.processor = new DuplicateEliminator((ShreddedProcessor) processor);
                 } else if (processor instanceof DocumentExtent.Processor) {
                     this.processor = new DuplicateEliminator(new TupleUnshredder((DocumentExtent.Processor) processor));
                 } else if (processor instanceof org.galagosearch.tupleflow.Processor) {
                     this.processor = new DuplicateEliminator(new TupleUnshredder((org.galagosearch.tupleflow.Processor<DocumentExtent>) processor));
                 } else {
                     throw new IncompatibleProcessorException(processor.getClass().getName() + " is not supported by " + this.getClass().getName());                                                                       
                 }
             }                                
             
             public Class<DocumentExtent> getOutputClass() {
                 return DocumentExtent.class;
             }
             
             public void initialize() throws IOException {
                 for (ShreddedReader reader : readers) {
                     reader.fill();                                        
                     
                     if (!reader.getBuffer().isAtEnd())
                         queue.add(reader);
                 }   
 
                 uninitialized = false;
             }
 
             public void run() throws IOException {
                 initialize();
                
                 while (queue.size() > 0) {
                     ShreddedReader top = queue.poll();
                     ShreddedReader next = null;
                     ShreddedBuffer nextBuffer = null; 
                     
                     assert !top.getBuffer().isAtEnd();
                                                   
                     if (queue.size() > 0) {
                         next = queue.peek();
                         nextBuffer = next.getBuffer();
                         assert !nextBuffer.isAtEnd();
                     }
                     
                     top.getBuffer().copyUntil(nextBuffer, processor);
                     if (top.getBuffer().isAtEnd())
                         top.fill();                 
                         
                     if (!top.getBuffer().isAtEnd())
                         queue.add(top);
                 }              
                 
                 if (closeOnExit)
                     processor.close();
             }
 
             public DocumentExtent read() throws IOException {
                 if (uninitialized)
                     initialize();
 
                 DocumentExtent result = null;
 
                 while (queue.size() > 0) {
                     ShreddedReader top = queue.poll();
                     result = top.read();
 
                     if (result != null) {
                         if (top.getBuffer().isAtEnd())
                             top.fill();
 
                         queue.offer(top);
                         break;
                     } 
                 }
 
                 return result;
             }
         } 
         public static class ShreddedReader implements Step, Comparable<ShreddedReader>, TypeReader<DocumentExtent>, ShreddedSource {      
             public ShreddedProcessor processor;
             ShreddedBuffer buffer;
             DocumentExtent last = new DocumentExtent();         
             long updateIdentifierCount = -1;
             long tupleCount = 0;
             long bufferStartCount = 0;  
             ArrayInput input;
             
             public ShreddedReader(ArrayInput input) {
                 this.input = input; 
                 this.buffer = new ShreddedBuffer();
             }                               
             
             public ShreddedReader(ArrayInput input, int bufferSize) { 
                 this.input = input;
                 this.buffer = new ShreddedBuffer(bufferSize);
             }
                  
             public final int compareTo(ShreddedReader other) {
                 ShreddedBuffer otherBuffer = other.getBuffer();
                 
                 if (buffer.isAtEnd() && otherBuffer.isAtEnd()) {
                     return 0;                 
                 } else if (buffer.isAtEnd()) {
                     return -1;
                 } else if (otherBuffer.isAtEnd()) {
                     return 1;
                 }
                                    
                 int result = 0;
                 do {
                     result = + Utility.compare(buffer.getIdentifier(), otherBuffer.getIdentifier());
                     if(result != 0) break;
                 } while (false);                                             
                 
                 return result;
             }
             
             public final ShreddedBuffer getBuffer() {
                 return buffer;
             }                
             
             public final DocumentExtent read() throws IOException {
                 if (buffer.isAtEnd()) {
                     fill();             
                 
                     if (buffer.isAtEnd()) {
                         return null;
                     }
                 }
                       
                 assert !buffer.isAtEnd();
                 DocumentExtent result = new DocumentExtent();
                 
                 result.identifier = buffer.getIdentifier();
                 result.extentName = buffer.getExtentName();
                 result.begin = buffer.getBegin();
                 result.end = buffer.getEnd();
                 
                 buffer.incrementTuple();
                 buffer.autoIncrementIdentifier();
                 
                 return result;
             }           
             
             public final void fill() throws IOException {
                 try {   
                     buffer.reset();
                     
                     if (tupleCount != 0) {
                                                       
                         if(updateIdentifierCount - tupleCount > 0) {
                             buffer.identifiers.add(last.identifier);
                             buffer.identifierTupleIdx.add((int) (updateIdentifierCount - tupleCount));
                         }
                         bufferStartCount = tupleCount;
                     }
                     
                     while (!buffer.isFull()) {
                         updateIdentifier();
                         buffer.processTuple(input.readString(), input.readInt(), input.readInt());
                         tupleCount++;
                     }
                 } catch(EOFException e) {}
             }
 
             public final void updateIdentifier() throws IOException {
                 if (updateIdentifierCount > tupleCount)
                     return;
                      
                 last.identifier = input.readString();
                 updateIdentifierCount = tupleCount + input.readInt();
                                       
                 buffer.processIdentifier(last.identifier);
             }
 
             public void run() throws IOException {
                 while (true) {
                     fill();
                     
                     if (buffer.isAtEnd())
                         break;
                     
                     buffer.copyUntil(null, processor);
                 }      
                 processor.close();
             }
             
             public void setProcessor(Step processor) throws IncompatibleProcessorException {  
                 if (processor instanceof ShreddedProcessor) {
                     this.processor = new DuplicateEliminator((ShreddedProcessor) processor);
                 } else if (processor instanceof DocumentExtent.Processor) {
                     this.processor = new DuplicateEliminator(new TupleUnshredder((DocumentExtent.Processor) processor));
                 } else if (processor instanceof org.galagosearch.tupleflow.Processor) {
                     this.processor = new DuplicateEliminator(new TupleUnshredder((org.galagosearch.tupleflow.Processor<DocumentExtent>) processor));
                 } else {
                     throw new IncompatibleProcessorException(processor.getClass().getName() + " is not supported by " + this.getClass().getName());                                                                       
                 }
             }                                
             
             public Class<DocumentExtent> getOutputClass() {
                 return DocumentExtent.class;
             }                
         }
         
         public static class DuplicateEliminator implements ShreddedProcessor {
             public ShreddedProcessor processor;
             DocumentExtent last = new DocumentExtent();
             boolean identifierProcess = true;
                                            
             public DuplicateEliminator() {}
             public DuplicateEliminator(ShreddedProcessor processor) {
                 this.processor = processor;
             }
             
             public void setShreddedProcessor(ShreddedProcessor processor) {
                 this.processor = processor;
             }
 
             public void processIdentifier(String identifier) throws IOException {  
                 if (identifierProcess || Utility.compare(identifier, last.identifier) != 0) {
                     last.identifier = identifier;
                     processor.processIdentifier(identifier);
                     identifierProcess = false;
                 }
             }  
             
             public void resetIdentifier() {
                  identifierProcess = true;
             }                                                
                                
             public void processTuple(String extentName, int begin, int end) throws IOException {
                 processor.processTuple(extentName, begin, end);
             } 
             
             public void close() throws IOException {
                 processor.close();
             }                    
         }
         public static class TupleUnshredder implements ShreddedProcessor {
             DocumentExtent last = new DocumentExtent();
             public org.galagosearch.tupleflow.Processor<DocumentExtent> processor;                               
             
             public TupleUnshredder(DocumentExtent.Processor processor) {
                 this.processor = processor;
             }         
             
             public TupleUnshredder(org.galagosearch.tupleflow.Processor<DocumentExtent> processor) {
                 this.processor = processor;
             }
             
             public DocumentExtent clone(DocumentExtent object) {
                 DocumentExtent result = new DocumentExtent();
                 if (object == null) return result;
                 result.extentName = object.extentName; 
                 result.identifier = object.identifier; 
                 result.begin = object.begin; 
                 result.end = object.end; 
                 return result;
             }                 
             
             public void processIdentifier(String identifier) throws IOException {
                 last.identifier = identifier;
             }   
                 
             
             public void processTuple(String extentName, int begin, int end) throws IOException {
                 last.extentName = extentName;
                 last.begin = begin;
                 last.end = end;
                 processor.process(clone(last));
             }               
             
             public void close() throws IOException {
                 processor.close();
             }
         }     
         public static class TupleShredder implements Processor {
             DocumentExtent last = new DocumentExtent();
             public ShreddedProcessor processor;
             
             public TupleShredder(ShreddedProcessor processor) {
                 this.processor = processor;
             }                              
             
             public DocumentExtent clone(DocumentExtent object) {
                 DocumentExtent result = new DocumentExtent();
                 if (object == null) return result;
                 result.extentName = object.extentName; 
                 result.identifier = object.identifier; 
                 result.begin = object.begin; 
                 result.end = object.end; 
                 return result;
             }                 
             
             public void process(DocumentExtent object) throws IOException {                                                                                                                                                   
                 boolean processAll = false;
                 if(last == null || Utility.compare(last.identifier, object.identifier) != 0 || processAll) { processor.processIdentifier(object.identifier); processAll = true; }
                 processor.processTuple(object.extentName, object.begin, object.end);                                         
             }
                           
             public Class<DocumentExtent> getInputClass() {
                 return DocumentExtent.class;
             }
             
             public void close() throws IOException {
                 processor.close();
             }                     
         }
     } 
 }    

1		// This file was automatically generated with the command:
2		// java org.galagosearch.tupleflow.typebuilder.TypeBuilderMojo ...
3		package org.galagosearch.core.types;
4
5		import org.galagosearch.tupleflow.Utility;
6		import org.galagosearch.tupleflow.ArrayInput;
7		import org.galagosearch.tupleflow.ArrayOutput;
8		import org.galagosearch.tupleflow.Order;
9		import org.galagosearch.tupleflow.OrderedWriter;
10		import org.galagosearch.tupleflow.Type;
11		import org.galagosearch.tupleflow.TypeReader;
12		import org.galagosearch.tupleflow.Step;
13		import org.galagosearch.tupleflow.IncompatibleProcessorException;
14		import org.galagosearch.tupleflow.ReaderSource;
15		import java.io.IOException;
16		import java.io.EOFException;
17		import java.io.UnsupportedEncodingException;
18		import java.util.ArrayList;
19		import java.util.Arrays;
20		import java.util.Comparator;
21		import java.util.PriorityQueue;
22		import java.util.Collection;
23
24		public class DocumentExtent implements Type<DocumentExtent> {
25		public String extentName;
26		public String identifier;
27		public int begin;
28		public int end;
29
30	20	public DocumentExtent() {}
31	0	public DocumentExtent(String extentName, String identifier, int begin, int end) {
32	0	this.extentName = extentName;
33	0	this.identifier = identifier;
34	0	this.begin = begin;
35	0	this.end = end;
36	0	}
37
38		public String toString() {
39	0	return String.format("%s,%s,%d,%d",
40		extentName, identifier, begin, end);
41		}
42
43		public Order<DocumentExtent> getOrder(String... spec) {
44	20	if (Arrays.equals(spec, new String[] { "+identifier" })) {
45	20	return new IdentifierOrder();
46		}
47	0	return null;
48		}
49
50		public interface Processor extends Step, org.galagosearch.tupleflow.Processor<DocumentExtent> {
51		public void process(DocumentExtent object) throws IOException;
52		public void close() throws IOException;
53		}
54		public interface Source extends Step {
55		}
56	76	public static class IdentifierOrder implements Order<DocumentExtent> {
57		public int hash(DocumentExtent object) {
58	0	int h = 0;
59	0	h += Utility.hash(object.identifier);
60	0	return h;
61		}
62		public Comparator<DocumentExtent> greaterThan() {
63	0	return new Comparator<DocumentExtent>() {
64	0	public int compare(DocumentExtent one, DocumentExtent two) {
65	0	int result = 0;
66		do {
67	0	result = + Utility.compare(one.identifier, two.identifier);
68	0	if(result != 0) break;
69		} while (false);
70	0	return -result;
71		}
72		};
73		}
74		public Comparator<DocumentExtent> lessThan() {
75	0	return new Comparator<DocumentExtent>() {
76	0	public int compare(DocumentExtent one, DocumentExtent two) {
77	0	int result = 0;
78		do {
79	0	result = + Utility.compare(one.identifier, two.identifier);
80	0	if(result != 0) break;
81		} while (false);
82	0	return result;
83		}
84		};
85		}
86		public TypeReader<DocumentExtent> orderedReader(ArrayInput _input) {
87	0	return new ShreddedReader(_input);
88		}
89
90		public TypeReader<DocumentExtent> orderedReader(ArrayInput _input, int bufferSize) {
91	0	return new ShreddedReader(_input, bufferSize);
92		}
93		public OrderedWriter<DocumentExtent> orderedWriter(ArrayOutput _output) {
94	0	ShreddedWriter w = new ShreddedWriter(_output);
95	0	return new OrderedWriterClass(w);
96		}
97	0	public static class OrderedWriterClass extends OrderedWriter< DocumentExtent > {
98	0	DocumentExtent last = null;
99	0	ShreddedWriter shreddedWriter = null;
100
101	0	public OrderedWriterClass(ShreddedWriter s) {
102	0	this.shreddedWriter = s;
103	0	}
104
105		public void process(DocumentExtent object) throws IOException {
106	0	boolean processAll = false;
107	0	if (processAll \|\| last == null \|\| 0 != Utility.compare(object.identifier, last.identifier)) { processAll = true; shreddedWriter.processIdentifier(object.identifier); }
108	0	shreddedWriter.processTuple(object.extentName, object.begin, object.end);
109	0	last = object;
110	0	}
111
112		public void close() throws IOException {
113	0	shreddedWriter.close();
114	0	}
115
116		public Class<DocumentExtent> getInputClass() {
117	0	return DocumentExtent.class;
118		}
119		}
120		public ReaderSource<DocumentExtent> orderedCombiner(Collection<TypeReader<DocumentExtent>> readers, boolean closeOnExit) {
121	0	ArrayList<ShreddedReader> shreddedReaders = new ArrayList();
122
123	0	for (TypeReader<DocumentExtent> reader : readers) {
124	0	shreddedReaders.add((ShreddedReader)reader);
125		}
126
127	0	return new ShreddedCombiner(shreddedReaders, closeOnExit);
128		}
129		public DocumentExtent clone(DocumentExtent object) {
130	0	DocumentExtent result = new DocumentExtent();
131	0	if (object == null) return result;
132	0	result.extentName = object.extentName;
133	0	result.identifier = object.identifier;
134	0	result.begin = object.begin;
135	0	result.end = object.end;
136	0	return result;
137		}
138		public Class<DocumentExtent> getOrderedClass() {
139	56	return DocumentExtent.class;
140		}
141		public String[] getOrderSpec() {
142	56	return new String[] {"+identifier"};
143		}
144
145		public static String getSpecString() {
146	0	return "+identifier";
147		}
148
149		public interface ShreddedProcessor extends Step {
150		public void processIdentifier(String identifier) throws IOException;
151		public void processTuple(String extentName, int begin, int end) throws IOException;
152		public void close() throws IOException;
153		}
154		public interface ShreddedSource extends Step {
155		}
156
157	0	public static class ShreddedWriter implements ShreddedProcessor {
158		ArrayOutput output;
159	0	ShreddedBuffer buffer = new ShreddedBuffer();
160		String lastIdentifier;
161	0	boolean lastFlush = false;
162
163	0	public ShreddedWriter(ArrayOutput output) {
164	0	this.output = output;
165	0	}
166
167		public void close() throws IOException {
168	0	flush();
169	0	}
170
171		public void processIdentifier(String identifier) {
172	0	lastIdentifier = identifier;
173	0	buffer.processIdentifier(identifier);
174	0	}
175		public final void processTuple(String extentName, int begin, int end) throws IOException {
176	0	if (lastFlush) {
177	0	if(buffer.identifiers.size() == 0) buffer.processIdentifier(lastIdentifier);
178	0	lastFlush = false;
179		}
180	0	buffer.processTuple(extentName, begin, end);
181	0	if (buffer.isFull())
182	0	flush();
183	0	}
184		public final void flushTuples(int pauseIndex) throws IOException {
185
186	0	while (buffer.getReadIndex() < pauseIndex) {
187
188	0	output.writeString(buffer.getExtentName());
189	0	output.writeInt(buffer.getBegin());
190	0	output.writeInt(buffer.getEnd());
191	0	buffer.incrementTuple();
192		}
193	0	}
194		public final void flushIdentifier(int pauseIndex) throws IOException {
195	0	while (buffer.getReadIndex() < pauseIndex) {
196	0	int nextPause = buffer.getIdentifierEndIndex();
197	0	int count = nextPause - buffer.getReadIndex();
198
199	0	output.writeString(buffer.getIdentifier());
200	0	output.writeInt(count);
201	0	buffer.incrementIdentifier();
202
203	0	flushTuples(nextPause);
204	0	assert nextPause == buffer.getReadIndex();
205	0	}
206	0	}
207		public void flush() throws IOException {
208	0	flushIdentifier(buffer.getWriteIndex());
209	0	buffer.reset();
210	0	lastFlush = true;
211	0	}
212		}
213	0	public static class ShreddedBuffer {
214	0	ArrayList<String> identifiers = new ArrayList();
215	0	ArrayList<Integer> identifierTupleIdx = new ArrayList();
216	0	int identifierReadIdx = 0;
217
218		String[] extentNames;
219		int[] begins;
220		int[] ends;
221	0	int writeTupleIndex = 0;
222	0	int readTupleIndex = 0;
223		int batchSize;
224
225	0	public ShreddedBuffer(int batchSize) {
226	0	this.batchSize = batchSize;
227
228	0	extentNames = new String[batchSize];
229	0	begins = new int[batchSize];
230	0	ends = new int[batchSize];
231	0	}
232
233		public ShreddedBuffer() {
234	0	this(10000);
235	0	}
236
237		public void processIdentifier(String identifier) {
238	0	identifiers.add(identifier);
239	0	identifierTupleIdx.add(writeTupleIndex);
240	0	}
241		public void processTuple(String extentName, int begin, int end) {
242	0	assert identifiers.size() > 0;
243	0	extentNames[writeTupleIndex] = extentName;
244	0	begins[writeTupleIndex] = begin;
245	0	ends[writeTupleIndex] = end;
246	0	writeTupleIndex++;
247	0	}
248		public void resetData() {
249	0	identifiers.clear();
250	0	identifierTupleIdx.clear();
251	0	writeTupleIndex = 0;
252	0	}
253
254		public void resetRead() {
255	0	readTupleIndex = 0;
256	0	identifierReadIdx = 0;
257	0	}
258
259		public void reset() {
260	0	resetData();
261	0	resetRead();
262	0	}
263		public boolean isFull() {
264	0	return writeTupleIndex >= batchSize;
265		}
266
267		public boolean isEmpty() {
268	0	return writeTupleIndex == 0;
269		}
270
271		public boolean isAtEnd() {
272	0	return readTupleIndex >= writeTupleIndex;
273		}
274		public void incrementIdentifier() {
275	0	identifierReadIdx++;
276	0	}
277
278		public void autoIncrementIdentifier() {
279	0	while (readTupleIndex >= getIdentifierEndIndex() && readTupleIndex < writeTupleIndex)
280	0	identifierReadIdx++;
281	0	}
282		public void incrementTuple() {
283	0	readTupleIndex++;
284	0	}
285		public int getIdentifierEndIndex() {
286	0	if ((identifierReadIdx+1) >= identifierTupleIdx.size())
287	0	return writeTupleIndex;
288	0	return identifierTupleIdx.get(identifierReadIdx+1);
289		}
290		public int getReadIndex() {
291	0	return readTupleIndex;
292		}
293
294		public int getWriteIndex() {
295	0	return writeTupleIndex;
296		}
297		public String getIdentifier() {
298	0	assert readTupleIndex < writeTupleIndex;
299	0	assert identifierReadIdx < identifiers.size();
300
301	0	return identifiers.get(identifierReadIdx);
302		}
303		public String getExtentName() {
304	0	assert readTupleIndex < writeTupleIndex;
305	0	return extentNames[readTupleIndex];
306		}
307		public int getBegin() {
308	0	assert readTupleIndex < writeTupleIndex;
309	0	return begins[readTupleIndex];
310		}
311		public int getEnd() {
312	0	assert readTupleIndex < writeTupleIndex;
313	0	return ends[readTupleIndex];
314		}
315		public void copyTuples(int endIndex, ShreddedProcessor output) throws IOException {
316	0	while (getReadIndex() < endIndex) {
317	0	output.processTuple(getExtentName(), getBegin(), getEnd());
318	0	incrementTuple();
319		}
320	0	}
321		public void copyUntilIndexIdentifier(int endIndex, ShreddedProcessor output) throws IOException {
322	0	while (getReadIndex() < endIndex) {
323	0	output.processIdentifier(getIdentifier());
324	0	assert getIdentifierEndIndex() <= endIndex;
325	0	copyTuples(getIdentifierEndIndex(), output);
326	0	incrementIdentifier();
327		}
328	0	}
329		public void copyUntilIdentifier(ShreddedBuffer other, ShreddedProcessor output) throws IOException {
330	0	while (!isAtEnd()) {
331	0	if (other != null) {
332	0	assert !other.isAtEnd();
333	0	int c = + Utility.compare(getIdentifier(), other.getIdentifier());
334
335	0	if (c > 0) {
336	0	break;
337		}
338
339	0	output.processIdentifier(getIdentifier());
340
341	0	copyTuples(getIdentifierEndIndex(), output);
342	0	} else {
343	0	output.processIdentifier(getIdentifier());
344	0	copyTuples(getIdentifierEndIndex(), output);
345		}
346	0	incrementIdentifier();
347
348
349		}
350	0	}
351		public void copyUntil(ShreddedBuffer other, ShreddedProcessor output) throws IOException {
352	0	copyUntilIdentifier(other, output);
353	0	}
354
355		}
356	0	public static class ShreddedCombiner implements ReaderSource<DocumentExtent>, ShreddedSource {
357		public ShreddedProcessor processor;
358		Collection<ShreddedReader> readers;
359	0	boolean closeOnExit = false;
360	0	boolean uninitialized = true;
361	0	PriorityQueue<ShreddedReader> queue = new PriorityQueue<ShreddedReader>();
362
363	0	public ShreddedCombiner(Collection<ShreddedReader> readers, boolean closeOnExit) {
364	0	this.readers = readers;
365	0	this.closeOnExit = closeOnExit;
366	0	}
367
368		public void setProcessor(Step processor) throws IncompatibleProcessorException {
369	0	if (processor instanceof ShreddedProcessor) {
370	0	this.processor = new DuplicateEliminator((ShreddedProcessor) processor);
371	0	} else if (processor instanceof DocumentExtent.Processor) {
372	0	this.processor = new DuplicateEliminator(new TupleUnshredder((DocumentExtent.Processor) processor));
373	0	} else if (processor instanceof org.galagosearch.tupleflow.Processor) {
374	0	this.processor = new DuplicateEliminator(new TupleUnshredder((org.galagosearch.tupleflow.Processor<DocumentExtent>) processor));
375		} else {
376	0	throw new IncompatibleProcessorException(processor.getClass().getName() + " is not supported by " + this.getClass().getName());
377		}
378	0	}
379
380		public Class<DocumentExtent> getOutputClass() {
381	0	return DocumentExtent.class;
382		}
383
384		public void initialize() throws IOException {
385	0	for (ShreddedReader reader : readers) {
386	0	reader.fill();
387
388	0	if (!reader.getBuffer().isAtEnd())
389	0	queue.add(reader);
390		}
391
392	0	uninitialized = false;
393	0	}
394
395		public void run() throws IOException {
396	0	initialize();
397
398	0	while (queue.size() > 0) {
399	0	ShreddedReader top = queue.poll();
400	0	ShreddedReader next = null;
401	0	ShreddedBuffer nextBuffer = null;
402
403	0	assert !top.getBuffer().isAtEnd();
404
405	0	if (queue.size() > 0) {
406	0	next = queue.peek();
407	0	nextBuffer = next.getBuffer();
408	0	assert !nextBuffer.isAtEnd();
409		}
410
411	0	top.getBuffer().copyUntil(nextBuffer, processor);
412	0	if (top.getBuffer().isAtEnd())
413	0	top.fill();
414
415	0	if (!top.getBuffer().isAtEnd())
416	0	queue.add(top);
417	0	}
418
419	0	if (closeOnExit)
420	0	processor.close();
421	0	}
422
423		public DocumentExtent read() throws IOException {
424	0	if (uninitialized)
425	0	initialize();
426
427	0	DocumentExtent result = null;
428
429	0	while (queue.size() > 0) {
430	0	ShreddedReader top = queue.poll();
431	0	result = top.read();
432
433	0	if (result != null) {
434	0	if (top.getBuffer().isAtEnd())
435	0	top.fill();
436
437	0	queue.offer(top);
438	0	break;
439		}
440	0	}
441
442	0	return result;
443		}
444		}
445	0	public static class ShreddedReader implements Step, Comparable<ShreddedReader>, TypeReader<DocumentExtent>, ShreddedSource {
446		public ShreddedProcessor processor;
447		ShreddedBuffer buffer;
448	0	DocumentExtent last = new DocumentExtent();
449	0	long updateIdentifierCount = -1;
450	0	long tupleCount = 0;
451	0	long bufferStartCount = 0;
452		ArrayInput input;
453
454	0	public ShreddedReader(ArrayInput input) {
455	0	this.input = input;
456	0	this.buffer = new ShreddedBuffer();
457	0	}
458
459	0	public ShreddedReader(ArrayInput input, int bufferSize) {
460	0	this.input = input;
461	0	this.buffer = new ShreddedBuffer(bufferSize);
462	0	}
463
464		public final int compareTo(ShreddedReader other) {
465	0	ShreddedBuffer otherBuffer = other.getBuffer();
466
467	0	if (buffer.isAtEnd() && otherBuffer.isAtEnd()) {
468	0	return 0;
469	0	} else if (buffer.isAtEnd()) {
470	0	return -1;
471	0	} else if (otherBuffer.isAtEnd()) {
472	0	return 1;
473		}
474
475	0	int result = 0;
476		do {
477	0	result = + Utility.compare(buffer.getIdentifier(), otherBuffer.getIdentifier());
478	0	if(result != 0) break;
479		} while (false);
480
481	0	return result;
482		}
483
484		public final ShreddedBuffer getBuffer() {
485	0	return buffer;
486		}
487
488		public final DocumentExtent read() throws IOException {
489	0	if (buffer.isAtEnd()) {
490	0	fill();
491
492	0	if (buffer.isAtEnd()) {
493	0	return null;
494		}
495		}
496
497	0	assert !buffer.isAtEnd();
498	0	DocumentExtent result = new DocumentExtent();
499
500	0	result.identifier = buffer.getIdentifier();
501	0	result.extentName = buffer.getExtentName();
502	0	result.begin = buffer.getBegin();
503	0	result.end = buffer.getEnd();
504
505	0	buffer.incrementTuple();
506	0	buffer.autoIncrementIdentifier();
507
508	0	return result;
509		}
510
511		public final void fill() throws IOException {
512		try {
513	0	buffer.reset();
514
515	0	if (tupleCount != 0) {
516
517	0	if(updateIdentifierCount - tupleCount > 0) {
518	0	buffer.identifiers.add(last.identifier);
519	0	buffer.identifierTupleIdx.add((int) (updateIdentifierCount - tupleCount));
520		}
521	0	bufferStartCount = tupleCount;
522		}
523
524	0	while (!buffer.isFull()) {
525	0	updateIdentifier();
526	0	buffer.processTuple(input.readString(), input.readInt(), input.readInt());
527	0	tupleCount++;
528		}
529	0	} catch(EOFException e) {}
530	0	}
531
532		public final void updateIdentifier() throws IOException {
533	0	if (updateIdentifierCount > tupleCount)
534	0	return;
535
536	0	last.identifier = input.readString();
537	0	updateIdentifierCount = tupleCount + input.readInt();
538
539	0	buffer.processIdentifier(last.identifier);
540	0	}
541
542		public void run() throws IOException {
543		while (true) {
544	0	fill();
545
546	0	if (buffer.isAtEnd())
547	0	break;
548
549	0	buffer.copyUntil(null, processor);
550		}
551	0	processor.close();
552	0	}
553
554		public void setProcessor(Step processor) throws IncompatibleProcessorException {
555	0	if (processor instanceof ShreddedProcessor) {
556	0	this.processor = new DuplicateEliminator((ShreddedProcessor) processor);
557	0	} else if (processor instanceof DocumentExtent.Processor) {
558	0	this.processor = new DuplicateEliminator(new TupleUnshredder((DocumentExtent.Processor) processor));
559	0	} else if (processor instanceof org.galagosearch.tupleflow.Processor) {
560	0	this.processor = new DuplicateEliminator(new TupleUnshredder((org.galagosearch.tupleflow.Processor<DocumentExtent>) processor));
561		} else {
562	0	throw new IncompatibleProcessorException(processor.getClass().getName() + " is not supported by " + this.getClass().getName());
563		}
564	0	}
565
566		public Class<DocumentExtent> getOutputClass() {
567	0	return DocumentExtent.class;
568		}
569		}
570
571		public static class DuplicateEliminator implements ShreddedProcessor {
572		public ShreddedProcessor processor;
573	0	DocumentExtent last = new DocumentExtent();
574	0	boolean identifierProcess = true;
575
576	0	public DuplicateEliminator() {}
577	0	public DuplicateEliminator(ShreddedProcessor processor) {
578	0	this.processor = processor;
579	0	}
580
581		public void setShreddedProcessor(ShreddedProcessor processor) {
582	0	this.processor = processor;
583	0	}
584
585		public void processIdentifier(String identifier) throws IOException {
586	0	if (identifierProcess \|\| Utility.compare(identifier, last.identifier) != 0) {
587	0	last.identifier = identifier;
588	0	processor.processIdentifier(identifier);
589	0	identifierProcess = false;
590		}
591	0	}
592
593		public void resetIdentifier() {
594	0	identifierProcess = true;
595	0	}
596
597		public void processTuple(String extentName, int begin, int end) throws IOException {
598	0	processor.processTuple(extentName, begin, end);
599	0	}
600
601		public void close() throws IOException {
602	0	processor.close();
603	0	}
604		}
605		public static class TupleUnshredder implements ShreddedProcessor {
606	0	DocumentExtent last = new DocumentExtent();
607		public org.galagosearch.tupleflow.Processor<DocumentExtent> processor;
608
609	0	public TupleUnshredder(DocumentExtent.Processor processor) {
610	0	this.processor = processor;
611	0	}
612
613	0	public TupleUnshredder(org.galagosearch.tupleflow.Processor<DocumentExtent> processor) {
614	0	this.processor = processor;
615	0	}
616
617		public DocumentExtent clone(DocumentExtent object) {
618	0	DocumentExtent result = new DocumentExtent();
619	0	if (object == null) return result;
620	0	result.extentName = object.extentName;
621	0	result.identifier = object.identifier;
622	0	result.begin = object.begin;
623	0	result.end = object.end;
624	0	return result;
625		}
626
627		public void processIdentifier(String identifier) throws IOException {
628	0	last.identifier = identifier;
629	0	}
630
631
632		public void processTuple(String extentName, int begin, int end) throws IOException {
633	0	last.extentName = extentName;
634	0	last.begin = begin;
635	0	last.end = end;
636	0	processor.process(clone(last));
637	0	}
638
639		public void close() throws IOException {
640	0	processor.close();
641	0	}
642		}
643	76	public static class TupleShredder implements Processor {
644	0	DocumentExtent last = new DocumentExtent();
645		public ShreddedProcessor processor;
646
647	0	public TupleShredder(ShreddedProcessor processor) {
648	0	this.processor = processor;
649	0	}
650
651		public DocumentExtent clone(DocumentExtent object) {
652	0	DocumentExtent result = new DocumentExtent();
653	0	if (object == null) return result;
654	0	result.extentName = object.extentName;
655	0	result.identifier = object.identifier;
656	0	result.begin = object.begin;
657	0	result.end = object.end;
658	0	return result;
659		}
660
661		public void process(DocumentExtent object) throws IOException {
662	0	boolean processAll = false;
663	0	if(last == null \|\| Utility.compare(last.identifier, object.identifier) != 0 \|\| processAll) { processor.processIdentifier(object.identifier); processAll = true; }
664	0	processor.processTuple(object.extentName, object.begin, object.end);
665	0	}
666
667		public Class<DocumentExtent> getInputClass() {
668	0	return DocumentExtent.class;
669		}
670
671		public void close() throws IOException {
672	0	processor.close();
673	0	}
674		}
675		}
676		}