Coverage Report

 // BSD License (http://www.galagosearch.org/license)
 package org.galagosearch.core.parse;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.galagosearch.tupleflow.IncompatibleProcessorException;
 import org.galagosearch.tupleflow.InputClass;
 import org.galagosearch.tupleflow.Linkage;
 import org.galagosearch.tupleflow.NullProcessor;
 import org.galagosearch.tupleflow.OutputClass;
 import org.galagosearch.tupleflow.Processor;
 import org.galagosearch.tupleflow.Source;
 import org.galagosearch.tupleflow.Step;
 import org.galagosearch.tupleflow.Utility;
 import org.galagosearch.tupleflow.execution.Verified;
 
 /**
  * <p>This class processes document text into tokens that can be indexed.</p>
  * 
  * <p>The text is assumed to contain some HTML/XML tags.  The tokenizer tries
  * to extract as much data as possible from each document, even if it is not
  * well formed (e.g. there are start tags with no ending tags).  The resulting
  * document object contains an array of terms and an array of tags.</p> 
  * 
  * @author trevor
  */
 @Verified
 @InputClass(className = "org.galagosearch.core.parse.Document")
 @OutputClass(className = "org.galagosearch.core.parse.Document")
 public class TagTokenizer implements Source<Document>, Processor<Document> {
     private static final boolean[] splits;
     private static HashSet<String> ignoredTags;
     public Processor<Document> processor = new NullProcessor(Document.class);
     private StringPooler pooler = new StringPooler();
     private String ignoreUntil;
     
 
     static {
         splits = buildSplits();
         ignoredTags = buildIgnoredTags();
     }
     private String text;
     private int position;
     private int lastSplit;
     ArrayList<String> tokens;
     HashMap<String, ArrayList<BeginTag>> openTags;
     ArrayList<ClosedTag> closedTags;
     ArrayList<Pair> tokenPositions;
 
     public static class Pair {
         public Pair(int start, int end) {
             this.start = start;
             this.end = end;
         }
         public int start;
         public int end;
 
         public String toString() {
             return String.format("%d,%d", start, end);
         }
     }
 
     private enum StringStatus {
         Clean,
         NeedsSimpleFix,
         NeedsComplexFix,
         NeedsAcronymProcessing
     }
 
     public TagTokenizer() {
         text = null;
         position = 0;
         lastSplit = -1;
 
         tokens = new ArrayList<String>();
         openTags = new HashMap<String, ArrayList<BeginTag>>();
         closedTags = new ArrayList<ClosedTag>();
         tokenPositions = new ArrayList<Pair>();
     }
 
     private static boolean[] buildSplits() {
         boolean[] localSplits = new boolean[257];
 
         for (int i = 0; i < localSplits.length; i++) {
             localSplits[i] = false;
         }
         char[] splitChars = {' ', '\t', '\n', '\r', // spaces
             ';', '\"', '&', '/', ':', '!', '#',
             '?', '$', '%', '(', ')', '@', '^',
             '*', '+', '-', ',', '=', '>', '<', '[',
             ']', '{', '}', '|', '`', '~', '_'
         };
 
         for (char c : splitChars) {
             localSplits[(byte) c] = true;
         }
 
         for (byte c = 0; c <= 32; c++) {
             localSplits[c] = true;
         }
 
         return localSplits;
     }
 
     private static HashSet<String> buildIgnoredTags() {
         HashSet<String> tags = new HashSet<String>();
         tags.add("style");
         tags.add("script");
         return tags;
     }
 
     class ClosedTag {
         public ClosedTag(BeginTag begin) {
             this.name = begin.name;
             this.attributes = begin.attributes;
 
             this.byteStart = begin.bytePosition;
             this.termStart = begin.termPosition;
 
             this.byteEnd = position;
             this.termEnd = tokens.size();
         }
         String name;
         Map<String, String> attributes;
         int byteStart;
         int termStart;
         int byteEnd;
         int termEnd;
     }
 
     class BeginTag {
         public BeginTag(String name, Map<String, String> attributes) {
             this.name = name;
             this.attributes = attributes;
 
             this.bytePosition = position;
             this.termPosition = tokens.size();
         }
         String name;
         Map<String, String> attributes;
         int bytePosition;
         int termPosition;
     }
 
     /**
      * Resets parsing in preparation for the next document.
      */
     public void reset() {
         ignoreUntil = null;
         text = null;
         position = 0;
         lastSplit = -1;
 
         tokens.clear();
         openTags.clear();
         closedTags.clear();
 
         if (tokenPositions != null) {
             tokenPositions.clear();
         }
     }
 
     private void skipComment() {
         if (text.substring(position).startsWith("<!--")) {
             position = text.indexOf("-->", position + 1);
 
             if (position >= 0) {
                 position += 2;
             }
         } else {
             position = text.indexOf(">", position + 1);
         }
 
         if (position < 0) {
             position = text.length();
         }
     }
 
     private void skipProcessingInstruction() {
         position = text.indexOf("?>", position + 1);
 
         if (position < 0) {
             position = text.length();
         }
     }
 
     private void parseEndTag() {
         // 1. read name (skipping the </ part)
         int i;
 
         for (i = position + 2; i < text.length(); i++) {
             char c = text.charAt(i);
             if (Character.isSpaceChar(c) || c == '>') {
                 break;
             }
         }
 
         String tagName = text.substring(position + 2, i).toLowerCase();
 
         if (ignoreUntil != null && ignoreUntil.equals(tagName)) {
             ignoreUntil = null;
         }
         if (ignoreUntil == null) {
             closeTag(tagName);        // advance to end '>'
         }
         while (i < text.length() && text.charAt(i) != '>') {
             i++;
         }
         position = i;
     }
 
     private void closeTag(final String tagName) {
         if (!openTags.containsKey(tagName)) {
             return;
         }
         ArrayList<BeginTag> tagList = openTags.get(tagName);
 
         if (tagList.size() > 0) {
             int last = tagList.size() - 1;
 
             BeginTag openTag = tagList.get(last);
             ClosedTag closedTag = new ClosedTag(openTag);
             closedTags.add(closedTag);
 
             tagList.remove(last);
         }
     }
 
     private int indexOfNonSpace(int start) {
         if (start < 0) {
             return Integer.MIN_VALUE;
         }
         for (int i = start; i < text.length(); i++) {
             char c = text.charAt(i);
             if (!Character.isSpaceChar(c)) {
                 return i;
             }
         }
 
         return Integer.MIN_VALUE;
     }
 
     private int indexOfEndAttribute(int start, int tagEnd) {
         if (start < 0) {
             return Integer.MIN_VALUE;        // attribute ends at the first non-quoted space, or
         // the first '>'.
         }
         boolean inQuote = false;
         boolean lastEscape = false;
 
         for (int i = start; i <= tagEnd; i++) {
             char c = text.charAt(i);
 
             if ((c == '\"' || c == '\'') && !lastEscape) {
                 inQuote = !inQuote;
                 if (!inQuote) {
                     return i;
                 }
             } else if (!inQuote && (Character.isSpaceChar(c) || c == '>')) {
                 return i;
             } else if (c == '\\' && !lastEscape) {
                 lastEscape = true;
             } else {
                 lastEscape = false;
             }
         }
 
         return Integer.MIN_VALUE;
     }
 
     private int indexOfSpace(int start) {
         if (start < 0) {
             return Integer.MIN_VALUE;
         }
         for (int i = start; i < text.length(); i++) {
             char c = text.charAt(i);
             if (Character.isSpaceChar(c)) {
                 return i;
             }
         }
 
         return Integer.MIN_VALUE;
     }
 
     private int indexOfEquals(int start, int end) {
         if (start < 0) {
             return Integer.MIN_VALUE;
         }
         for (int i = start; i < end; i++) {
             char c = text.charAt(i);
             if (c == '=') {
                 return i;
             }
         }
 
         return Integer.MIN_VALUE;
     }
 
     private void parseBeginTag() {
         // 1. read the name, skipping the '<'
         int i;
 
         for (i = position + 1; i < text.length(); i++) {
             char c = text.charAt(i);
             if (Character.isSpaceChar(c) || c == '>') {
                 break;
             }
         }
 
         String tagName = text.substring(position + 1, i).toLowerCase();
 
         // 2. read attr pairs
         i = indexOfNonSpace(i);
         int tagEnd = text.indexOf(">", i + 1);
         boolean closeIt = false;
 
         HashMap<String, String> attributes = new HashMap<String, String>();
         while (i < tagEnd && i >= 0 && tagEnd >= 0) {
             // scan ahead for non space
             int start = indexOfNonSpace(i);
 
             if (start > 0) {
                 if (text.charAt(start) == '>') {
                     i = start;
                     break;
                 } else if (text.charAt(start) == '/' &&
                         text.length() > start + 1 &&
                         text.charAt(start + 1) == '>') {
                     i = start + 1;
                     closeIt = true;
                     break;
                 }
             }
 
             int end = indexOfEndAttribute(start, tagEnd);
             int equals = indexOfEquals(start, end);
 
             // try to find an equals sign
             if (equals < 0 || equals == start || end == equals) {
                 // if there's no equals, try to move to the next thing
                 if (end < 0) {
                     i = tagEnd;
                     break;
                 } else {
                     i = end;
                     continue;
                 }
             }
 
             // there is an equals, so try to parse the value
             int startKey = start;
             int endKey = equals;
 
             int startValue = equals + 1;
             int endValue = end;
 
             if (text.charAt(startValue) == '\"' || text.charAt(startValue) == '\'') {
                 startValue++;
             }
             if (startValue >= endValue || startKey >= endKey) {
                 i = end;
                 continue;
             }
 
             String key = text.substring(startKey, endKey);
             String value = text.substring(startValue, endValue);
 
             attributes.put(key.toLowerCase(), value);
 
             if (end >= text.length()) {
                 endParsing();
                 break;
             }
 
             if (text.charAt(end) == '\"' || text.charAt(end) == '\'') {
                 end++;
             }
 
             i = end;
         }
 
         if (!ignoredTags.contains(tagName)) {
             BeginTag tag = new BeginTag(tagName, attributes);
 
             if (!openTags.containsKey(tagName)) {
                 ArrayList tagList = new ArrayList();
                 tagList.add(tag);
                 openTags.put(tagName, tagList);
             } else {
                 openTags.get(tagName).add(tag);
             }
 
             if (closeIt) {
                 closeTag(tagName);
             }
         } else if (!closeIt) {
             ignoreUntil = tagName;
         }
 
         position = i;
     }
 
     private void endParsing() {
         position = text.length();
     }
 
     private void onSplit() {
         if (position - lastSplit > 1) {
             int start = lastSplit + 1;
             String token = text.substring(start, position);
             StringStatus status = checkTokenStatus(token);
 
             switch (status) {
                 case NeedsSimpleFix:
                     token = tokenSimpleFix(token);
                     break;
 
                 case NeedsComplexFix:
                     token = tokenComplexFix(token);
                     break;
 
                 case NeedsAcronymProcessing:
                     tokenAcronymProcessing(token, start, position);
                     break;
 
                 case Clean:
                     // do nothing
                     break;
             }
 
             if (status != StringStatus.NeedsAcronymProcessing) {
                 addToken(token, start, position);
             }
         }
 
         lastSplit = position;
     }
 
     /**
      * Adds a token to the document object.  This method currently drops tokens 
      * longer than 100 bytes long right now.
      * 
      * @param token  The token to add.
      * @param start  The starting byte offset of the token in the document text.
      * @param end    The ending byte offset of the token in the document text.
      */
     private void addToken(final String token, int start, int end) {
         final int maxTokenLength = 100;
         // zero length tokens aren't interesting
         if (token.length() <= 0) {
             return;
         }
         // we want to make sure the token is short enough that someone
         // might actually type it.  UTF-8 can expand one character to 6 bytes.
         if (token.length() > maxTokenLength / 6 &&
             Utility.makeBytes(token).length >= maxTokenLength) {
             return;
         }
         tokens.add(token);
         tokenPositions.add(new Pair(start, end));
     }
 
     private String tokenComplexFix(String token) {
         token = tokenSimpleFix(token);
         token = token.toLowerCase();
 
         return token;
     }
 
     /**
      * This method does three kinds of processing:
      * <ul>
      *  <li>If the token contains periods at the beginning or the end,
      *      they are removed.</li>
      *  <li>If the token contains single letters followed by periods, such
      *      as I.B.M., C.I.A., or U.S.A., the periods are removed.</li>
      *  <li>If, instead, the token contains longer strings of text with
      *      periods in the middle, the token is split into 
      *      smaller tokens ("umass.edu" becomes {"umass", "edu"}).  Notice
      *      that this means ("ph.d." becomes {"ph", "d"}).</li>
      * </ul>
      * 
      * @param token
      * @param start
      * @param end
      */
     private void tokenAcronymProcessing(String token, int start, int end) {
         token = tokenComplexFix(token);
 
         // remove start and ending periods
         while (token.startsWith(".")) {
             token = token.substring(1);
             start = start + 1;
         }
 
         while (token.endsWith(".")) {
             token = token.substring(0, token.length() - 1);
             end -= 1;
         }
 
         // does the token have any periods left?
         if (token.indexOf('.') >= 0) {
             // is this an acronym?  then there will be periods
             // at odd positions:
             boolean isAcronym = token.length() > 0;
             for (int pos = 1; pos < token.length(); pos += 2) {
                 if (token.charAt(pos) != '.') {
                     isAcronym = false;
                 }
             }
 
             if (isAcronym) {
                 token = token.replace(".", "");
                 addToken(token, start, end);
             } else {
                 int s = 0;
                 for (int e = 0; e < token.length(); e++) {
                     if (token.charAt(e) == '.') {
                         if (e - s > 1) {
                             String subtoken = token.substring(s, e);
                             addToken(subtoken, start + s, start + e);
                         }
                         s = e + 1;
                     }
                 }
 
                 if (token.length() - s > 1) {
                     String subtoken = token.substring(s);
                     addToken(subtoken, start + s, end);
                 }
             }
         } else {
             addToken(token, start, end);
         }
     }
 
     /**
      * Scans through the token, removing apostrophes and converting
      * uppercase to lowercase letters.
      * 
      * @param token
      * @return
      */
     private String tokenSimpleFix(String token) {
         char[] chars = token.toCharArray();
         int j = 0;
 
         for (int i = 0; i < chars.length; i++) {
             char c = chars[i];
             boolean isAsciiUppercase = (c >= 'A' && c <= 'Z');
             boolean isApostrophe = (c == '\'');
 
             if (isAsciiUppercase) {
                 chars[j] = (char) (chars[i] + 'a' - 'A');
             } else if (isApostrophe) {
                 // it's an apostrophe, skip it
                 j--;
             } else {
                 chars[j] = chars[i];
             }
 
             j++;
         }
 
         token = new String(chars, 0, j);
         return token;
     }
 
     /**
      * This method scans the token, looking for uppercase characters and
      * special characters.  If the token contains only numbers and lowercase
      * letters, it needs no further processing, and it returns Clean.
      * If it also contains uppercase letters or apostrophes, it returns 
      * NeedsSimpleFix.  If it contains special characters (especially Unicode
      * characters), it returns NeedsComplexFix.  Finally, if any periods are
      * present, this returns NeedsAcronymProcessing.
      * 
      * @param token
      * @return
      */
     private StringStatus checkTokenStatus(final String token) {
         StringStatus status = StringStatus.Clean;
         char[] chars = token.toCharArray();
 
         for (int i = 0; i < chars.length; i++) {
             char c = chars[i];
             boolean isAsciiLowercase = (c >= 'a' && c <= 'z');
             boolean isAsciiNumber = (c >= '0' && c <= '9');
 
             if (isAsciiLowercase || isAsciiNumber) {
                 continue;
             }
             boolean isAsciiUppercase = (c >= 'A' && c <= 'Z');
             boolean isPeriod = (c == '.');
             boolean isApostrophe = (c == '\'');
 
             if ((isAsciiUppercase || isApostrophe) && status == StringStatus.Clean) {
                 status = StringStatus.NeedsSimpleFix;
             } else if (!isPeriod) {
                 status = StringStatus.NeedsComplexFix;
             } else {
                 status = StringStatus.NeedsAcronymProcessing;
                 break;
             }
         }
 
         return status;
     }
 
     private void onStartBracket() {
         if (position + 1 < text.length()) {
             char c = text.charAt(position + 1);
 
             if (c == '/') {
                 parseEndTag();
             } else if (c == '!') {
                 skipComment();
             } else if (c == '?') {
                 skipProcessingInstruction();
             } else {
                 parseBeginTag();
             }
         } else {
             endParsing();
         }
 
         lastSplit = position;
     }
 
     /**
      * Translates tags from the internal ClosedTag format to the
      * Tag type.
      */
     private ArrayList<Tag> coalesceTags() {
         ArrayList<Tag> result = new ArrayList();
 
         // close all open tags
         for (ArrayList<BeginTag> tagList : openTags.values()) {
             for (BeginTag tag : tagList) {
                 result.add(new Tag(tag.name, tag.attributes, tag.termPosition, tag.termPosition));
             }
         }
 
         for (ClosedTag tag : closedTags) {
             result.add(new Tag(tag.name, tag.attributes, tag.termStart, tag.termEnd));
         }
 
         Collections.sort(result);
         return result;
     }
 
     public void onAmpersand() {
         onSplit();
 
         for (int i = position + 1; i < text.length(); i++) {
             char c = text.charAt(i);
 
             if (c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '#') {
                 continue;
             }
             if (c == ';') {
                 position = i;
                 lastSplit = i;
                 return;
             }
 
             // not a valid escape sequence
             break;
         }
     }
 
     /**
      * Parses the text in the document.text attribute and fills in the 
      * document.terms and document.tags arrays, then passes that document
      * to the next processing stage.
      * 
      * @param document
      * @throws java.io.IOException
      */
     public void process(Document document) throws IOException {
         tokenize(document);
         processor.process(document);
     }
 
     /**
      * Parses the text in the document.text attribute and fills in the 
      * document.terms and document.tags arrays.
      * 
      * @param document
      * @throws java.io.IOException
      */
     public void tokenize(Document document) {
         reset();
         text = document.text;
 
         try {
             // this loop is looking for tags, split characters, and XML escapes,
             // which start with ampersands.  All other characters are assumed to
             // be word characters.  The onSplit() method takes care of extracting
             // word text and storing it in the terms array.  The onStartBracket
             // method parses tags.  ignoreUntil is used to ignore comments and
             // script data.
             for (; position >= 0 && position < text.length(); position++) {
                 char c = text.charAt(position);
 
                 if (c == '<') {
                     if (ignoreUntil == null) {
                         onSplit();
                     }
                     onStartBracket();
                 } else if (ignoreUntil != null) {
                     continue;
                 } else if (c == '&') {
                     onAmpersand();
                 } else if (c < 256 && splits[c]) {
                     onSplit();
                 }
             }
         } catch (Exception e) {
             Logger.getLogger(getClass().toString()).log(Level.WARNING,
                                                         "Parse failure: " + document.identifier);
         }
 
         if (ignoreUntil == null) {
             onSplit();
         }
         document.terms = new ArrayList<String>(this.tokens);
         document.tags = coalesceTags();
         pooler.transform(document);
     }
 
     /**
      * Parses the text in the input string and returns a document object.
      * This method calls the {#link tokenize(Document) other variant}.
      * 
      * @return A new document object containing the parsed text from the input string.
      */
     public Document tokenize(String text) throws IOException {
         Document document = new Document();
         document.text = text;
         tokenize(document);
 
         return document;
     }
 
     public ArrayList<Pair> getTokenPositions() {
         return this.tokenPositions;
     }
 
     public void setProcessor(final Step processor) throws IncompatibleProcessorException {
         Linkage.link(this, processor);
     }
 
     public void close() throws IOException {
         processor.close();
     }
 
     public Class<Document> getInputClass() {
         return Document.class;
     }
 
     public Class<Document> getOutputClass() {
         return Document.class;
     }
 }

1		// BSD License (http://www.galagosearch.org/license)
2		package org.galagosearch.core.parse;
3
4		import java.io.IOException;
5		import java.util.ArrayList;
6		import java.util.Collections;
7		import java.util.HashMap;
8		import java.util.HashSet;
9		import java.util.Map;
10		import java.util.logging.Level;
11		import java.util.logging.Logger;
12		import org.galagosearch.tupleflow.IncompatibleProcessorException;
13		import org.galagosearch.tupleflow.InputClass;
14		import org.galagosearch.tupleflow.Linkage;
15		import org.galagosearch.tupleflow.NullProcessor;
16		import org.galagosearch.tupleflow.OutputClass;
17		import org.galagosearch.tupleflow.Processor;
18		import org.galagosearch.tupleflow.Source;
19		import org.galagosearch.tupleflow.Step;
20		import org.galagosearch.tupleflow.Utility;
21		import org.galagosearch.tupleflow.execution.Verified;
22
23		/**
24		* <p>This class processes document text into tokens that can be indexed.</p>
25		*
26		* <p>The text is assumed to contain some HTML/XML tags. The tokenizer tries
27		* to extract as much data as possible from each document, even if it is not
28		* well formed (e.g. there are start tags with no ending tags). The resulting
29		* document object contains an array of terms and an array of tags.</p>
30		*
31		* @author trevor
32		*/
33		@Verified
34		@InputClass(className = "org.galagosearch.core.parse.Document")
35		@OutputClass(className = "org.galagosearch.core.parse.Document")
36	24	public class TagTokenizer implements Source<Document>, Processor<Document> {
37		private static final boolean[] splits;
38		private static HashSet<String> ignoredTags;
39	16	public Processor<Document> processor = new NullProcessor(Document.class);
40	16	private StringPooler pooler = new StringPooler();
41		private String ignoreUntil;
42
43
44		static {
45	4	splits = buildSplits();
46	4	ignoredTags = buildIgnoredTags();
47	4	}
48		private String text;
49		private int position;
50		private int lastSplit;
51		ArrayList<String> tokens;
52		HashMap<String, ArrayList<BeginTag>> openTags;
53		ArrayList<ClosedTag> closedTags;
54		ArrayList<Pair> tokenPositions;
55
56		public static class Pair {
57	80	public Pair(int start, int end) {
58	80	this.start = start;
59	80	this.end = end;
60	80	}
61		public int start;
62		public int end;
63
64		public String toString() {
65	0	return String.format("%d,%d", start, end);
66		}
67		}
68
69	24	private enum StringStatus {
70	4	Clean,
71	4	NeedsSimpleFix,
72	4	NeedsComplexFix,
73	4	NeedsAcronymProcessing
74		}
75
76	16	public TagTokenizer() {
77	16	text = null;
78	16	position = 0;
79	16	lastSplit = -1;
80
81	16	tokens = new ArrayList<String>();
82	16	openTags = new HashMap<String, ArrayList<BeginTag>>();
83	16	closedTags = new ArrayList<ClosedTag>();
84	16	tokenPositions = new ArrayList<Pair>();
85	16	}
86
87		private static boolean[] buildSplits() {
88	4	boolean[] localSplits = new boolean[257];
89
90	1032	for (int i = 0; i < localSplits.length; i++) {
91	1028	localSplits[i] = false;
92		}
93	4	char[] splitChars = {' ', '\t', '\n', '\r', // spaces
94		';', '\"', '&', '/', ':', '!', '#',
95		'?', '$', '%', '(', ')', '@', '^',
96		'*', '+', '-', ',', '=', '>', '<', '[',
97		']', '{', '}', '\|', '`', '~', '_'
98		};
99
100	136	for (char c : splitChars) {
101	132	localSplits[(byte) c] = true;
102		}
103
104	136	for (byte c = 0; c <= 32; c++) {
105	132	localSplits[c] = true;
106		}
107
108	4	return localSplits;
109		}
110
111		private static HashSet<String> buildIgnoredTags() {
112	4	HashSet<String> tags = new HashSet<String>();
113	4	tags.add("style");
114	4	tags.add("script");
115	4	return tags;
116		}
117
118		class ClosedTag {
119	12	public ClosedTag(BeginTag begin) {
120	12	this.name = begin.name;
121	12	this.attributes = begin.attributes;
122
123	12	this.byteStart = begin.bytePosition;
124	12	this.termStart = begin.termPosition;
125
126	12	this.byteEnd = position;
127	12	this.termEnd = tokens.size();
128	12	}
129		String name;
130		Map<String, String> attributes;
131		int byteStart;
132		int termStart;
133		int byteEnd;
134		int termEnd;
135		}
136
137		class BeginTag {
138	12	public BeginTag(String name, Map<String, String> attributes) {
139	12	this.name = name;
140	12	this.attributes = attributes;
141
142	12	this.bytePosition = position;
143	12	this.termPosition = tokens.size();
144	12	}
145		String name;
146		Map<String, String> attributes;
147		int bytePosition;
148		int termPosition;
149		}
150
151		/**
152		* Resets parsing in preparation for the next document.
153		*/
154		public void reset() {
155	16	ignoreUntil = null;
156	16	text = null;
157	16	position = 0;
158	16	lastSplit = -1;
159
160	16	tokens.clear();
161	16	openTags.clear();
162	16	closedTags.clear();
163
164	16	if (tokenPositions != null) {
165	16	tokenPositions.clear();
166		}
167	16	}
168
169		private void skipComment() {
170	0	if (text.substring(position).startsWith("<!--")) {
171	0	position = text.indexOf("-->", position + 1);
172
173	0	if (position >= 0) {
174	0	position += 2;
175		}
176		} else {
177	0	position = text.indexOf(">", position + 1);
178		}
179
180	0	if (position < 0) {
181	0	position = text.length();
182		}
183	0	}
184
185		private void skipProcessingInstruction() {
186	0	position = text.indexOf("?>", position + 1);
187
188	0	if (position < 0) {
189	0	position = text.length();
190		}
191	0	}
192
193		private void parseEndTag() {
194		// 1. read name (skipping the </ part)
195		int i;
196
197	52	for (i = position + 2; i < text.length(); i++) {
198	52	char c = text.charAt(i);
199	52	if (Character.isSpaceChar(c) \|\| c == '>') {
200	12	break;
201		}
202		}
203
204	12	String tagName = text.substring(position + 2, i).toLowerCase();
205
206	12	if (ignoreUntil != null && ignoreUntil.equals(tagName)) {
207	0	ignoreUntil = null;
208		}
209	12	if (ignoreUntil == null) {
210	12	closeTag(tagName); // advance to end '>'
211		}
212	12	while (i < text.length() && text.charAt(i) != '>') {
213	0	i++;
214		}
215	12	position = i;
216	12	}
217
218		private void closeTag(final String tagName) {
219	12	if (!openTags.containsKey(tagName)) {
220	0	return;
221		}
222	12	ArrayList<BeginTag> tagList = openTags.get(tagName);
223
224	12	if (tagList.size() > 0) {
225	12	int last = tagList.size() - 1;
226
227	12	BeginTag openTag = tagList.get(last);
228	12	ClosedTag closedTag = new ClosedTag(openTag);
229	12	closedTags.add(closedTag);
230
231	12	tagList.remove(last);
232		}
233	12	}
234
235		private int indexOfNonSpace(int start) {
236	24	if (start < 0) {
237	0	return Integer.MIN_VALUE;
238		}
239	28	for (int i = start; i < text.length(); i++) {
240	28	char c = text.charAt(i);
241	28	if (!Character.isSpaceChar(c)) {
242	24	return i;
243		}
244		}
245
246	0	return Integer.MIN_VALUE;
247		}
248
249		private int indexOfEndAttribute(int start, int tagEnd) {
250	4	if (start < 0) {
251	0	return Integer.MIN_VALUE; // attribute ends at the first non-quoted space, or
252		// the first '>'.
253		}
254	4	boolean inQuote = false;
255	4	boolean lastEscape = false;
256
257	108	for (int i = start; i <= tagEnd; i++) {
258	108	char c = text.charAt(i);
259
260	108	if ((c == '\"' \|\| c == '\'') && !lastEscape) {
261	8	inQuote = !inQuote;
262	8	if (!inQuote) {
263	4	return i;
264		}
265	100	} else if (!inQuote && (Character.isSpaceChar(c) \|\| c == '>')) {
266	0	return i;
267	100	} else if (c == '\\' && !lastEscape) {
268	0	lastEscape = true;
269		} else {
270	100	lastEscape = false;
271		}
272		}
273
274	0	return Integer.MIN_VALUE;
275		}
276
277		private int indexOfSpace(int start) {
278	0	if (start < 0) {
279	0	return Integer.MIN_VALUE;
280		}
281	0	for (int i = start; i < text.length(); i++) {
282	0	char c = text.charAt(i);
283	0	if (Character.isSpaceChar(c)) {
284	0	return i;
285		}
286		}
287
288	0	return Integer.MIN_VALUE;
289		}
290
291		private int indexOfEquals(int start, int end) {
292	4	if (start < 0) {
293	0	return Integer.MIN_VALUE;
294		}
295	20	for (int i = start; i < end; i++) {
296	20	char c = text.charAt(i);
297	20	if (c == '=') {
298	4	return i;
299		}
300		}
301
302	0	return Integer.MIN_VALUE;
303		}
304
305		private void parseBeginTag() {
306		// 1. read the name, skipping the '<'
307		int i;
308
309	52	for (i = position + 1; i < text.length(); i++) {
310	52	char c = text.charAt(i);
311	52	if (Character.isSpaceChar(c) \|\| c == '>') {
312	8	break;
313		}
314		}
315
316	12	String tagName = text.substring(position + 1, i).toLowerCase();
317
318		// 2. read attr pairs
319	12	i = indexOfNonSpace(i);
320	12	int tagEnd = text.indexOf(">", i + 1);
321	12	boolean closeIt = false;
322
323	12	HashMap<String, String> attributes = new HashMap<String, String>();
324	16	while (i < tagEnd && i >= 0 && tagEnd >= 0) {
325		// scan ahead for non space
326	12	int start = indexOfNonSpace(i);
327
328	12	if (start > 0) {
329	12	if (text.charAt(start) == '>') {
330	8	i = start;
331	8	break;
332	4	} else if (text.charAt(start) == '/' &&
333		text.length() > start + 1 &&
334		text.charAt(start + 1) == '>') {
335	0	i = start + 1;
336	0	closeIt = true;
337	0	break;
338		}
339		}
340
341	4	int end = indexOfEndAttribute(start, tagEnd);
342	4	int equals = indexOfEquals(start, end);
343
344		// try to find an equals sign
345	4	if (equals < 0 \|\| equals == start \|\| end == equals) {
346		// if there's no equals, try to move to the next thing
347	0	if (end < 0) {
348	0	i = tagEnd;
349	0	break;
350		} else {
351	0	i = end;
352	0	continue;
353		}
354		}
355
356		// there is an equals, so try to parse the value
357	4	int startKey = start;
358	4	int endKey = equals;
359
360	4	int startValue = equals + 1;
361	4	int endValue = end;
362
363	4	if (text.charAt(startValue) == '\"' \|\| text.charAt(startValue) == '\'') {
364	4	startValue++;
365		}
366	4	if (startValue >= endValue \|\| startKey >= endKey) {
367	0	i = end;
368	0	continue;
369		}
370
371	4	String key = text.substring(startKey, endKey);
372	4	String value = text.substring(startValue, endValue);
373
374	4	attributes.put(key.toLowerCase(), value);
375
376	4	if (end >= text.length()) {
377	0	endParsing();
378	0	break;
379		}
380
381	4	if (text.charAt(end) == '\"' \|\| text.charAt(end) == '\'') {
382	4	end++;
383		}
384
385	4	i = end;
386	4	}
387
388	12	if (!ignoredTags.contains(tagName)) {
389	12	BeginTag tag = new BeginTag(tagName, attributes);
390
391	12	if (!openTags.containsKey(tagName)) {
392	12	ArrayList tagList = new ArrayList();
393	12	tagList.add(tag);
394	12	openTags.put(tagName, tagList);
395	12	} else {
396	0	openTags.get(tagName).add(tag);
397		}
398
399	12	if (closeIt) {
400	0	closeTag(tagName);
401		}
402	12	} else if (!closeIt) {
403	0	ignoreUntil = tagName;
404		}
405
406	12	position = i;
407	12	}
408
409		private void endParsing() {
410	0	position = text.length();
411	0	}
412
413		private void onSplit() {
414	104	if (position - lastSplit > 1) {
415	68	int start = lastSplit + 1;
416	68	String token = text.substring(start, position);
417	68	StringStatus status = checkTokenStatus(token);
418
419	68	switch (status) {
420		case NeedsSimpleFix:
421	8	token = tokenSimpleFix(token);
422	8	break;
423
424		case NeedsComplexFix:
425	4	token = tokenComplexFix(token);
426	4	break;
427
428		case NeedsAcronymProcessing:
429	8	tokenAcronymProcessing(token, start, position);
430	8	break;
431
432		case Clean:
433		// do nothing
434		break;
435		}
436
437	68	if (status != StringStatus.NeedsAcronymProcessing) {
438	60	addToken(token, start, position);
439		}
440		}
441
442	104	lastSplit = position;
443	104	}
444
445		/**
446		* Adds a token to the document object. This method currently drops tokens
447		* longer than 100 bytes long right now.
448		*
449		* @param token The token to add.
450		* @param start The starting byte offset of the token in the document text.
451		* @param end The ending byte offset of the token in the document text.
452		*/
453		private void addToken(final String token, int start, int end) {
454	80	final int maxTokenLength = 100;
455		// zero length tokens aren't interesting
456	80	if (token.length() <= 0) {
457	0	return;
458		}
459		// we want to make sure the token is short enough that someone
460		// might actually type it. UTF-8 can expand one character to 6 bytes.
461	80	if (token.length() > maxTokenLength / 6 &&
462		Utility.makeBytes(token).length >= maxTokenLength) {
463	0	return;
464		}
465	80	tokens.add(token);
466	80	tokenPositions.add(new Pair(start, end));
467	80	}
468
469		private String tokenComplexFix(String token) {
470	12	token = tokenSimpleFix(token);
471	12	token = token.toLowerCase();
472
473	12	return token;
474		}
475
476		/**
477		* This method does three kinds of processing:
478		* <ul>
479		* <li>If the token contains periods at the beginning or the end,
480		* they are removed.</li>
481		* <li>If the token contains single letters followed by periods, such
482		* as I.B.M., C.I.A., or U.S.A., the periods are removed.</li>
483		* <li>If, instead, the token contains longer strings of text with
484		* periods in the middle, the token is split into
485		* smaller tokens ("umass.edu" becomes {"umass", "edu"}). Notice
486		* that this means ("ph.d." becomes {"ph", "d"}).</li>
487		* </ul>
488		*
489		* @param token
490		* @param start
491		* @param end
492		*/
493		private void tokenAcronymProcessing(String token, int start, int end) {
494	8	token = tokenComplexFix(token);
495
496		// remove start and ending periods
497	8	while (token.startsWith(".")) {
498	0	token = token.substring(1);
499	0	start = start + 1;
500		}
501
502	12	while (token.endsWith(".")) {
503	4	token = token.substring(0, token.length() - 1);
504	4	end -= 1;
505		}
506
507		// does the token have any periods left?
508	8	if (token.indexOf('.') >= 0) {
509		// is this an acronym? then there will be periods
510		// at odd positions:
511	8	boolean isAcronym = token.length() > 0;
512	48	for (int pos = 1; pos < token.length(); pos += 2) {
513	40	if (token.charAt(pos) != '.') {
514	24	isAcronym = false;
515		}
516		}
517
518	8	if (isAcronym) {
519	4	token = token.replace(".", "");
520	4	addToken(token, start, end);
521		} else {
522	4	int s = 0;
523	72	for (int e = 0; e < token.length(); e++) {
524	68	if (token.charAt(e) == '.') {
525	12	if (e - s > 1) {
526	12	String subtoken = token.substring(s, e);
527	12	addToken(subtoken, start + s, start + e);
528		}
529	12	s = e + 1;
530		}
531		}
532
533	4	if (token.length() - s > 1) {
534	4	String subtoken = token.substring(s);
535	4	addToken(subtoken, start + s, end);
536		}
537		}
538	8	} else {
539	0	addToken(token, start, end);
540		}
541	8	}
542
543		/**
544		* Scans through the token, removing apostrophes and converting
545		* uppercase to lowercase letters.
546		*
547		* @param token
548		* @return
549		*/
550		private String tokenSimpleFix(String token) {
551	20	char[] chars = token.toCharArray();
552	20	int j = 0;
553
554	172	for (int i = 0; i < chars.length; i++) {
555	152	char c = chars[i];
556	152	boolean isAsciiUppercase = (c >= 'A' && c <= 'Z');
557	152	boolean isApostrophe = (c == '\'');
558
559	152	if (isAsciiUppercase) {
560	12	chars[j] = (char) (chars[i] + 'a' - 'A');
561	140	} else if (isApostrophe) {
562		// it's an apostrophe, skip it
563	4	j--;
564		} else {
565	136	chars[j] = chars[i];
566		}
567
568	152	j++;
569		}
570
571	20	token = new String(chars, 0, j);
572	20	return token;
573		}
574
575		/**
576		* This method scans the token, looking for uppercase characters and
577		* special characters. If the token contains only numbers and lowercase
578		* letters, it needs no further processing, and it returns Clean.
579		* If it also contains uppercase letters or apostrophes, it returns
580		* NeedsSimpleFix. If it contains special characters (especially Unicode
581		* characters), it returns NeedsComplexFix. Finally, if any periods are
582		* present, this returns NeedsAcronymProcessing.
583		*
584		* @param token
585		* @return
586		*/
587		private StringStatus checkTokenStatus(final String token) {
588	68	StringStatus status = StringStatus.Clean;
589	68	char[] chars = token.toCharArray();
590
591	344	for (int i = 0; i < chars.length; i++) {
592	284	char c = chars[i];
593	284	boolean isAsciiLowercase = (c >= 'a' && c <= 'z');
594	284	boolean isAsciiNumber = (c >= '0' && c <= '9');
595
596	284	if (isAsciiLowercase \|\| isAsciiNumber) {
597	0	continue;
598		}
599	24	boolean isAsciiUppercase = (c >= 'A' && c <= 'Z');
600	24	boolean isPeriod = (c == '.');
601	24	boolean isApostrophe = (c == '\'');
602
603	24	if ((isAsciiUppercase \|\| isApostrophe) && status == StringStatus.Clean) {
604	12	status = StringStatus.NeedsSimpleFix;
605	12	} else if (!isPeriod) {
606	4	status = StringStatus.NeedsComplexFix;
607		} else {
608	8	status = StringStatus.NeedsAcronymProcessing;
609	8	break;
610		}
611		}
612
613	68	return status;
614		}
615
616		private void onStartBracket() {
617	24	if (position + 1 < text.length()) {
618	24	char c = text.charAt(position + 1);
619
620	24	if (c == '/') {
621	12	parseEndTag();
622	12	} else if (c == '!') {
623	0	skipComment();
624	12	} else if (c == '?') {
625	0	skipProcessingInstruction();
626		} else {
627	12	parseBeginTag();
628		}
629	24	} else {
630	0	endParsing();
631		}
632
633	24	lastSplit = position;
634	24	}
635
636		/**
637		* Translates tags from the internal ClosedTag format to the
638		* Tag type.
639		*/
640		private ArrayList<Tag> coalesceTags() {
641	12	ArrayList<Tag> result = new ArrayList();
642
643		// close all open tags
644	12	for (ArrayList<BeginTag> tagList : openTags.values()) {
645	12	for (BeginTag tag : tagList) {
646	0	result.add(new Tag(tag.name, tag.attributes, tag.termPosition, tag.termPosition));
647		}
648		}
649
650	12	for (ClosedTag tag : closedTags) {
651	12	result.add(new Tag(tag.name, tag.attributes, tag.termStart, tag.termEnd));
652		}
653
654	12	Collections.sort(result);
655	12	return result;
656		}
657
658		public void onAmpersand() {
659	0	onSplit();
660
661	0	for (int i = position + 1; i < text.length(); i++) {
662	0	char c = text.charAt(i);
663
664	0	if (c >= 'a' && c <= 'z' \|\| c >= '0' && c <= '9' \|\| c == '#') {
665	0	continue;
666		}
667	0	if (c == ';') {
668	0	position = i;
669	0	lastSplit = i;
670	0	return;
671		}
672
673		// not a valid escape sequence
674		break;
675		}
676	0	}
677
678		/**
679		* Parses the text in the document.text attribute and fills in the
680		* document.terms and document.tags arrays, then passes that document
681		* to the next processing stage.
682		*
683		* @param document
684		* @throws java.io.IOException
685		*/
686		public void process(Document document) throws IOException {
687	12	tokenize(document);
688	12	processor.process(document);
689	12	}
690
691		/**
692		* Parses the text in the document.text attribute and fills in the
693		* document.terms and document.tags arrays.
694		*
695		* @param document
696		* @throws java.io.IOException
697		*/
698		public void tokenize(Document document) {
699	12	reset();
700	12	text = document.text;
701
702		try {
703		// this loop is looking for tags, split characters, and XML escapes,
704		// which start with ampersands. All other characters are assumed to
705		// be word characters. The onSplit() method takes care of extracting
706		// word text and storing it in the terms array. The onStartBracket
707		// method parses tags. ignoreUntil is used to ignore comments and
708		// script data.
709	892	for (; position >= 0 && position < text.length(); position++) {
710	440	char c = text.charAt(position);
711
712	440	if (c == '<') {
713	24	if (ignoreUntil == null) {
714	24	onSplit();
715		}
716	24	onStartBracket();
717	416	} else if (ignoreUntil != null) {
718	0	continue;
719	416	} else if (c == '&') {
720	0	onAmpersand();
721	416	} else if (c < 256 && splits[c]) {
722	68	onSplit();
723		}
724		}
725	0	} catch (Exception e) {
726	0	Logger.getLogger(getClass().toString()).log(Level.WARNING,
727		"Parse failure: " + document.identifier);
728	12	}
729
730	12	if (ignoreUntil == null) {
731	12	onSplit();
732		}
733	12	document.terms = new ArrayList<String>(this.tokens);
734	12	document.tags = coalesceTags();
735	12	pooler.transform(document);
736	12	}
737
738		/**
739		* Parses the text in the input string and returns a document object.
740		* This method calls the {#link tokenize(Document) other variant}.
741		*
742		* @return A new document object containing the parsed text from the input string.
743		*/
744		public Document tokenize(String text) throws IOException {
745	0	Document document = new Document();
746	0	document.text = text;
747	0	tokenize(document);
748
749	0	return document;
750		}
751
752		public ArrayList<Pair> getTokenPositions() {
753	8	return this.tokenPositions;
754		}
755
756		public void setProcessor(final Step processor) throws IncompatibleProcessorException {
757	4	Linkage.link(this, processor);
758	4	}
759
760		public void close() throws IOException {
761	0	processor.close();
762	0	}
763
764		public Class<Document> getInputClass() {
765	0	return Document.class;
766		}
767
768		public Class<Document> getOutputClass() {
769	0	return Document.class;
770		}
771		}