比較常用的幾種英文分析器，他們之間的區(qū)別見程序中的注釋。
SimpleAnalyzer
StandardAnalyzer
WhitespaceAnalyzer
StopAnalyzer
package analyzer;
import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
public class TestAnalyzer 
{
private static String testString1 = "The quick brown fox jumped over the lazy dogs";
private static String testString2 = "xy&z mail is - xyz@sohu.com";
public static void testWhitespace(String testString) throws Exception{
Analyzer analyzer = new WhitespaceAnalyzer();
Reader r = new StringReader(testString);
Tokenizer ts = (Tokenizer) analyzer.tokenStream("", r);
System.err.println("=====Whitespace analyzer====");
System.err.println("分析方法：空格分割");
Token t;
while ((t = ts.next()) != null) {
System.out.println(t.termText());
}
}
public static void testSimple(String testString) throws Exception{
Analyzer analyzer = new SimpleAnalyzer();
Reader r = new StringReader(testString);
Tokenizer ts = (Tokenizer) analyzer.tokenStream("", r);
System.err.println("=====Simple analyzer====");
System.err.println("分析方法：空格及各種符號分割");
Token t;
while ((t = ts.next()) != null) {
System.out.println(t.termText());
}
}
public static void testStop(String testString) throws Exception{
Analyzer analyzer = new StopAnalyzer();
Reader r = new StringReader(testString);
StopFilter sf = (StopFilter) analyzer.tokenStream("", r);
System.err.println("=====stop analyzer====");
System.err.println("分析方法：空格及各種符號分割,去掉停止詞，停止詞包括 is,are,in,on,the等無實(shí)際意義的詞");
//停止詞
Token t;
while ((t = sf.next()) != null) {
System.out.println(t.termText());
}
}
public static void testStandard(String testString) throws Exception{
Analyzer analyzer = new StandardAnalyzer();
Reader r = new StringReader(testString);
StopFilter sf = (StopFilter) analyzer.tokenStream("", r);
System.err.println("=====standard analyzer====");
System.err.println("分析方法：混合分割,包括了去掉停止詞，支持漢語");
Token t;
while ((t = sf.next()) != null) {
System.out.println(t.termText());
}
}
public static void main(String[] args) throws Exception{
// String testString = testString1;
String testString = testString2;
System.out.println(testString);
testWhitespace(testString);
testSimple(testString);
testStop(testString);
testStandard(testString);
}
}