應用HtmlParser處理含HTML標簽的字符串或網頁

String Extraction

To get all the text content from a web page, use the TextExtractingVisitor, like so:

If you want a more browser like behaviour, use the StringBean like so:

To get all the text content from a web page you already have in a string:

最近在研究lucene，主要做ftp搜索和網頁的站內搜索。
ftp搜索比較好做，主流的FTP有兩種一種是IIS的一種是Server-U的.
真對這兩種FTP分別進行分析就可以得到FTP資源的文件名和路徑及大小和日期
然后對其進行索引就可以了，比較簡單。
網頁檢索可不像ftp那樣了，我試著用lucene自帶的htmlparser，解析純英文的網頁
沒有問題，可解析中文的網頁時有時會遇到編碼問題。郁悶。。。
SourceForge搜到了一個開源的HTMLParser。網址是http://htmlparser.sourceforge.net
目前的版本為1.6。

測試代碼為：

import java.io.*;
import org.htmlparser.filters.*;
import org.htmlparser.*;
import org.htmlparser.nodes.*;
import org.htmlparser.tags.*;
import org.htmlparser.util.*;
import org.htmlparser.visitors.*;

public class HTMLParserTest
{
?public static void main(String args[]) throws Exception
?{
? String path = "D://Webdup//MyWebsites//biti//download//latest//cisco.biti.edu.cn//index.html";
? StringBuffer sbStr = new StringBuffer();
? BufferedReader reader? = new BufferedReader(new FileReader(new File(path)));
? String temp = "";
? while((temp=reader.readLine())!=null)
? {
?? sbStr.append(temp);
?? sbStr.append("/r/n");
? }
? reader.close();
?
?
? String result = sbStr.toString();
?? readAll(result);
??????? readTextAndLink(result);
??????? readByHtml(result);
???? readTextAndTitle(result);
?}
?

?//按頁面方式處理.解析標準的html頁面
?public static void readByHtml(String content) throws Exception
??? {
??????? Parser myParser;
??????? myParser = Parser.createParser(content, "GB2312");

??????? HtmlPage visitor = new HtmlPage(myParser);

??????? myParser.visitAllNodesWith(visitor);

??????? String textInPage = visitor.getTitle();
??????? System.out.println(textInPage);
??????? NodeList nodelist ;
??????? nodelist = visitor.getBody();
??????? System.out.print(nodelist.asString().trim());
?
?
??? }
???
??? //讀取文本內容和標題
?public static void readTextAndTitle(String result) throws Exception
?{
? Parser parser ;
? NodeList nodelist ;
? parser = Parser.createParser(result,"GB2312");
? NodeFilter textFilter = new NodeClassFilter(TextNode.class);
? NodeFilter titleFilter = new NodeClassFilter(TitleTag.class);
? OrFilter lastFilter = new OrFilter();
? lastFilter.setPredicates(new NodeFilter[]{textFilter,titleFilter});
? nodelist = parser.parse(lastFilter);
? Node[] nodes = nodelist.toNodeArray();
? String line ="";
? for(int i=0;i<nodes.length;i++)
? {
?? Node node = nodes[i];
?? if(node instanceof TextNode)
?? {
??? TextNode textnode = (TextNode) node;
??? line = textnode.getText();
?? }
?? else
?? if(node instanceof TitleTag)
?? {
??? TitleTag titlenode = (TitleTag) node;
??? line = titlenode.getTitle();
?? }
?? if (isTrimEmpty(line))
??????????????? continue;
??????????? System.out.println(line);
? }
?}
?
?//分別讀純文本和鏈接
?
?public static void readTextAndLink(String result) throws Exception
?{
? Parser parser;
? NodeList nodelist;
? parser = Parser.createParser(result,"GB2312");
? NodeFilter textFilter = new NodeClassFilter(TextNode.class);
??NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);
? OrFilter lastFilter = new OrFilter();
? lastFilter.setPredicates(new NodeFilter[] { textFilter, linkFilter });
? nodelist = parser.parse(lastFilter);
? Node[] nodes = nodelist.toNodeArray();
? String line ="";
? for(int i=0;i<nodes.length;i++)
? {
?? Node node = nodes[i];
?? if(node instanceof TextNode)
?? {
??? TextNode textnode = (TextNode) node;
??? line = textnode.getText();
?? }
?? else
?? if(node instanceof LinkTag)
?? {
??? LinkTag link = (LinkTag)node;
??? line = link.getLink();
?? }
?? if (isTrimEmpty(line))
??????????????? continue;
??????????? System.out.println(line);
? }
?}
?
?
?
?
?
?public static void readAll(String result) throws Exception
?{
? Parser parser;
? Node[] nodes ;
? parser = Parser.createParser(result,"GB2312");
? nodes = parser.extractAllNodesThatAre(TextNode.class);
?
? //讀取所有的內容節點
? for (int i = 0; i < nodes.length; i++)
??????? {
??????????? TextNode textnode = (TextNode) nodes[i];
??????????? String line = textnode.toPlainTextString().trim();
??????????? if (line.equals(""))
??????????????? continue;
??????????? System.out.println(line);
??????? }
?}
?
?
?/**
???? * 去掉左右空格后字符串是否為空
???? */
??? public static boolean isTrimEmpty(String astr)
??? {
??????? if ((null == astr) || (astr.length() == 0))
??????? {
??????????? return true;
??????? }
??????? if (isBlank(astr.trim()))
??????? {
??????????? return true;
??????? }
??????? return false;
??? }

??? /**
???? * 字符串是否為空:null或者長度為0.
???? */
??? public static boolean isBlank(String astr)
??? {
??????? if ((null == astr) || (astr.length() == 0))
??????? {
??????????? return true;
??????? }
??????? else
??????? {
??????????? return false;
??????? }
??? }

?
}

?