Java實(shí)現(xiàn)一個(gè)簡(jiǎn)單的爬蟲(chóng)程序。

package SearchCrawler;import java.util.*;import java.net.*;import java.io.*;import java.util.regex.*;/* *搜索Web爬行者(起始的URL、處理url的最大數(shù)、要搜索的字符串) */public class SearchCrawler implements Runnable{/* disallowListCache緩存robot不允許搜索的URL。 Robot協(xié)議在Web站點(diǎn)的根目錄下設(shè)置一個(gè)robots.txt文件, *規(guī)定站點(diǎn)上的哪些頁(yè)面是限制搜索的。 搜索程序應(yīng)該在搜索過(guò)程中跳過(guò)這些區(qū)域,下面是robots.txt的一個(gè)例子: # robots.txt for http://somehost.com/ User-agent: * Disallow: /cgi-bin/ Disallow: /registration # /Disallow robots on registration page Disallow: /login */private HashMap> disallowListCache = new HashMap>();ArrayList errorList = new ArrayList();// 錯(cuò)誤信息ArrayList result = new ArrayList(); // 搜索到的結(jié)果String startUrl;// 開(kāi)始搜索的起點(diǎn)int maxUrl;// 最大處理的url數(shù)String searchString;// 要搜索的字符串(英文)boolean caseSensitive = false;// 是否區(qū)分大小寫(xiě)boolean limitHost = false;// 是否在限制的主機(jī)內(nèi)搜索public SearchCrawler(String startUrl, int maxUrl, String searchString){this.startUrl = startUrl;this.maxUrl = maxUrl;this.searchString = searchString;}public ArrayList getResult(){return result;}public void run(){// 啟動(dòng)搜索線(xiàn)程crawl(startUrl, maxUrl, searchString, limitHost, caseSensitive);}// 檢測(cè)URL格式private URL verifyUrl(String url){// 只處理HTTP URLs.if (!url.toLowerCase().startsWith("http://"))return null;URL verifiedUrl = null;try{verifiedUrl = new URL(url);} catch (Exception e){return null;}return verifiedUrl;}// 檢測(cè)robot是否允許訪(fǎng)問(wèn)給出的URL.private boolean isRobotAllowed(URL urlToCheck){String host = urlToCheck.getHost().toLowerCase();// 獲取給出RUL的主機(jī)// System.out.println("主機(jī)="+host);// 獲取主機(jī)不允許搜索的URL緩存ArrayList disallowList = disallowListCache.get(host);// 如果還沒(méi)有緩存,下載并緩存。if (disallowList == null){disallowList = new ArrayList();try{URL robotsFileUrl = new URL("http://" + host + "/robots.txt");BufferedReader reader = new BufferedReader(new InputStreamReader(robotsFileUrl.openStream()));// 讀robot文件，創(chuàng)建不允許訪(fǎng)問(wèn)的路徑列表。String line;while ((line = reader.readLine()) != null){if (line.indexOf("Disallow:") == 0){// 是否包含"Disallow:"String disallowPath = line.substring("Disallow:".length());// 獲取不允許訪(fǎng)問(wèn)路徑// 檢查是否有注釋。int commentIndex = disallowPath.indexOf("#");if (commentIndex != -1){disallowPath = disallowPath.substring(0,commentIndex);// 去掉注釋}disallowPath = disallowPath.trim();disallowList.add(disallowPath);}}// 緩存此主機(jī)不允許訪(fǎng)問(wèn)的路徑。disallowListCache.put(host, disallowList);} catch (Exception e){return true; // web站點(diǎn)根目錄下沒(méi)有robots.txt文件,返回真}}String file = urlToCheck.getFile();// System.out.println("文件getFile()="+file);for (int i = 0; i < disallowList.size(); i++){String disallow = disallowList.get(i);if (file.startsWith(disallow)){return false;}}return true;}private String downloadPage(URL pageUrl){try{// Open connection to URL for reading.BufferedReader reader = new BufferedReader(new InputStreamReader(pageUrl.openStream()));// Read page into buffer.String line;StringBuffer pageBuffer = new StringBuffer();while ((line = reader.readLine()) != null){pageBuffer.append(line);}return pageBuffer.toString();} catch (Exception e){}return null;}// 從URL中去掉"www"private String removeWwwFromUrl(String url){int index = url.indexOf("://www.");if (index != -1){return url.substring(0, index + 3) + url.substring(index + 7);}return (url);}// 解析頁(yè)面并找出鏈接private ArrayList retrieveLinks(URL pageUrl, String pageContents,HashSet crawledList, boolean limitHost){// 用正則表達(dá)式編譯鏈接的匹配模式。Pattern p = Pattern.compile("]",Pattern.CASE_INSENSITIVE);Matcher m = p.matcher(pageContents);ArrayList linkList = new ArrayList();while (m.find()){String link = m.group(1).trim();if (link.length() < 1){continue;}// 跳過(guò)鏈到本頁(yè)面內(nèi)鏈接。if (link.charAt(0) == '#'){continue;}if (link.indexOf("mailto:") != -1){continue;}if (link.toLowerCase().indexOf("javascript") != -1){continue;}if (link.indexOf("://") == -1){if (link.charAt(0) == '/'){// 處理絕對(duì)地link = "http://" + pageUrl.getHost() + ":"+ pageUrl.getPort() + link;} else{String file = pageUrl.getFile();if (file.indexOf('/') == -1){// 處理相對(duì)地址link = "http://" + pageUrl.getHost() + ":"+ pageUrl.getPort() + "/" + link;} else{String path = file.substring(0,file.lastIndexOf('/') + 1);link = "http://" + pageUrl.getHost() + ":"+ pageUrl.getPort() + path + link;}}}int index = link.indexOf('#');if (index != -1){link = link.substring(0, index);}link = removeWwwFromUrl(link);URL verifiedLink = verifyUrl(link);if (verifiedLink == null){continue;}/* 如果限定主機(jī)，排除那些不合條件的URL*/if (limitHost&& !pageUrl.getHost().toLowerCase().equals(verifiedLink.getHost().toLowerCase())){continue;}// 跳過(guò)那些已經(jīng)處理的鏈接.if (crawledList.contains(link)){continue;}linkList.add(link);}return (linkList);}// 搜索下載Web頁(yè)面的內(nèi)容，判斷在該頁(yè)面內(nèi)有沒(méi)有指定的搜索字符串private boolean searchStringMatches(String pageContents,String searchString, boolean caseSensitive){String searchContents = pageContents;if (!caseSensitive){// 如果不區(qū)分大小寫(xiě)searchContents = pageContents.toLowerCase();}Pattern p = Pattern.compile("[\\s]+");String[] terms = p.split(searchString);for (int i = 0; i < terms.length; i++){if (caseSensitive){if (searchContents.indexOf(terms[i]) == -1){return false;}} else{if (searchContents.indexOf(terms[i].toLowerCase()) == -1){return false;}}}return true;}// 執(zhí)行實(shí)際的搜索操作public ArrayList crawl(String startUrl, int maxUrls,String searchString, boolean limithost, boolean caseSensitive){System.out.println("searchString=" + searchString);HashSet crawledList = new HashSet();LinkedHashSet toCrawlList = new LinkedHashSet();if (maxUrls < 1){errorList.add("Invalid Max URLs value.");System.out.println("Invalid Max URLs value.");}if (searchString.length() < 1){errorList.add("Missing Search String.");System.out.println("Missing search String");}if (errorList.size() > 0){System.out.println("err!!!");return errorList;}// 從開(kāi)始URL中移出wwwstartUrl = removeWwwFromUrl(startUrl);toCrawlList.add(startUrl);while (toCrawlList.size() > 0){if (maxUrls != -1){if (crawledList.size() == maxUrls){break;}}// Get URL at bottom of the list.String url = toCrawlList.iterator().next();// Remove URL from the to crawl list.toCrawlList.remove(url);// Convert string url to URL object.URL verifiedUrl = verifyUrl(url);// Skip URL if robots are not allowed to access it.if (!isRobotAllowed(verifiedUrl)){continue;}// 增加已處理的URL到crawledListcrawledList.add(url);String pageContents = downloadPage(verifiedUrl);if (pageContents != null && pageContents.length() > 0){// 從頁(yè)面中獲取有效的鏈接ArrayList links = retrieveLinks(verifiedUrl,pageContents, crawledList, limitHost);toCrawlList.addAll(links);if (searchStringMatches(pageContents, searchString,caseSensitive)){result.add(url);System.out.println(url);}}}return result;}// 主函數(shù)public static void main(String[] args){if (args.length != 3){System.out.println("Usage:java SearchCrawler startUrl maxUrl searchString");return;}int max = Integer.parseInt(args[1]);SearchCrawler crawler = new SearchCrawler(args[0], max, args[2]);Thread search = new Thread(crawler);System.out.println("Start searching...");System.out.println("result:");search.start();}}

本文轉(zhuǎn)載自：CSDN博客

歡迎加入我愛(ài)機(jī)器學(xué)習(xí)QQ14群：336582044

微信掃一掃，關(guān)注我愛(ài)機(jī)器學(xué)習(xí)公眾號(hào)

總結(jié)

以上是生活随笔為你收集整理的java写一个搜索引擎_搜索引擎—-Java实现一个简单的网络爬虫的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問(wèn)題。

如果覺(jué)得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。