學習java的正則表達式,抓取網(wǎng)頁并解析HTML部分內(nèi)容
package com.xiaofeng.picup; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; /** *//** * * @抓取頁面文章標題及內(nèi)容(測試) 手動輸入網(wǎng)址抓取,可進一步自動抓取整個頁面的全部內(nèi)容 * */ public class WebContent ...{ /** *//** * 讀取一個網(wǎng)頁全部內(nèi)容 */ public String getOneHtml(String htmlurl) throws IOException...{ URL url; String temp; StringBuffer sb = new StringBuffer(); try ...{ url = new URL(htmlurl); BufferedReader in = new BufferedReader(new InputStreamReader(url .openStream(), "utf-8"));// 讀取網(wǎng)頁全部內(nèi)容 while ((temp = in.readLine()) != null) ...{ sb.append(temp); } in.close(); }catch(MalformedURLException me)...{ System.out.println("你輸入的URL格式有問題!請仔細輸入"); me.getMessage(); throw me; }catch (IOException e) ...{ e.printStackTrace(); throw e; } return sb.toString(); } /** *//** * * @param s * @return 獲得網(wǎng)頁標題 */ public String getTitle(String s) ...{ String regex; String title = ""; ListString> list = new ArrayListString>(); regex = "title>.*?/title>"; Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ); Matcher ma = pa.matcher(s); while (ma.find()) ...{ list.add(ma.group()); } for (int i = 0; i list.size(); i++) ...{ title = title + list.get(i); } return outTag(title); } /** *//** * * @param s * @return 獲得鏈接 */ public ListString> getLink(String s) ...{ String regex; ListString> list = new ArrayListString>(); regex = "a[^>]*href=("([^"]*)"|'([^']*)'|([^s>]*))[^>]*>(.*?)/a>"; Pattern pa = Pattern.compile(regex, Pattern.DOTALL); Matcher ma = pa.matcher(s); while (ma.find()) ...{ list.add(ma.group()); } return list; } /** *//** * * @param s * @return 獲得腳本代碼 */ public ListString> getScript(String s) ...{ String regex; ListString> list = new ArrayListString>(); regex = "script.*?/script>"; Pattern pa = Pattern.compile(regex, Pattern.DOTALL); Matcher ma = pa.matcher(s); while (ma.find()) ...{ list.add(ma.group()); } return list; } /** *//** * * @param s * @return 獲得CSS */ public ListString> getCSS(String s) ...{ String regex; ListString> list = new ArrayListString>(); regex = "style.*?/style>"; Pattern pa = Pattern.compile(regex, Pattern.DOTALL); Matcher ma = pa.matcher(s); while (ma.find()) ...{ list.add(ma.group()); } return list; } /** *//** * * @param s * @return 去掉標記 */ public String outTag(String s) ...{ return s.replaceAll(".*?>", ""); }
上一篇:MongoDB正則表達式及應用