使用asp.net或者asp檢查某個(gè)url地址,某篇文章是否被搜索引擎,如百度,谷歌,搜狗收錄。
實(shí)現(xiàn)原理:直接搜索你那篇文章的url地址(不帶協(xié)議,但上協(xié)議也行,代碼會(huì)自動(dòng)去掉協(xié)議內(nèi)容),如果被索引會(huì)返回搜索結(jié)果,否則會(huì)提示找不到信息。
Asp.Net檢查百度,谷歌,搜狗搜索引擎是否收錄文章網(wǎng)址源代碼:
using System;
using System.Net;
using System.Text;
using System.IO;
using System.Web;
public class SearchEngineIndex
{
public static string[] urls = { //搜索引擎檢查地址
"http://www.baidu.com/s?ie=utf-8wd=",//百度索引url檢查地址
"https://www.google.com.hk/search?q=",//谷歌索引url檢查地址
"http://www.sogou.com/web?ie=utf8query="http://搜狗索引url檢查地址
}
, noFindKeyword = { "抱歉,沒有找到與", "找不到和您的查詢", "未收錄?" };//搜索引擎未索引url地址時(shí)的關(guān)鍵字
/// summary>
/// 獲取響應(yīng)的編碼
/// /summary>
/// param name="contenttype">/param>
/// returns>/returns>
private static Encoding GetEncoding(string contenttype)
{
if (!string.IsNullOrEmpty(contenttype))
{
contenttype = contenttype.ToLower();
if (contenttype.IndexOf("gb2312") != -1 || contenttype.IndexOf("gbk") != -1) return Encoding.GetEncoding(936);
if (contenttype.IndexOf("big5") != -1) return Encoding.GetEncoding(950);
}
return Encoding.UTF8;
}
/// summary>
/// 使用HttpWebRequest對(duì)象,自動(dòng)識(shí)別字符集
/// /summary>
/// param name="url">/param>
/// param name="addUseragent">是否添加UserAgent,采集其他網(wǎng)站時(shí)防止被攔截/param>
/// returns>/returns>
public static string GetHtml(string url, bool addUseragent)
{
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
if (addUseragent) request.UserAgent = "Googlebot|Feedfetcher-Google|Baiduspider";
string html = null;
try
{
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
StreamReader srd = new StreamReader(response.GetResponseStream(), GetEncoding(response.ContentType));
html = srd.ReadToEnd();
srd.Close();
response.Close();
}
catch { }
return html;
}
/// summary>
/// 檢查某個(gè)url是否被搜索引擎索引
/// /summary>
/// param name="url">url地址/param>
/// param name="engin">0:百度 1:谷歌 2:搜狗,其他搜索引擎如bing和360直接查網(wǎng)址顯示的結(jié)果不是直接得到網(wǎng)址的,有些出入,不做檢查/param>
/// returns>/returns>
public static bool CheckIndex(string url, int engin)
{
if (string.IsNullOrEmpty(url)) return false;
if (engin 0 || engin > 2) engin = 0;
url = urls[engin] + HttpUtility.UrlEncode(url.ToLower().Replace("http://", "").Replace("https://", ""));
bool r = true;
string html = GetHtml(url, true);
if (html == null || html.IndexOf(noFindKeyword[engin]) != -1) r = false;
return r;
}
}
//調(diào)用方法示例
SearchEngineIndex.CheckIndex("www.jb51.net/article/20101014/2902.aspx", 0);//檢查百度索引
SearchEngineIndex.CheckIndex("www.jb51.net/article/20101014/2902.aspx", 1);//檢查谷歌索引
SearchEngineIndex.CheckIndex("www.jb51.net/article/20101014/2902.aspx", 2);//檢查搜狗索引
Asp檢查百度,谷歌,搜狗搜索引擎是否收錄文章網(wǎng)址源代碼:
%
class SearchEnginIndex
dim urls,noFindKeyword
private sub Class_Initialize
'百度,谷歌,搜狗url地址索引查詢地址
urls=array("http://www.baidu.com/s?ie=utf-8wd=","https://www.google.com.hk/search?q=","http://www.sogou.com/web?ie=utf8query=")
'搜索引擎未索引url地址時(shí)的關(guān)鍵字
NoFindKeyword=array("抱歉,沒有找到與", "找不到和您的查詢", "未收錄?")
End sub
private function GetEncoding(contenttype)
contenttype=lcase(contenttype)
if instr(contenttype,"gb2312")>0 and instr(contenttype,"gbk")>0 then
GetEncoding="gb2312"
elseif instr(contenttype,"big5")>0 then
GetEncoding="big5"
else
GetEncoding="utf-8"
end if
end function
private function BinToString(bin,encoding)'將2進(jìn)制流數(shù)據(jù)依據(jù)編碼轉(zhuǎn)為對(duì)應(yīng)的字符串內(nèi)容
dim obj
set obj=Server.CreateObject("Adodb.Stream")
obj.Type=1:obj.Mode=3:obj.Open
obj.Write bin
obj.Position=0:obj.Type=2:obj.Charset=encoding
BinToString=obj.ReadText
obj.Close:set obj=nothing
end function
public function GetHtml(url)
dim xhr
set xhr=server.CreateObject("microsoft.xmlhttp")
xhr.open "get",url,false
xhr.send
encoding=GetEncoding(xhr.getResponseHeader("content-type"))
response.CharSet=encoding
GetHtml=BinToString(xhr.responsebody,encoding)
set xhr=nothing
end function
public function CheckIndex(url,engin)
if len(url)=0 then exit function
if engin0 or engin>2 then engin=1
url=urls(engin)server.URLEncode(url)
dim html
html=GetHtml(url)
CheckIndex=instr(html,NoFindKeyword(engin))=0
End function
end Class
set sei=new SearchEnginIndex
response.Write sei.CheckIndex("www.jb51.net/article/20101014/2902.aspx",0)'百度索引
response.Write sei.CheckIndex("www.jb51.net/article/20101014/2902.aspx",1)'谷歌索引
response.Write sei.CheckIndex("www.jb51.net/article/20101014/2902.aspx",2)'搜狗索引
set sei=nothing
%>
您可能感興趣的文章:- 基于Lucene的Java搜索服務(wù)器Elasticsearch安裝使用教程
- 安裝ElasticSearch搜索工具并配置Python驅(qū)動(dòng)的方法
- 使用Python操作Elasticsearch數(shù)據(jù)索引的教程
- ASP.NET 調(diào)用百度搜索引擎的代碼
- asp.net(c#)捕捉搜索引擎蜘蛛和機(jī)器人
- ASP.NET通過(guò)分布式Session提升性能
- Elasticsearch.Net使用入門教程(1)