umtheme 通过对蜘蛛标识符（如Baiduspider、Googlebot）的识别就可以知道是搜索蜘蛛还是普通用户来过我们的网站，以下是PHP判断来访是搜索引擎蜘蛛还是普通用户的几种方法。1、摘自Discuz x3.2<?phpfunction checkrobot($useragent=''){ static $kw_spiders =...
通过对蜘蛛标识符（如Baiduspider、Googlebot）的识别就可以知道是搜索蜘蛛还是普通用户来过我们的网站，以下是PHP判断来访是搜索引擎蜘蛛还是普通用户的几种方法。
1、摘自Discuz x3.2
<?phpfunction checkrobot($useragent=''){
static $kw_spiders = array('bot', 'crawl', 'spider' ,'slurp', 'sohu-search', 'lycos', 'robozilla');
static $kw_browsers = array('msie', 'netscape', 'opera', 'konqueror', 'mozilla');
$useragent = strtolower(empty($useragent) ? $_SERVER['HTTP_USER_AGENT'] : $useragent);
if(strpos($useragent, 'http://') === false && dstrpos($useragent, $kw_browsers)) return false;
if(dstrpos($useragent, $kw_spiders)) return true;
return false;}function dstrpos($string, $arr, $returnvalue = false) {
if(empty($string)) return false;
foreach((array)$arr as $v) {
if(strpos($string, $v) !== false) {
$return = $returnvalue ? $v : true;
return $return;
}
}
return false;}if(checkrobot()){
echo '机器人爬虫';
}else{
echo '普通用户';
}
?>实际应用，可以判断非搜索引擎（蜘蛛）：
<?php
if(!checkrobot()){
//do something
}
?>第二种方法，使用PHP实现蜘蛛访问日志统计
<?php
$useragent = addslashes(strtolower($_SERVER['HTTP_USER_AGENT']));
if (strpos($useragent, 'googlebot')!== false){
$bot = 'Google';
}elseif (strpos($useragent,'mediapartners-google') !== false){
$bot = 'Google Adsense';
}elseif (strpos($useragent,'baiduspider') !== false){
$bot = 'Baidu';
}elseif (strpos($useragent,'sogou spider') !== false){
$bot = 'Sogou';
}elseif (strpos($useragent,'sogou web') !== false){
$bot = 'Sogou web';
}elseif (strpos($useragent,'sosospider') !== false){
$bot = 'SOSO';
}elseif (strpos($useragent,'360spider') !== false){
$bot = '360Spider';
}elseif (strpos($useragent,'yahoo') !== false){
$bot = 'Yahoo';
}elseif (strpos($useragent,'msn') !== false){
$bot = 'MSN';
}elseif (strpos($useragent,'msnbot') !== false){
$bot = 'msnbot';
}elseif (strpos($useragent,'sohu') !== false){
$bot = 'Sohu';
}elseif (strpos($useragent,'yodaoBot') !== false){
$bot = 'Yodao';
}elseif (strpos($useragent,'twiceler') !== false){
$bot = 'Twiceler';
}elseif (strpos($useragent,'ia_archiver') !== false){
$bot = 'Alexa_';
}elseif (strpos($useragent,'iaarchiver') !== false){
$bot = 'Alexa';
}elseif (strpos($useragent,'slurp') !== false){
$bot = '雅虎';
}elseif (strpos($useragent,'bot') !== false){
$bot = '其它蜘蛛';
}
if(isset($bot)){
$fp = @fopen('bot.txt','a');
fwrite($fp,date('Y-m-d H:i:s')."\t".$_SERVER["REMOTE_ADDR"]."\t".$bot."\t".'http://'.$_SERVER['SERVER_NAME'].$_SERVER["REQUEST_URI"]."\r\n");
fclose($fp);
}
?>第三种方法：
搜索引擎的蜘蛛都有自己的独特标志，通过HTTP_USER_AGENT来判断是否是蜘蛛。
function isCrawler() {
echo $agent= strtolower($_SERVER['HTTP_USER_AGENT']);
if (!empty($agent)) {
$spiderSite= array(
"TencentTraveler",
"Baiduspider+",
"BaiduGame",
"Googlebot",
"msnbot",
"Sosospider+",
"Sogou web spider",
"ia_archiver",
"Yahoo! Slurp",
"YoudaoBot",
"Yahoo Slurp",
"MSNBot",
"Java (Often spam bot)",
"BaiDuSpider",
"Voila",
"Yandex bot",
"BSpider",
"twiceler",
"Sogou Spider",
"Speedy Spider",
"Google AdSense",
"Heritrix",
"Python-urllib",
"Alexa (IA Archiver)",
"Ask",
"Exabot",
"Custo",
"OutfoxBot/YodaoBot",
"yacy",
"SurveyBot",
"legs",
"lwp-trivial",
"Nutch",
"StackRambler",
"The web archive (IA Archiver)",
"Perl tool",
"MJ12bot",
"Netcraft",
"MSIECrawler",
"WGet tools",
"larbin",
"Fish search",
//其它蜘蛛,
);
foreach($spiderSite as $val) {
$str = strtolower($val);
if (strpos($agent, $str) !== false) {
return true;
}
}
}else{
return false;
}
}
if (isCrawler()){
echo "你好蜘蛛精！";
}else{
echo "你好普通用户！";
}
?>比较常见的蜘蛛标识，如果有错误或者没有收集到的，可以留言。
百度蜘蛛：Baiduspider
百度图片：Baiduspider-image
百度WAP：Baiduspider-mobile
百度视频：Baiduspider-video
百度新闻：Baiduspider-news
谷歌蜘蛛：Googlebot
360蜘蛛：360Spider
SOSO蜘蛛：Sosospider
雅虎蜘蛛：Yahoo
有道蜘蛛：YoudaoBot，YodaoBot
搜狗蜘蛛：Sogou News Spider，Sogou web spider、Sogou inst spider、Sogou blog、Sogou Orion spider
必应蜘蛛：bingbot
MSN蜘蛛：msnbot，msnbot-media
一搜蜘蛛：YisouSpider
Alexa蜘蛛：ia_archiver
宜搜蜘蛛：EasouSpider
即刻蜘蛛：JikeSpider
一淘网蜘蛛：EtaoSpider
