spiderList/spiderList.xml

   1 <?xml version="1.0" encoding="UTF-8"?>
   2 <data xmlns="http://www.woltlab.com" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.woltlab.com http://www.woltlab.com/XSD/5.4/spiderList.xsd">
   3         <spider ident="ABCdatos">
   4                 <name>ABCdatos BotLink</name>
   5                 <url>http://www.robotstxt.org/wc/active/html/abcdatos.html</url>
   6         </spider>
   7         <spider ident="AbraveSpider">
   8                 <name>Abrave</name>
   9                 <url>http://robot.abrave.co.uk/</url>
  10         </spider>
  11         <spider ident="abot/">
  12                 <name>abot</name>
  13         </spider>
  14         <spider ident="Accelatech RSSCrawler">
  15                 <name>Accelatech</name>
  16         </spider>
  17         <spider ident="Accoona-AI-Agent">
  18                 <name>Accoona</name>
  19                 <url>http://accoona.com/about/about_accoona.jsp</url>
  20         </spider>
  21         <spider ident="aconon Index">
  22                 <name>aconon Index (raubfische.de)</name>
  23         </spider>
  24         <spider ident="AcoonBot">
  25                 <name>Acoon</name>
  26                 <url>http://www.acoon.de/robot.asp</url>
  27         </spider>
  28         <spider ident="Ahoy!">
  29                 <name>Ahoy!</name>
  30                 <url>http://www.robotstxt.org/wc/active/html/ahoythehomepagefinder.html</url>
  31         </spider>
  32         <spider ident="AhrefsBot">
  33                 <name>Ahrefs</name>
  34                 <url>http://ahrefs.com/robot/</url>
  35         </spider>
  36         <spider ident="AlkalineBOT">
  37                 <name>Alkaline</name>
  38                 <url>http://www.robotstxt.org/wc/active/html/Alkaline.html</url>
  39         </spider>
  40         <spider ident="AlexaBOT">
  41                 <name>Alexa</name>
  42                 <url>http://www.alexa.com/help/webmasters</url>
  43         </spider>
  44         <spider ident="http://www.almaden.ibm.com/cs/crawler">
  45                 <name>Almaden Crawler</name>
  46                 <url>http://www.almaden.ibm.com/cs/crawler/</url>
  47         </spider>
  48         <spider ident="EMC Spider">
  49                 <name>ananzi</name>
  50         </spider>
  51         <spider ident="Anthill">
  52                 <name>Anthill</name>
  53                 <url>http://www.robotstxt.org/wc/active/html/anthill.html</url>
  54         </spider>
  55         <spider ident="Aport">
  56                 <name>Aport</name>
  57                 <url>http://www.aport.ru/</url>
  58         </spider>
  59         <spider ident="AppleSyndication">
  60                 <name>Apple</name>
  61         </spider>
  62         <spider ident="Applebot">
  63                 <name>Applebot</name>
  64                 <url>https://support.apple.com/en-gb/HT204683</url>
  65         </spider>
  66         <spider ident="Arachnophilia">
  67                 <name>Arachnophilia</name>
  68                 <url>http://www.robotstxt.org/wc/active/html/arachnophilia.html</url>
  69         </spider>
  70         <spider ident="Araneo">
  71                 <name>Araneo</name>
  72                 <url>http://www.robotstxt.org/wc/active/html/araneo.html</url>
  73         </spider>
  74         <spider ident="ArchitextSpider">
  75                 <name>ArchitextSpider</name>
  76                 <url>http://www.robotstxt.org/wc/active/html/architext.html</url>
  77         </spider>
  78         <spider ident="archive.org_bot">
  79                 <name>Archive.org</name>
  80                 <url>https://archive.org/details/archive.org_bot</url>
  81         </spider>
  82         <spider ident="arks/1.0">
  83                 <name>arks</name>
  84                 <url>http://www.robotstxt.org/wc/active/html/arks.html</url>
  85         </spider>
  86         <spider ident="ASpider">
  87                 <name>ASpider</name>
  88                 <url>http://www.robotstxt.org/wc/active/html/aspider.html</url>
  89         </spider>
  90         <spider ident="ATN_Worldwide">
  91                 <name>ATN Worldwide</name>
  92                 <url>http://www.robotstxt.org/wc/active/html/atn.txt.html</url>
  93         </spider>
  94         <spider ident="Atomz">
  95                 <name>Atomz.com</name>
  96                 <url>http://www.robotstxt.org/wc/active/html/atomz.html</url>
  97         </spider>
  98         <spider ident="AURESYS">
  99                 <name>AURESYS</name>
 100                 <url>http://www.robotstxt.org/wc/active/html/auresys.html</url>
 101         </spider>
 102         <spider ident="BackRub">
 103                 <name>BackRub</name>
 104                 <url>http://www.robotstxt.org/wc/active/html/backrub.html</url>
 105         </spider>
 106         <spider ident="Baiduspider">
 107                 <name>Baiduspider</name>
 108                 <url>http://www.baidu.com/search/spider.htm</url>
 109         </spider>
 110         <spider ident="bbot">
 111                 <name>BBot</name>
 112                 <url>http://www.robotstxt.org/wc/active/html/bbot.html</url>
 113         </spider>
 114         <spider ident="BecomeBot">
 115                 <name>BecomeBot</name>
 116                 <url>http://www.become.com/webmasters.html</url>
 117         </spider>
 118         <spider ident="Big Brother">
 119                 <name>Big Brother</name>
 120                 <url>http://www.robotstxt.org/wc/active/html/bigbrother.html</url>
 121         </spider>
 122         <spider ident="BigmirSpider">
 123                 <name>Bigmir</name>
 124                 <url>http://www.bigmir.net/</url>
 125         </spider>
 126         <spider ident="bingbot">
 127                 <name>Bing</name>
 128                 <url>http://www.bing.com/bingbot.htm</url>
 129         </spider>
 130         <spider ident="Bitacle bot">
 131                 <name>Bitacle</name>
 132                 <url>http://bitacle.org/</url>
 133         </spider>
 134         <spider ident="Biz360 Spider">
 135                 <name>Biz</name>
 136                 <url>http://www.biz360.com</url>
 137         </spider>
 138         <spider ident="Bjaaland">
 139                 <name>Bjaaland</name>
 140                 <url>http://www.robotstxt.org/wc/active/html/bjaaland.html</url>
 141         </spider>
 142         <spider ident="BlackWidow">
 143                 <name>BlackWidow</name>
 144                 <url>http://www.robotstxt.org/wc/active/html/blackwidow.html</url>
 145         </spider>
 146         <spider ident="BLEXBot">
 147                 <name>BLEXBot</name>
 148                 <url>http://webmeup.com/crawler.html</url>
 149         </spider>
 150         <spider ident="BlogCrawler by Xango">
 151                 <name>BlogCrawler</name>
 152         </spider>
 153         <spider ident="blogdb">
 154                 <name>BlogDb</name>
 155                 <url>http://blogdb.jp</url>
 156         </spider>
 157         <spider ident="blog search engine by BlogFan.ORG">
 158                 <name>BlogFan</name>
 159                 <url>http://www.blogfan.org</url>
 160         </spider>
 161         <spider ident="Bloglines">
 162                 <name>Bloglies</name>
 163                 <url>http://www.bloglines.com</url>
 164         </spider>
 165         <spider ident="BlogPulse (ISSpider-3.0)">
 166                 <name>BlogPulse</name>
 167         </spider>
 168         <spider ident="BlogSearch">
 169                 <name>BlogSearch</name>
 170                 <url>http://www.icerocket.com</url>
 171         </spider>
 172         <spider ident="BlogsNowBot">
 173                 <name>BlogsNow</name>
 174                 <url>http://www.blogsnow.com/</url>
 175         </spider>
 176         <spider ident="BlogStreetBot">
 177                 <name>BlogStreetBot</name>
 178                 <url>http://www.blogstreet.com/</url>
 179         </spider>
 180         <spider ident="Bulkfeeds">
 181                 <name>BlogStreetBot</name>
 182                 <url>http://bulkfeeds.net</url>
 183         </spider>
 184         <spider ident="BoardPulse">
 185                 <name>BoardPulse</name>
 186                 <url>http://www.boardpulse.com/</url>
 187         </spider>
 188         <spider ident="BoardReader">
 189                 <name>BoardReader</name>
 190                 <url>http://www.boardreader.com/aboutus.asp</url>
 191         </spider>
 192         <spider ident="BoardViewer">
 193                 <name>BoardViewer</name>
 194                 <url>http://www.boardviewer.com/</url>
 195         </spider>
 196         <spider ident="boitho.com-robot">
 197                 <name>Boitho</name>
 198                 <url>http://www.boitho.com/bot.html</url>
 199         </spider>
 200         <spider ident="borg-bot">
 201                 <name>Borg-Bot</name>
 202                 <url>http://www.robotstxt.org/wc/active/html/borg-bot.html</url>
 203         </spider>
 204         <spider ident="BSpider">
 205                 <name>BSpider</name>
 206                 <url>http://www.robotstxt.org/wc/active/html/bspider.html</url>
 207         </spider>
 208         <spider ident="Buck/2.2">
 209                 <name>Buck</name>
 210                 <url>https://app.hypefactors.com/media-monitoring/about.html</url>
 211         </spider>
 212         <spider ident="CACTVS Chemistry Spider">
 213                 <name>CACTVS Chemistry</name>
 214         </spider>
 215         <spider ident="Calif">
 216                 <name>Calif</name>
 217         </spider>
 218         <spider ident="CaRP/3.6Evolution">
 219                 <name>CaRP</name>
 220                 <url>http://www.biz360.com</url>
 221         </spider>
 222         <spider ident="Checkbot">
 223                 <name>Checkbot</name>
 224         </spider>
 225         <spider ident="ChristCrawler.com">
 226                 <name>ChristCrawler.com</name>
 227         </spider>
 228         <spider ident="www.cienciaficcion.net">
 229                 <name>cIeNcIaFiCcIoN.nEt</name>
 230         </spider>
 231         <spider ident="CipinetBot">
 232                 <name>Cipinet</name>
 233                 <url>http://www.cipinet.com/bot.html</url>
 234         </spider>
 235         <spider ident="CJNetworkQuality">
 236                 <name>CJNetworkQuality</name>
 237                 <url>http://www.cj.com/networkquality/</url>
 238         </spider>
 239         <spider ident="CMC/0.01">
 240                 <name>CMC/0.01</name>
 241         </spider>
 242         <spider ident="ColdFusion">
 243                 <name>ColdFusion</name>
 244         </spider>
 245         <spider ident="combine">
 246                 <name>Combine System</name>
 247         </spider>
 248         <spider ident="Crawler (cometsearch@cometsystems.com)">
 249                 <name>cometsystems.com</name>
 250         </spider>
 251         <spider ident="ComputingSite Robi/1.0">
 252                 <name>ComputingSite Robi/1.0</name>
 253         </spider>
 254         <spider ident="conceptbot">
 255                 <name>Conceptbot</name>
 256         </spider>
 257         <spider ident="Cooby.de Crawler">
 258                 <name>Cooby.de Crawler</name>
 259         </spider>
 260         <spider ident="CoolBot">
 261                 <name>CoolBot</name>
 262         </spider>
 263         <spider ident="Cusco">
 264                 <name>Cusco</name>
 265         </spider>
 266         <spider ident="CyberSpyder">
 267                 <name>CyberSpyder</name>
 268         </spider>
 269         <spider ident="daypopbot">
 270                 <name>daypop</name>
 271         </spider>
 272         <spider ident="DesertRealm.com">
 273                 <name>Desert Realm</name>
 274         </spider>
 275         <spider ident="Deweb">
 276                 <name>DeWeb(c)</name>
 277         </spider>
 278         <spider ident="Die Blinde Kuh">
 279                 <name>Die Blinde Kuh</name>
 280                 <url>http://www.robotstxt.org/wc/active/html/blindekuh.html</url>
 281         </spider>
 282         <spider ident="dienstspider">
 283                 <name>DienstSpider</name>
 284         </spider>
 285         <spider ident="Digger/1.0 JDK/1.3.0">
 286                 <name>Digger</name>
 287         </spider>
 288         <spider ident="Digimarc WebReader">
 289                 <name>Digimarc MarcSpider</name>
 290         </spider>
 291         <spider ident="Digimarc CGIReader">
 292                 <name>Digimarc Marcspider/CGI</name>
 293         </spider>
 294         <spider ident="DIIbot">
 295                 <name>Digital Integrity Robot</name>
 296         </spider>
 297         <spider ident="grabber">
 298                 <name>Direct Hit Grabber</name>
 299         </spider>
 300         <spider ident="discobot">
 301                 <name>Discovery</name>
 302                 <url>http://discoveryengine.com/discobot.html</url>
 303         </spider>
 304         <spider ident="Discordbot">
 305                 <name>Discord</name>
 306                 <url>https://discordapp.com/</url>
 307         </spider>
 308         <spider ident="DNAbot/1.0">
 309                 <name>DNAbot</name>
 310         </spider>
 311         <spider ident="DotBot/1.1">
 312                 <name>Moz DotBot</name>
 313                 <url>http://www.opensiteexplorer.org/dotbot</url>
 314         </spider>
 315         <spider ident="DragonBot/1.0 libwww/5.0">
 316                 <name>DragonBot</name>
 317         </spider>
 318         <spider ident="DuckDuckBot">
 319                 <name>DuckDuckGo</name>
 320                 <url>https://duckduckgo.com/</url>
 321         </spider>
 322         <spider ident="DWCP/2.0">
 323                 <name>DWCP (Dridus' Web Cataloging Project)</name>
 324         </spider>
 325         <spider ident="e-SocietyRobot">
 326                 <name>e-Society</name>
 327                 <url>http://www.yama.info.waseda.ac.jp/~yamana/es/index_eng.htm</url>
 328         </spider>
 329         <spider ident="exactseek-pagereaper">
 330                 <name>eaxactseek-page</name>
 331         </spider>
 332         <spider ident="EbiNess/0.01a">
 333                 <name>EbiNess</name>
 334         </spider>
 335         <spider ident="edgeio-retriever">
 336                 <name>Edgeio</name>
 337                 <url>http://www.edgeio.com</url>
 338         </spider>
 339         <spider ident="EIT-Link-Verifier-Robot/0.2">
 340                 <name>EIT Link Verifier Robot</name>
 341         </spider>
 342         <spider ident="elfinbot">
 343                 <name>ELFINBOT</name>
 344         </spider>
 345         <spider ident="Emacs-w3/v[0-9\.]+">
 346                 <name>Emacs-w3 Search Engine</name>
 347         </spider>
 348         <spider ident="esther">
 349                 <name>Esther</name>
 350         </spider>
 351         <spider ident="EuripBot/">
 352                 <name>EuripBot</name>
 353         </spider>
 354         <spider ident="Evliya Celebi">
 355                 <name>Evliya Celebi</name>
 356         </spider>
 357         <spider ident="ExactSeek_Spider">
 358                 <name>ExactSeek_Spider</name>
 359                 <url>http://www.askjeevs.com</url>
 360         </spider>
 361         <spider ident="NG/2.0">
 362                 <name>ExaLead</name>
 363                 <url>http://botspotter.net/bs-389.html</url>
 364         </spider>
 365         <spider ident="ExaBot">
 366                 <name>ExaLead Beta</name>
 367                 <url>http://beta.exalead.com/search/C=0/2p=Help.7</url>
 368         </spider>
 369         <spider ident="ExaLead">
 370                 <name>ExaLead</name>
 371                 <url>http://www.exalead.com/search</url>
 372         </spider>
 373         <spider ident="facebookexternalhit">
 374                 <name>Facebook</name>
 375                 <url>http://www.facebook.com/externalhit_uatext.php</url>
 376         </spider>
 377         <spider ident="fast-webcrawler">
 378                 <name>FAST / AlltheWeb</name>
 379                 <url>http://help.yahoo.com/help/us/ysearch/slurp/index.html</url>
 380         </spider>
 381         <spider ident="FastCrawler">
 382                 <name>FastCrawler</name>
 383         </spider>
 384         <spider ident="Feed24.com">
 385                 <name>Feed24</name>
 386                 <url>http://www.feed24.com</url>
 387         </spider>
 388         <spider ident="FeedBlitz">
 389                 <name>FeedBlitz</name>
 390                 <url>http://www.feedblitz.com</url>
 391         </spider>
 392         <spider ident="FeedBurner">
 393                 <name>FeedBurner</name>
 394                 <url>http://www.FeedBurner.com</url>
 395         </spider>
 396         <spider ident="Feedly">
 397                 <name>Feedly</name>
 398                 <url>http://www.feedly.com/fetcher.html</url>
 399         </spider>
 400         <spider ident="Feedfetcher-Google">
 401                 <name>FeedFetcher-Google</name>
 402                 <url>http://www.google.com/feedfetcher.html</url>
 403         </spider>
 404         <spider ident="UniversalFeedParser">
 405                 <name>FeedParser</name>
 406                 <url>http://www.feedparser.org</url>
 407         </spider>
 408         <spider ident="Feedster Crawler">
 409                 <name>Feedster</name>
 410                 <url>http://www.feedstermedia.com/</url>
 411         </spider>
 412         <spider ident="FEHLSTART Superspider">
 413                 <name>FEHLSTART</name>
 414         </spider>
 415         <spider ident="FelixIDE">
 416                 <name>Felix IDE</name>
 417         </spider>
 418         <spider ident="ESIRover">
 419                 <name>FetchRover</name>
 420         </spider>
 421         <spider ident="fido">
 422                 <name>fido</name>
 423         </spider>
 424         <spider ident="findlinks">
 425                 <name>FindLinks</name>
 426                 <url>http://wortschatz.uni-leipzig.de/findlinks/</url>
 427         </spider>
 428         <spider ident="FindoryBot">
 429                 <name>Findroy</name>
 430                 <url>http://www.findory.com</url>
 431         </spider>
 432         <spider ident="Fish-Search-Robot">
 433                 <name>Fish search</name>
 434         </spider>
 435         <spider ident="Mozilla/4.0 (compatible: FDSE robot)">
 436                 <name>Fluid Dynamics</name>
 437         </spider>
 438         <spider ident="fouineur.9bit.qc.ca">
 439                 <name>Fouineur</name>
 440         </spider>
 441         <spider ident="Freecrawl">
 442                 <name>Freecrawl</name>
 443         </spider>
 444         <spider ident="FreshpingBot">
 445                 <name>Freshping</name>
 446                 <url>https://freshping.io/</url>
 447         </spider>
 448         <spider ident="FunnelWeb">
 449                 <name>FunnelWeb</name>
 450         </spider>
 451         <spider ident="GaisBot">
 452                 <name>Gais</name>
 453                 <url>http://gais.cs.ccu.edu.tw/robot.php</url>
 454         </spider>
 455         <spider ident="gamekitbot">
 456                 <name>GAMEKIT</name>
 457                 <url>http://www.uchoose.de/crawler/gamekitbot/</url>
 458         </spider>
 459         <spider ident="gammaSpider">
 460                 <name>gammaSpider</name>
 461         </spider>
 462         <spider ident="gazz">
 463                 <name>gazz</name>
 464         </spider>
 465         <spider ident="gcreep">
 466                 <name>GCreep</name>
 467         </spider>
 468         <spider ident="genieBot">
 469                 <name>genieBot</name>
 470                 <url>http://64.5.245.11/faq/faq.html</url>
 471         </spider>
 472         <spider ident="geourl">
 473                 <name>GeoURL</name>
 474                 <url>http://geourl.org/bot.html</url>
 475         </spider>
 476         <spider ident="GetterroboPlus">
 477                 <name>GetterroboPlus Puu</name>
 478         </spider>
 479         <spider ident="GetURL.rexx">
 480                 <name>GetURL</name>
 481         </spider>
 482         <spider ident="Gigabot">
 483                 <name>Gigabot</name>
 484                 <url>http://www.gigablast.com/spider.html</url>
 485         </spider>
 486         <spider ident="Girafabot">
 487                 <name>Girafabot</name>
 488                 <url>http://www.girafa.com/</url>
 489         </spider>
 490         <spider ident="Goku">
 491                 <name>Goku</name>
 492                 <url>http://goku.ru/bot.htm; bot@goku.ru</url>
 493         </spider>
 494         <spider ident="Golem">
 495                 <name>Golem</name>
 496         </spider>
 497         <spider ident="gonzo">
 498                 <name>Gonzo</name>
 499         </spider>
 500         <spider ident="Googlebot/">
 501                 <name>Google</name>
 502                 <url>http://www.google.com/bot.html</url>
 503         </spider>
 504         <spider ident="Mediapartners-Google">
 505                 <name>Google AdSense</name>
 506                 <url>https://www.google.com/adsense/faq</url>
 507         </spider>
 508         <spider ident="Googlebot-Image">
 509                 <name>Googlebot-Image</name>
 510                 <url>http://www.googlebot.com/bot.html</url>
 511         </spider>
 512         <spider ident="Googlebot-Mobile">
 513                 <name>Googlebot-Mobile</name>
 514                 <url>http://www.google.com/bot.html</url>
 515         </spider>
 516         <spider ident="Googlebot-Video/1.0">
 517                 <name>Googlebot-Video</name>
 518                 <url>https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers</url>
 519         </spider>
 520         <spider ident="GoogleStackdriverMonitoring-UptimeChecks">
 521                 <name>Google Stackdriver Monitoring</name>
 522                 <url>https://cloud.google.com/monitoring/alerts/uptime-checks</url>
 523         </spider>
 524         <spider ident="Google-Ads-Creatives-Assistant">
 525                 <name>Google-Ads-Creatives-Assistant</name>
 526         </spider>
 527         <spider ident="Google-AdWords-Express">
 528                 <name>Google-AdWords-Express</name>
 529         </spider>
 530         <spider ident="AdsBot-Google">
 531                 <name>Google Ads-Bot</name>
 532                 <url>http://www.google.com/adsbot.html</url>
 533         </spider>
 534         <spider ident="Gpostbot">
 535                 <name>Gpostbot</name>
 536                 <url>http://www.gpost.info/help.php?c=bot</url>
 537         </spider>
 538         <spider ident="griffon">
 539                 <name>Griffon</name>
 540         </spider>
 541         <spider ident="Gromit">
 542                 <name>Gromit</name>
 543         </spider>
 544         <spider ident="http://grub.org">
 545                 <name>Grub Client</name>
 546         </spider>
 547         <spider ident="Gulper Web Bot">
 548                 <name>Gulper Bot</name>
 549         </spider>
 550         <spider ident="havIndex">
 551                 <name>havIndex</name>
 552         </spider>
 553         <spider ident="HeinrichderMiragoRobot">
 554                 <name>HeinrichderMiragoRobot</name>
 555         </spider>
 556         <spider ident="HenryTheMiragoRobot">
 557                 <name>HenryTheMiragoRobot</name>
 558         </spider>
 559         <spider ident="heritrix">
 560                 <name>Heritrix</name>
 561                 <url>http://www.worio.com</url>
 562         </spider>
 563         <spider ident="HKU WWW Robot">
 564                 <name>HKU WWW Octopus</name>
 565         </spider>
 566         <spider ident="HolyCowDude">
 567                 <name>HolyCowDude</name>
 568                 <url>http://www.holycowdude.com/spider.htm</url>
 569         </spider>
 570         <spider ident="HomeTags">
 571                 <name>HomeTags</name>
 572                 <url>http://www.hometags.nl/bot</url>
 573         </spider>
 574         <spider ident="Hometown">
 575                 <name>Hometown</name>
 576         </spider>
 577         <spider ident="htdig">
 578                 <name>ht://Dig</name>
 579         </spider>
 580         <spider ident="AITCSRobot">
 581                 <name>HTML Index</name>
 582         </spider>
 583         <spider ident="HTMLgobble">
 584                 <name>HTMLgobble</name>
 585         </spider>
 586         <spider ident="I Robot">
 587                 <name>I, Robot</name>
 588         </spider>
 589         <spider ident="iajaBot">
 590                 <name>iajaBot</name>
 591         </spider>
 592         <spider ident="IBM_Planetwide">
 593                 <name>IBM_Planetwide</name>
 594         </spider>
 595         <spider ident="+http://www.icerocket.com/">
 596                 <name>IceRocket</name>
 597                 <url>http://www.icerocket.com/</url>
 598         </spider>
 599         <spider ident="ichiro">
 600                 <name>ichiro</name>
 601         </spider>
 602         <spider ident="IlTrovatore-Setaccio">
 603                 <name>IlTrovatore-Setaccio</name>
 604                 <url>http://www.iltrovatore.it/aiuto/faq.html</url>
 605         </spider>
 606         <spider ident="image.kapsi.net">
 607                 <name>image.kapsi.net</name>
 608         </spider>
 609         <spider ident="Mozilla 3.01 PBWF (Win95)">
 610                 <name>Imagelock</name>
 611         </spider>
 612         <spider ident="IncyWincy">
 613                 <name>IncyWincy</name>
 614         </spider>
 615         <spider ident="Informant">
 616                 <name>Informant</name>
 617         </spider>
 618         <spider ident="InfoSeek Robot">
 619                 <name>InfoSeek Robot 1.0</name>
 620         </spider>
 621         <spider ident="Infoseek Sidewinder">
 622                 <name>Infoseek Sidewinder</name>
 623         </spider>
 624         <spider ident="InfoSpiders">
 625                 <name>InfoSpiders</name>
 626         </spider>
 627         <spider ident="INGRID">
 628                 <name>Ingrid</name>
 629         </spider>
 630         <spider ident="slurp@inktomi">
 631                 <name>Inktomi</name>
 632         </spider>
 633         <spider ident="Insitor">
 634                 <name>Insitor</name>
 635                 <url>http://www.insitor.de/</url>
 636         </spider>
 637         <spider ident="inspectorwww">
 638                 <name>Inspector Web</name>
 639         </spider>
 640         <spider ident="IAGENT">
 641                 <name>IntelliAgent</name>
 642         </spider>
 643         <spider ident="Intelliseek">
 644                 <name>Intelliseek</name>
 645                 <url>http://www.intelliseek.com/</url>
 646         </spider>
 647         <spider ident="Internet Cruiser Robot">
 648                 <name>Internet Cruiser</name>
 649         </spider>
 650         <spider ident="internetseer">
 651                 <name>Internet Seer</name>
 652         </spider>
 653         <spider ident="sharp-info-agent">
 654                 <name>Internet Shinchakubin</name>
 655         </spider>
 656         <spider ident="InternetLinkAgent">
 657                 <name>InternetLinkAgent</name>
 658         </spider>
 659         <spider ident="IRLbot">
 660                 <name>IRL Crawler</name>
 661                 <url>http://irl.cs.tamu.edu/crawler</url>
 662         </spider>
 663         <spider ident="Iron33">
 664                 <name>Iron33</name>
 665         </spider>
 666         <spider ident="IsraeliSearch">
 667                 <name>Israeli-search</name>
 668         </spider>
 669         <spider ident="itchBot">
 670                 <name>itch</name>
 671         </spider>
 672         <spider ident="JavaBee">
 673                 <name>JavaBee</name>
 674         </spider>
 675         <spider ident="JBot">
 676                 <name>JBot</name>
 677         </spider>
 678         <spider ident="JCrawler">
 679                 <name>JCrawler</name>
 680         </spider>
 681         <spider ident="JetBot">
 682                 <name>JetEye</name>
 683                 <url>http://www.jeteye.com/jetbot.html</url>
 684         </spider>
 685         <spider ident="JoBo">
 686                 <name>JoBo</name>
 687         </spider>
 688         <spider ident="Jobot">
 689                 <name>Jobot</name>
 690         </spider>
 691         <spider ident="jobs.de">
 692                 <name>Jobs.de</name>
 693                 <url>http://www.jobs.de/</url>
 694         </spider>
 695         <spider ident="JoeBot">
 696                 <name>JoeBot</name>
 697         </spider>
 698         <spider ident="jumpstation">
 699                 <name>JumpStation</name>
 700         </spider>
 701         <spider ident="Katipo">
 702                 <name>Katipo</name>
 703         </spider>
 704         <spider ident="KDD-Explorer">
 705                 <name>KDD-Explorer</name>
 706         </spider>
 707         <spider ident="KIT-Fireball">
 708                 <name>KIT-Fireball</name>
 709         </spider>
 710         <spider ident="KO_Yappo_Robot">
 711                 <name>KO_Yappo_Robot</name>
 712         </spider>
 713         <spider ident="LabelGrab">
 714                 <name>LabelGrabber</name>
 715         </spider>
 716         <spider ident="larbin">
 717                 <name>larbin</name>
 718         </spider>
 719         <spider ident="legs">
 720                 <name>legs</name>
 721         </spider>
 722         <spider ident="linkdexbot">
 723                 <name>Linkdex</name>
 724                 <url>http://www.linkdex.com/bots/</url>
 725         </spider>
 726         <spider ident="LinkScan Server">
 727                 <name>LinkScan</name>
 728         </spider>
 729         <spider ident="LinkWalker">
 730                 <name>LinkWalker</name>
 731         </spider>
 732         <spider ident="Linguee Bot">
 733                 <name>Linguee</name>
 734                 <url>http://www.linguee.com/bot</url>
 735         </spider>
 736         <spider ident="livedoorCheckers/">
 737                 <name>livedoorCheckers</name>
 738         </spider>
 739         <spider ident="Lockon">
 740                 <name>Lockon</name>
 741         </spider>
 742         <spider ident="logo.gif crawler">
 743                 <name>logo.gif</name>
 744         </spider>
 745         <spider ident="Lycos">
 746                 <name>Lycos</name>
 747         </spider>
 748         <spider ident="Magpie">
 749                 <name>Magpie</name>
 750         </spider>
 751         <spider ident="MJ12bot">
 752                 <name>Majestics MJ12bot</name>
 753         </spider>
 754         <spider ident="Mammoth">
 755                 <name>Mammoth</name>
 756                 <url>http://www.sli-systems.com</url>
 757         </spider>
 758         <spider ident="Marvin">
 759                 <name>Marvin</name>
 760         </spider>
 761         <spider ident="marvin/infoseek">
 762                 <name>marvin/infoseek</name>
 763         </spider>
 764         <spider ident="M/3.8">
 765                 <name>Mattie</name>
 766         </spider>
 767         <spider ident="MediaFox">
 768                 <name>MediaFox</name>
 769         </spider>
 770         <spider ident="memorybot">
 771                 <name>Memorybot</name>
 772                 <url>http://archivethe.net/en/index.php/about/internet_memory1</url>
 773         </spider>
 774         <spider ident="mercator">
 775                 <name>Mercator</name>
 776                 <url>http://research.compaq.com/SRC/mercator/</url>
 777         </spider>
 778         <spider ident="MerzScope">
 779                 <name>MerzScope</name>
 780         </spider>
 781         <spider ident="METASpider">
 782                 <name>META</name>
 783                 <url>http://www.meta.com.ua/</url>
 784         </spider>
 785         <spider ident="MetaGer-LinkChecker">
 786                 <name>MetaGer</name>
 787         </spider>
 788         <spider ident="MindCrawler">
 789                 <name>MindCrawler</name>
 790         </spider>
 791         <spider ident="Miva">
 792                 <name>Miva</name>
 793         </spider>
 794         <spider ident="UdmSearch">
 795                 <name>mnoGoSearch</name>
 796         </spider>
 797         <spider ident="moget">
 798                 <name>moget</name>
 799         </spider>
 800         <spider ident="MOMspider">
 801                 <name>MOMspider</name>
 802         </spider>
 803         <spider ident="Monster">
 804                 <name>Monster</name>
 805         </spider>
 806         <spider ident="Moreoverbot">
 807                 <name>Moreover</name>
 808                 <url>http://www.moreover.com</url>
 809         </spider>
 810         <spider ident="msnbot">
 811                 <name>MSNBot</name>
 812                 <url>http://search.msn.com/msnbot.htm</url>
 813         </spider>
 814         <spider ident="MSRBOT">
 815                 <name>MSRBOT</name>
 816                 <url>http://research.microsoft.com/research/sv/msrbot/</url>
 817         </spider>
 818         <spider ident="MuscatFerret">
 819                 <name>Muscat Ferret</name>
 820         </spider>
 821         <spider ident="MwdSearch">
 822                 <name>Mwd.Search</name>
 823         </spider>
 824         <spider ident="NPBot">
 825                 <name>NameProtect</name>
 826         </spider>
 827         <spider ident="NaverBot">
 828                 <name>NaverBot</name>
 829                 <url>http://www.spidermatic.com/en/robot-spider/20</url>
 830         </spider>
 831         <spider ident="NDSpider">
 832                 <name>NDSpider</name>
 833         </spider>
 834         <spider ident="NEC-MeshExplorer">
 835                 <name>NEC-MeshExplorer</name>
 836         </spider>
 837         <spider ident="Nederland.zoek">
 838                 <name>Nederland.zoek</name>
 839         </spider>
 840         <spider ident="NerdyBot">
 841                 <name>NerdyBot</name>
 842                 <url>http://nerdybot.com/</url>
 843         </spider>
 844         <spider ident="NetCarta CyberPilot Pro">
 845                 <name>NetCarta WebMap</name>
 846         </spider>
 847         <spider ident="Netcraft">
 848                 <name>Netcraft Web Server Survey</name>
 849                 <url>http://news.netcraft.com/</url>
 850         </spider>
 851         <spider ident="NetMechanic">
 852                 <name>NetMechanic</name>
 853         </spider>
 854         <spider ident="NetScoop">
 855                 <name>NetScoop</name>
 856         </spider>
 857         <spider ident="newscan-online">
 858                 <name>newscan-online</name>
 859         </spider>
 860         <spider ident="NextGenSearchBot 1">
 861                 <name>NextGenSearchBot</name>
 862                 <url>http://www.zoominfo.com/NextGenSearchBot</url>
 863         </spider>
 864         <spider ident="NHSEWalker">
 865                 <name>NHSE Web Forager</name>
 866         </spider>
 867         <spider ident="NIF">
 868                 <name>NIF</name>
 869                 <url>http://www.newsisfree.com/robot.php</url>
 870         </spider>
 871         <spider ident="NimbleCrawler">
 872                 <name>NimbleCrawler</name>
 873                 <url>http://www.healthline.com/aboutus.jsp</url>
 874         </spider>
 875         <spider ident="Nomad">
 876                 <name>Nomad</name>
 877         </spider>
 878         <spider ident="Norbert the Spider">
 879                 <name>Norbert</name>
 880                 <url>http://www.Burf.com</url>
 881         </spider>
 882         <spider ident="Gulliver">
 883                 <name>Northern Light</name>
 884         </spider>
 885         <spider ident="explorersearch">
 886                 <name>nzexplorer</name>
 887         </spider>
 888         <spider ident="Occam">
 889                 <name>Occam</name>
 890         </spider>
 891         <spider ident="Ocelli">
 892                 <name>Ocelli</name>
 893                 <url>http://www.globalspec.com/Ocelli</url>
 894         </spider>
 895         <spider ident="Online24-Bot">
 896                 <name>Online24-Bot</name>
 897         </spider>
 898         <spider ident="Openbot">
 899                 <name>Openbot</name>
 900                 <url>http://www.openfind.com.tw/robot.html</url>
 901         </spider>
 902         <spider ident="Openfind">
 903                 <name>Openfind data gatherer</name>
 904         </spider>
 905         <spider ident="Orbsearch">
 906                 <name>Orb Search</name>
 907         </spider>
 908         <spider ident="PackRat">
 909                 <name>Pack Rat</name>
 910         </spider>
 911         <spider ident="PageBoy">
 912                 <name>PageBoy</name>
 913         </spider>
 914         <spider ident="Pandalytics">
 915                 <name>Pandalytics</name>
 916                 <url>https://domainsbot.com/pandalytics/</url>
 917         </spider>
 918         <spider ident="ParaSite">
 919                 <name>ParaSite</name>
 920         </spider>
 921         <spider ident="Patric">
 922                 <name>Patric</name>
 923         </spider>
 924         <spider ident="PEGASUS">
 925                 <name>pegasus</name>
 926         </spider>
 927         <spider ident="PerlCrawler/1.0 Xavatoria/2.0">
 928                 <name>PerlCrawler 1.0</name>
 929         </spider>
 930         <spider ident="PetalBot">
 931                 <name>PetalBot</name>
 932                 <url>https://aspiegel.com/petalbot</url>
 933         </spider>
 934         <spider ident="PGP-KA">
 935                 <name>PGP Key Agent</name>
 936         </spider>
 937         <spider ident="Duppies">
 938                 <name>Phantom</name>
 939         </spider>
 940         <spider ident="phpdig">
 941                 <name>PhpDig</name>
 942         </spider>
 943         <spider ident="PiltdownMan">
 944                 <name>PiltdownMan</name>
 945         </spider>
 946         <spider ident="Pimptrain's robot">
 947                 <name>Pimptrain.com's</name>
 948         </spider>
 949         <spider ident="pingalink">
 950                 <name>PingALink</name>
 951         </spider>
 952         <spider ident="Pioneer">
 953                 <name>Pioneer</name>
 954         </spider>
 955         <spider ident="PluckFeedCrawler">
 956                 <name>Pluck</name>
 957                 <url>http://www.pluck.com</url>
 958         </spider>
 959         <spider ident="PlumtreeWebAccessor">
 960                 <name>PlumtreeWebAccessor</name>
 961         </spider>
 962         <spider ident="PodNova">
 963                 <name>PodNova</name>
 964                 <url>http://www.podnova.com</url>
 965         </spider>
 966         <spider ident="Pompos">
 967                 <name>Pompos</name>
 968                 <url>http://dir.com/pompos.html</url>
 969         </spider>
 970         <spider ident="Poppi">
 971                 <name>Poppi</name>
 972         </spider>
 973         <spider ident="publiclibraryarchive.org">
 974                 <name>publiclibraryarchive.org</name>
 975         </spider>
 976         <spider ident="gestaltIconoclast">
 977                 <name>Popular Iconoclast</name>
 978         </spider>
 979         <spider ident="PortalJuice.com">
 980                 <name>Portal Juice</name>
 981         </spider>
 982         <spider ident="PortalBSpider">
 983                 <name>PortalB Spider</name>
 984         </spider>
 985         <spider ident="Qualidator">
 986                 <name>Qualidator</name>
 987                 <url>www.qualidator.com/Web/de/Support/FAQ_OnlineTestStatistiken.htm</url>
 988         </spider>
 989         <spider ident="www.kolinka.com">
 990                 <name>Project Kolinka Forum Search</name>
 991                 <url>http://www.kolinka.com/</url>
 992         </spider>
 993         <spider ident="psbot">
 994                 <name>psbot</name>
 995         </spider>
 996         <spider ident="Qango.com Web Directory">
 997                 <name>Qango</name>
 998                 <url>http://www.qango.com</url>
 999         </spider>
1000         <spider ident="Qwant">
1001                 <name>Qwant</name>
1002                 <url>https://help.qwant.com/bot/</url>
1003         </spider>
1004         <spider ident="SBSearch">
1005                 <name>SecretSearchEngineLabs.com</name>
1006                 <url>http://www.secretsearchenginelabs.com/secret-web-crawler.php</url>
1007         </spider>
1008         <spider ident="SemrushBot">
1009                 <name>SemrushBot</name>
1010                 <url>http://semrush.com/bot/</url>
1011         </spider>
1012         <spider ident="StackRambler">
1013                 <name>Rambler</name>
1014                 <url>http://www.rambler.ru/</url>
1015         </spider>
1016         <spider ident="Raven">
1017                 <name>Raven Search</name>
1018         </spider>
1019         <spider ident="Resume Robot">
1020                 <name>Resume Robot</name>
1021         </spider>
1022         <spider ident="Road Runner: ImageScape Robot">
1023                 <name>Road Runner: The ImageScape Robot</name>
1024         </spider>
1025         <spider ident="RHCS">
1026                 <name>RoadHouse Crawling System</name>
1027         </spider>
1028         <spider ident="Robbie">
1029                 <name>Robbie the Robot</name>
1030         </spider>
1031         <spider ident="RoboCrawl">
1032                 <name>RoboCrawl</name>
1033         </spider>
1034         <spider ident="Robofox">
1035                 <name>RoboFox</name>
1036         </spider>
1037         <spider ident="Robot du CRIM 1.0a">
1038                 <name>Robot Francoroute</name>
1039         </spider>
1040         <spider ident="Robozilla">
1041                 <name>Robozilla</name>
1042         </spider>
1043         <spider ident="Roverbot">
1044                 <name>Roverbot</name>
1045         </spider>
1046         <spider ident="RSS-SPIDER">
1047                 <name>RSS Feed Seeker</name>
1048                 <url>http://www.rss-spider.com/fsb.php</url>
1049         </spider>
1050         <spider ident="RuLeS">
1051                 <name>RuLeS</name>
1052         </spider>
1053         <spider ident="RyzeCrawler">
1054                 <name>RyzeCrawler</name>
1055                 <url>http://www.domain2day.nl/crawler/</url>
1056         </spider>
1057         <spider ident="SafetyNet Robot">
1058                 <name>SafetyNet</name>
1059         </spider>
1060         <spider ident="SBIder">
1061                 <name>SBIder.</name>
1062                 <url>http://www.sitesell.com/sbider.html</url>
1063         </spider>
1064         <spider ident="Scharia">
1065                 <name>Scharia</name>
1066         </spider>
1067         <spider ident="Science-Index">
1068                 <name>Science-Index</name>
1069         </spider>
1070         <spider ident="Scooter">
1071                 <name>Scooter</name>
1072         </spider>
1073         <spider ident="SearchNZ">
1074                 <name>SearchNZ</name>
1075                 <url>http://www.searchnz.co.nz/</url>
1076         </spider>
1077         <spider ident="searchprocess">
1078                 <name>SearchProcess</name>
1079         </spider>
1080         <spider ident="SearchmetricsBot">
1081                 <name>SearchmetricsBot</name>
1082                 <url>http://www.searchmetrics.com/en/searchmetrics-bot/</url>
1083         </spider>
1084         <spider ident="Seekbot">
1085                 <name>Seekbot</name>
1086                 <url>http://www.seekbot.net/bot.html</url>
1087         </spider>
1088         <spider ident="Senrigan">
1089                 <name>Senrigan</name>
1090         </spider>
1091         <spider ident="Sensis Web Crawler">
1092                 <name>Sensis Web Crawler</name>
1093                 <url>http://www.sensis.com.au/help.do</url>
1094         </spider>
1095         <spider ident="SentiBot">
1096                 <name>SentiBot</name>
1097                 <url>http://www.sentibot.eu</url>
1098         </spider>
1099         <spider ident="SEOkicks">
1100                 <name>SEOkicks</name>
1101                 <url>https://www.seokicks.de/robot.html</url>
1102         </spider>
1103         <spider ident="serpstatbot">
1104                 <name>serpstatbot</name>
1105                 <url>http://serpstatbot.com/</url>
1106         </spider>
1107         <spider ident="SeznamBot/3.2">
1108                 <name>Seznam Bot</name>
1109                 <url>http://napoveda.seznam.cz/en/seznambot-intro/</url>
1110         </spider>
1111         <spider ident="SG-Scout">
1112                 <name>SG-Scout</name>
1113         </spider>
1114         <spider ident="Shagseeker">
1115                 <name>ShagSeeker</name>
1116         </spider>
1117         <spider ident="Shai'Hulud">
1118                 <name>Shai'Hulud</name>
1119         </spider>
1120         <spider ident="SimBot/1.0">
1121                 <name>Simmany Robot Ver1.0</name>
1122         </spider>
1123         <spider ident="SkypeUriPreview">
1124                 <name>Skype Preview</name>
1125                 <url>https://www.skype.com/</url>
1126         </spider>
1127         <spider ident="ssearcher100">
1128                 <name>Site Searcher</name>
1129         </spider>
1130         <spider ident="Site Valet">
1131                 <name>Site Valet</name>
1132         </spider>
1133         <spider ident="http://www.site-list.net">
1134                 <name>Site-List</name>
1135                 <url>http://www.site-list.net</url>
1136         </spider>
1137         <spider ident="SiteTech-Rover">
1138                 <name>SiteTech-Rover</name>
1139         </spider>
1140         <spider ident="+SitiDi.net/SitiDiBot/">
1141                 <name>SitiDi.net/SitiDiBot</name>
1142         </spider>
1143         <spider ident="aWapClient">
1144                 <name>Skymob.com</name>
1145         </spider>
1146         <spider ident="Slack">
1147                 <name>Slackbot</name>
1148                 <url>https://api.slack.com/robots</url>
1149         </spider>
1150         <spider ident="SLCrawler">
1151                 <name>SLCrawler</name>
1152         </spider>
1153         <spider ident="Sleek Spider">
1154                 <name>Sleek</name>
1155         </spider>
1156         <spider ident="ESISmartSpider">
1157                 <name>Smart Spider</name>
1158         </spider>
1159         <spider ident="Snapbot">
1160                 <name>Snapbot</name>
1161                 <url>http://www.snap.com/</url>
1162         </spider>
1163         <spider ident="Snooper">
1164                 <name>Snooper</name>
1165         </spider>
1166         <spider ident="sohu-search">
1167                 <name>sohu-search</name>
1168         </spider>
1169         <spider ident="Solbot">
1170                 <name>Solbot</name>
1171         </spider>
1172         <spider ident="Speedy Spider">
1173                 <name>Speedy Spider</name>
1174                 <url>http://www.entireweb.com/about/search_tech/speedyspider/</url>
1175         </spider>
1176         <spider ident="Sphere Scout">
1177                 <name>Sphere</name>
1178         </spider>
1179         <spider ident="Sphider2">
1180                 <name>Sphider</name>
1181         </spider>
1182         <spider ident="SpiderBot">
1183                 <name>SpiderBot</name>
1184         </spider>
1185         <spider ident="spiderline">
1186                 <name>Spiderline Crawler</name>
1187         </spider>
1188         <spider ident="SpiderMan">
1189                 <name>SpiderMan</name>
1190         </spider>
1191         <spider ident="SpiderView">
1192                 <name>SpiderView(tm)</name>
1193         </spider>
1194         <spider ident="mouse.house">
1195                 <name>spider_monkey</name>
1196         </spider>
1197         <spider ident="suke">
1198                 <name>Suke</name>
1199         </spider>
1200         <spider ident="suntek">
1201                 <name>suntek search engine</name>
1202         </spider>
1203         <spider ident="Szukacz">
1204                 <name>Szukacz</name>
1205                 <url>http://www.szukacz.pl/html/RobotEnglishVersion.html</url>
1206         </spider>
1207         <spider ident="T-H-U-N-D-E-R-S-T-O-N-E">
1208                 <name>T-H-U-N-D-E-R-S-T-O-N-E</name>
1209         </spider>
1210         <spider ident="TinEye Crawler">
1211                 <name>TinEye</name>
1212                 <url>http://tineye.com/crawler.html</url>
1213         </spider>
1214         <spider ident="Black Widow">
1215                 <name>TACH Black Widow</name>
1216         </spider>
1217         <spider ident="Tapatalk CloudSearch">
1218                 <name>Tapatalk CloudSearch</name>
1219         </spider>
1220         <spider ident="Tarantula">
1221                 <name>Tarantula</name>
1222         </spider>
1223         <spider ident="tarspider">
1224                 <name>tarspider</name>
1225         </spider>
1226         <spider ident="dlw3robot">
1227                 <name>Tcl W3 Robot</name>
1228         </spider>
1229         <spider ident="TechBOT">
1230                 <name>TechBOT</name>
1231         </spider>
1232         <spider ident="Technoratibot">
1233                 <name>Technorati</name>
1234                 <url>http://technorati.com/about/</url>
1235         </spider>
1236         <spider ident="Templeton">
1237                 <name>Templeton</name>
1238         </spider>
1239         <spider ident="teoma">
1240                 <name>Teoma/Ask Jeeves</name>
1241                 <url>http://sp.teoma.com/docs/teoma/about/</url>
1242         </spider>
1243         <spider ident="JubiiRobot">
1244                 <name>The Jubii</name>
1245         </spider>
1246         <spider ident="NorthStar">
1247                 <name>The NorthStar Robot</name>
1248         </spider>
1249         <spider ident="w3index">
1250                 <name>The NWI Robot</name>
1251         </spider>
1252         <spider ident="Peregrinator-Mathematics">
1253                 <name>The Peregrinator</name>
1254         </spider>
1255         <spider ident="Pixray-Seeker">
1256                 <name>Pixray</name>
1257                 <url>http://www.pixray.com/pixraybot/</url>
1258         </spider>
1259         <spider ident="TelegramBot (like TwitterBot)">
1260                 <name>TelegramBot (like TwitterBot)</name>
1261                 <url>https://telegram.org/</url>
1262         </spider>
1263         <spider ident="thumbshots-de-Bot">
1264                 <name>thumbshots-de-Bot</name>
1265         </spider>
1266         <spider ident="TITAN">
1267                 <name>TITAN</name>
1268         </spider>
1269         <spider ident="TitIn">
1270                 <name>TitIn</name>
1271         </spider>
1272         <spider ident="TLSpider">
1273                 <name>TLSpider</name>
1274         </spider>
1275         <spider ident="TMCrawler">
1276                 <name>TMCrawler</name>
1277         </spider>
1278         <spider ident="trendictionbot">
1279                 <name>Trendiction-Bot</name>
1280                 <url>http://www.trendiction.com/bot</url>
1281         </spider>
1282         <spider ident="slysearch">
1283                 <name>Turnitin.com</name>
1284                 <url>http://www.turnitin.com/static/products_services/search_engines.html</url>
1285         </spider>
1286         <spider ident="TurnitinBot/">
1287                 <name>TurnitinBot</name>
1288         </spider>
1289         <spider ident="TurtleScanner">
1290                 <name>Turtle</name>
1291                 <url>http://www.turtle.ru/</url>
1292         </spider>
1293         <spider ident="TwengaBot">
1294                 <name>Twenga</name>
1295                 <url>http://www.twenga.com/bot.html</url>
1296         </spider>
1297         <spider ident="Twiceler">
1298                 <name>Twiceler</name>
1299                 <url>http://www.cuill.com/twiceler/robot.html</url>
1300         </spider>
1301         <spider ident="Twitterbot">
1302                 <name>Twitterbot</name>
1303                 <url>https://twitter.com/</url>
1304         </spider>
1305         <spider ident="UCSD-Crawler">
1306                 <name>UCSD Crawl</name>
1307         </spider>
1308         <spider ident="UMBC-memeta-Bot">
1309                 <name>UMBC</name>
1310         </spider>
1311         <spider ident="unisterbot">
1312                 <name>Unister</name>
1313         </spider>
1314         <spider ident="Unpartisan">
1315                 <name>Unpartisan</name>
1316                 <url>http://www.unpartisan.com</url>
1317         </spider>
1318         <spider ident="UptimeRobot/2.0">
1319                 <name>Uptime Robot</name>
1320                 <url>http://uptimerobot.com/</url>
1321         </spider>
1322         <spider ident="urlck">
1323                 <name>URL Check</name>
1324         </spider>
1325         <spider ident="URL Spider Pro">
1326                 <name>URL Spider Pro</name>
1327         </spider>
1328         <spider ident="Valkyrie">
1329                 <name>Valkyrie</name>
1330         </spider>
1331         <spider ident="Verticrawl">
1332                 <name>Verticrawl</name>
1333         </spider>
1334         <spider ident="Victoria">
1335                 <name>Victoria</name>
1336         </spider>
1337         <spider ident="vision-search">
1338                 <name>vision-search</name>
1339         </spider>
1340         <spider ident="VoilaBot">
1341                 <name>VoilaBot</name>
1342                 <url>http://www.voila.com/</url>
1343         </spider>
1344         <spider ident="VisBot">
1345                 <name>VisBot</name>
1346                 <url>http://www.visvo.com/webmasters.html</url>
1347         </spider>
1348         <spider ident="Voyager">
1349                 <name>Voyager</name>
1350         </spider>
1351         <spider ident="VWbot_K">
1352                 <name>VWbot</name>
1353         </spider>
1354         <spider ident="W3M2">
1355                 <name>W3M2</name>
1356         </spider>
1357         <spider ident="w3mir">
1358                 <name>w3mir</name>
1359         </spider>
1360         <spider ident="w@pSpider">
1361                 <name>w@pSpider</name>
1362         </spider>
1363         <spider ident="appie">
1364                 <name>Walhello appie</name>
1365                 <url>http://www.robotstxt.org/wc/active/html/appie.html</url>
1366         </spider>
1367         <spider ident="CrawlPaper">
1368                 <name>WallPaper</name>
1369         </spider>
1370         <spider ident="root">
1371                 <name>Web Core / Roots</name>
1372         </spider>
1373         <spider ident="WBSearchBot">
1374                 <name>Ware Bay</name>
1375                 <url>http://www.warebay.com/bot.html</url>
1376         </spider>
1377         <spider ident="WebMoose">
1378                 <name>Web Moose</name>
1379         </spider>
1380         <spider ident="WebBandit">
1381                 <name>WebBandit</name>
1382         </spider>
1383         <spider ident="WebCatcher">
1384                 <name>WebCatcher</name>
1385         </spider>
1386         <spider ident="Webclipping">
1387                 <name>Webclipping</name>
1388         </spider>
1389         <spider ident="WebCopy">
1390                 <name>WebCopy</name>
1391         </spider>
1392         <spider ident="WebFetcher">
1393                 <name>webfetcher</name>
1394         </spider>
1395         <spider ident="weblayers">
1396                 <name>weblayers</name>
1397         </spider>
1398         <spider ident="WebLinker">
1399                 <name>WebLinker</name>
1400         </spider>
1401         <spider ident="wlm">
1402                 <name>Weblog Monitor</name>
1403         </spider>
1404         <spider ident="WebQuest">
1405                 <name>WebQuest</name>
1406         </spider>
1407         <spider ident="WebReaper">
1408                 <name>WebReaper</name>
1409         </spider>
1410         <spider ident="webs@recruit.co.jp">
1411                 <name>webs</name>
1412         </spider>
1413         <spider ident="websearchbench">
1414                 <name>WebSearchBench</name>
1415                 <url>http://websearchbench.cs.uni-dortmund.de/</url>
1416         </spider>
1417         <spider ident="WOLP">
1418                 <name>WebStolperer</name>
1419         </spider>
1420         <spider ident="webvac">
1421                 <name>WebVac</name>
1422         </spider>
1423         <spider ident="webwalk">
1424                 <name>webwalk</name>
1425         </spider>
1426         <spider ident="WebWalker">
1427                 <name>WebWalker</name>
1428         </spider>
1429         <spider ident="WebWatch">
1430                 <name>WebWatch</name>
1431         </spider>
1432         <spider ident="WebZinger">
1433                 <name>WebZinger</name>
1434         </spider>
1435         <spider ident="whatUseek_winona">
1436                 <name>whatUseek Winona</name>
1437         </spider>
1438         <spider ident="WhoWhere Robot">
1439                 <name>WebWatch</name>
1440                 <url>http://www.whowhere.com</url>
1441         </spider>
1442         <spider ident="SurveyBot">
1443                 <name>Whois Source</name>
1444                 <url>http://www.whois.sc/info/webmasters/surveybot.html</url>
1445         </spider>
1446         <spider ident="Hazel's Ferret Web hopper">
1447                 <name>Wild Ferret Web Hopper</name>
1448         </spider>
1449         <spider ident="HRCrawler">
1450                 <name>HRCrawler</name>
1451         </spider>
1452         <spider ident="WinHTTP">
1453                 <name>WinHTTP</name>
1454         </spider>
1455         <spider ident="wired-digital-newsbot">
1456                 <name>Wired Digital</name>
1457         </spider>
1458         <spider ident="zyborg">
1459                 <name>WiseNut</name>
1460         </spider>
1461         <spider ident="WoltLabSuite">
1462                 <name>WoltLab Suite</name>
1463         </spider>
1464         <spider ident="OmniExplorer_Bot">
1465                 <name>WorldIndexer</name>
1466                 <url>http://www.omni-explorer.com</url>
1467         </spider>
1468         <spider ident="WWWC">
1469                 <name>WWWC</name>
1470         </spider>
1471         <spider ident="WWWeasel Robot">
1472                 <name>WWWeasel Robot</name>
1473         </spider>
1474         <spider ident="wwwster">
1475                 <name>wwwster</name>
1476         </spider>
1477         <spider ident="WWWWanderer">
1478                 <name>WWWWanderer</name>
1479         </spider>
1480         <spider ident="TECOMAC-Crawler">
1481                 <name>X-Crawler</name>
1482         </spider>
1483         <spider ident="XGET">
1484                 <name>XGET</name>
1485         </spider>
1486         <spider ident="cosmos">
1487                 <name>XYLEME Robot</name>
1488         </spider>
1489         <spider ident="yacybot">
1490                 <name>YaCy-Bot</name>
1491                 <url>https://yacy.net/bot.html</url>
1492         </spider>
1493         <spider ident="YahooYSMcm">
1494                 <name>Yahoo Publisher Network</name>
1495                 <url>http://publisher.yahoo.com/</url>
1496         </spider>
1497         <spider ident="Yahoo-Blogs">
1498                 <name>Yahoo-Blogs</name>
1499                 <url>http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html</url>
1500         </spider>
1501         <spider ident="Yahoo Pipes">
1502                 <name>Yahoo Pipes</name>
1503         </spider>
1504         <spider ident="Yahoo! Slurp">
1505                 <name>Yahoo! Slurp</name>
1506                 <url>http://help.yahoo.com/help/us/ysearch/slurp</url>
1507         </spider>
1508         <spider ident="Yahoo-VerticalCrawler">
1509                 <name>Yahoo-VerticalCrawler</name>
1510         </spider>
1511         <spider ident="YahooFeedSeeker">
1512                 <name>YahooFeedSeeker</name>
1513                 <url>http://my.yahoo.com/s/publishers.html</url>
1514         </spider>
1515         <spider ident="Yandex">
1516                 <name>Yandex</name>
1517                 <url>http://www.yandex.ru/</url>
1518         </spider>
1519         <spider ident="zeus">
1520                 <name>Zeus Internet Marketing</name>
1521                 <url>http://www.cyber-robotics.com/</url>
1522         </spider>
1523         <spider ident="http://www.zorkk.com">
1524                 <name>Zork</name>
1525                 <url>http://www.zorkk.com</url>
1526         </spider>
1527         <spider ident="Zookabot">
1528                 <name>Zookabotk</name>
1529                 <url>http://zookabot.com/</url>
1530         </spider>
1531         <spider ident="ZoominfoBot">
1532                 <name>ZoominfoBot</name>
1533                 <url>https://www.zoominfo.com/about-zoominfo/zoominfobot</url>
1534         </spider>
1535         <spider ident="360Spider">
1536                 <name>360Spider</name>
1537         </spider>
1538 </data>