1 <?xml version=
"1.0" encoding=
"UTF-8"?>
2 <data xmlns=
"http://www.woltlab.com" xmlns:
xsi=
"http://www.w3.org/2001/XMLSchema-instance" xsi:
schemaLocation=
"http://www.woltlab.com http://www.woltlab.com/XSD/maelstrom/spiderList.xsd">
3 <spider ident=
"ABCdatos">
4 <name>ABCdatos BotLink
</name>
5 <url>http://www.robotstxt.org/wc/active/html/abcdatos.html
</url>
7 <spider ident=
"AbraveSpider">
9 <url>http://robot.abrave.co.uk/
</url>
11 <spider ident=
"abot/">
14 <spider ident=
"Accelatech RSSCrawler">
15 <name>Accelatech
</name>
17 <spider ident=
"Accoona-AI-Agent">
19 <url>http://accoona.com/about/about_accoona.jsp
</url>
21 <spider ident=
"aconon Index">
22 <name>aconon Index (raubfische.de)
</name>
24 <spider ident=
"AcoonBot">
26 <url>http://www.acoon.de/robot.asp
</url>
28 <spider ident=
"Ahoy!">
30 <url>http://www.robotstxt.org/wc/active/html/ahoythehomepagefinder.html
</url>
32 <spider ident=
"AhrefsBot">
34 <url>http://ahrefs.com/robot/
</url>
36 <spider ident=
"AlkalineBOT">
38 <url>http://www.robotstxt.org/wc/active/html/Alkaline.html
</url>
40 <spider ident=
"AlexaBOT">
42 <url>http://www.alexa.com/help/webmasters
</url>
44 <spider ident=
"http://www.almaden.ibm.com/cs/crawler">
45 <name>Almaden Crawler
</name>
46 <url>http://www.almaden.ibm.com/cs/crawler/
</url>
48 <spider ident=
"EMC Spider">
51 <spider ident=
"Anthill">
53 <url>http://www.robotstxt.org/wc/active/html/anthill.html
</url>
55 <spider ident=
"Aport">
57 <url>http://www.aport.ru/
</url>
59 <spider ident=
"AppleSyndication">
62 <spider ident=
"Arachnophilia">
63 <name>Arachnophilia
</name>
64 <url>http://www.robotstxt.org/wc/active/html/arachnophilia.html
</url>
66 <spider ident=
"Araneo">
68 <url>http://www.robotstxt.org/wc/active/html/araneo.html
</url>
70 <spider ident=
"ArchitextSpider">
71 <name>ArchitextSpider
</name>
72 <url>http://www.robotstxt.org/wc/active/html/architext.html
</url>
74 <spider ident=
"archive.org_bot">
75 <name>Archive.org
</name>
76 <url>https://archive.org/details/archive.org_bot
</url>
78 <spider ident=
"arks/1.0">
80 <url>http://www.robotstxt.org/wc/active/html/arks.html
</url>
82 <spider ident=
"ASpider">
84 <url>http://www.robotstxt.org/wc/active/html/aspider.html
</url>
86 <spider ident=
"ATN_Worldwide">
87 <name>ATN Worldwide
</name>
88 <url>http://www.robotstxt.org/wc/active/html/atn.txt.html
</url>
90 <spider ident=
"Atomz">
91 <name>Atomz.com
</name>
92 <url>http://www.robotstxt.org/wc/active/html/atomz.html
</url>
94 <spider ident=
"AURESYS">
96 <url>http://www.robotstxt.org/wc/active/html/auresys.html
</url>
98 <spider ident=
"BackRub">
100 <url>http://www.robotstxt.org/wc/active/html/backrub.html
</url>
102 <spider ident=
"Baiduspider">
103 <name>Baiduspider
</name>
104 <url>http://www.baidu.com/search/spider.htm
</url>
106 <spider ident=
"bbot">
108 <url>http://www.robotstxt.org/wc/active/html/bbot.html
</url>
110 <spider ident=
"BecomeBot">
111 <name>BecomeBot
</name>
112 <url>http://www.become.com/webmasters.html
</url>
114 <spider ident=
"Big Brother">
115 <name>Big Brother
</name>
116 <url>http://www.robotstxt.org/wc/active/html/bigbrother.html
</url>
118 <spider ident=
"BigmirSpider">
120 <url>http://www.bigmir.net/
</url>
122 <spider ident=
"bingbot">
124 <url>http://www.bing.com/bingbot.htm
</url>
126 <spider ident=
"Bitacle bot">
128 <url>http://bitacle.org/
</url>
130 <spider ident=
"Biz360 Spider">
132 <url>http://www.biz360.com
</url>
134 <spider ident=
"Bjaaland">
135 <name>Bjaaland
</name>
136 <url>http://www.robotstxt.org/wc/active/html/bjaaland.html
</url>
138 <spider ident=
"BlackWidow">
139 <name>BlackWidow
</name>
140 <url>http://www.robotstxt.org/wc/active/html/blackwidow.html
</url>
142 <spider ident=
"BLEXBot">
144 <url>http://webmeup.com/crawler.html
</url>
146 <spider ident=
"BlogCrawler by Xango">
147 <name>BlogCrawler
</name>
149 <spider ident=
"blogdb">
151 <url>http://blogdb.jp
</url>
153 <spider ident=
"blog search engine by BlogFan.ORG">
155 <url>http://www.blogfan.org
</url>
157 <spider ident=
"Bloglines">
158 <name>Bloglies
</name>
159 <url>http://www.bloglines.com
</url>
161 <spider ident=
"BlogPulse (ISSpider-3.0)">
162 <name>BlogPulse
</name>
164 <spider ident=
"BlogSearch">
165 <name>BlogSearch
</name>
166 <url>http://www.icerocket.com
</url>
168 <spider ident=
"BlogsNowBot">
169 <name>BlogsNow
</name>
170 <url>http://www.blogsnow.com/
</url>
172 <spider ident=
"BlogStreetBot">
173 <name>BlogStreetBot
</name>
174 <url>http://www.blogstreet.com/
</url>
176 <spider ident=
"Bulkfeeds">
177 <name>BlogStreetBot
</name>
178 <url>http://bulkfeeds.net
</url>
180 <spider ident=
"BoardPulse">
181 <name>BoardPulse
</name>
182 <url>http://www.boardpulse.com/
</url>
184 <spider ident=
"BoardReader">
185 <name>BoardReader
</name>
186 <url>http://www.boardreader.com/aboutus.asp
</url>
188 <spider ident=
"BoardViewer">
189 <name>BoardViewer
</name>
190 <url>http://www.boardviewer.com/
</url>
192 <spider ident=
"boitho.com-robot">
194 <url>http://www.boitho.com/bot.html
</url>
196 <spider ident=
"borg-bot">
197 <name>Borg-Bot
</name>
198 <url>http://www.robotstxt.org/wc/active/html/borg-bot.html
</url>
200 <spider ident=
"BSpider">
202 <url>http://www.robotstxt.org/wc/active/html/bspider.html
</url>
204 <spider ident=
"CACTVS Chemistry Spider">
205 <name>CACTVS Chemistry
</name>
207 <spider ident=
"Calif">
210 <spider ident=
"CaRP/3.6Evolution">
212 <url>http://www.biz360.com
</url>
214 <spider ident=
"Checkbot">
215 <name>Checkbot
</name>
217 <spider ident=
"ChristCrawler.com">
218 <name>ChristCrawler.com
</name>
220 <spider ident=
"www.cienciaficcion.net">
221 <name>cIeNcIaFiCcIoN.nEt
</name>
223 <spider ident=
"CipinetBot">
225 <url>http://www.cipinet.com/bot.html
</url>
227 <spider ident=
"CJNetworkQuality">
228 <name>CJNetworkQuality
</name>
229 <url>http://www.cj.com/networkquality/
</url>
231 <spider ident=
"CMC/0.01">
232 <name>CMC/
0.01</name>
234 <spider ident=
"ColdFusion">
235 <name>ColdFusion
</name>
237 <spider ident=
"combine">
238 <name>Combine System
</name>
240 <spider ident=
"Crawler (cometsearch@cometsystems.com)">
241 <name>cometsystems.com
</name>
243 <spider ident=
"ComputingSite Robi/1.0">
244 <name>ComputingSite Robi/
1.0</name>
246 <spider ident=
"conceptbot">
247 <name>Conceptbot
</name>
249 <spider ident=
"Cooby.de Crawler">
250 <name>Cooby.de Crawler
</name>
252 <spider ident=
"CoolBot">
255 <spider ident=
"Cusco">
258 <spider ident=
"CyberSpyder">
259 <name>CyberSpyder
</name>
261 <spider ident=
"daypopbot">
264 <spider ident=
"DesertRealm.com">
265 <name>Desert Realm
</name>
267 <spider ident=
"Deweb">
268 <name>DeWeb(c)
</name>
270 <spider ident=
"Die Blinde Kuh">
271 <name>Die Blinde Kuh
</name>
272 <url>http://www.robotstxt.org/wc/active/html/blindekuh.html
</url>
274 <spider ident=
"dienstspider">
275 <name>DienstSpider
</name>
277 <spider ident=
"Digger/1.0 JDK/1.3.0">
280 <spider ident=
"Digimarc WebReader">
281 <name>Digimarc MarcSpider
</name>
283 <spider ident=
"Digimarc CGIReader">
284 <name>Digimarc Marcspider/CGI
</name>
286 <spider ident=
"DIIbot">
287 <name>Digital Integrity Robot
</name>
289 <spider ident=
"grabber">
290 <name>Direct Hit Grabber
</name>
292 <spider ident=
"discobot">
293 <name>Discovery
</name>
294 <url>http://discoveryengine.com/discobot.html
</url>
296 <spider ident=
"Discordbot">
298 <url>https://discordapp.com/
</url>
300 <spider ident=
"DNAbot/1.0">
303 <spider ident=
"DotBot/1.1">
304 <name>Moz DotBot
</name>
305 <url>http://www.opensiteexplorer.org/dotbot
</url>
307 <spider ident=
"DragonBot/1.0 libwww/5.0">
308 <name>DragonBot
</name>
310 <spider ident=
"DuckDuckBot">
311 <name>DuckDuckGo
</name>
312 <url>https://duckduckgo.com/
</url>
314 <spider ident=
"DWCP/2.0">
315 <name>DWCP (Dridus' Web Cataloging Project)
</name>
317 <spider ident=
"e-SocietyRobot">
318 <name>e-Society
</name>
319 <url>http://www.yama.info.waseda.ac.jp/~yamana/es/index_eng.htm
</url>
321 <spider ident=
"exactseek-pagereaper">
322 <name>eaxactseek-page
</name>
324 <spider ident=
"EbiNess/0.01a">
327 <spider ident=
"edgeio-retriever">
329 <url>http://www.edgeio.com
</url>
331 <spider ident=
"EIT-Link-Verifier-Robot/0.2">
332 <name>EIT Link Verifier Robot
</name>
334 <spider ident=
"elfinbot">
335 <name>ELFINBOT
</name>
337 <spider ident=
"Emacs-w3/v[0-9\.]+">
338 <name>Emacs-w3 Search Engine
</name>
340 <spider ident=
"esther">
343 <spider ident=
"EuripBot/">
344 <name>EuripBot
</name>
346 <spider ident=
"Evliya Celebi">
347 <name>Evliya Celebi
</name>
349 <spider ident=
"ExactSeek_Spider">
350 <name>ExactSeek_Spider
</name>
351 <url>http://www.askjeevs.com
</url>
353 <spider ident=
"NG/2.0">
355 <url>http://botspotter.net/bs-
389.html
</url>
357 <spider ident=
"ExaBot">
358 <name>ExaLead Beta
</name>
359 <url>http://beta.exalead.com/search/C=
0/
2p=Help
.7</url>
361 <spider ident=
"ExaLead">
363 <url>http://www.exalead.com/search
</url>
365 <spider ident=
"facebookexternalhit">
366 <name>Facebook
</name>
367 <url>http://www.facebook.com/externalhit_uatext.php
</url>
369 <spider ident=
"fast-webcrawler">
370 <name>FAST / AlltheWeb
</name>
371 <url>http://help.yahoo.com/help/us/ysearch/slurp/index.html
</url>
373 <spider ident=
"FastCrawler">
374 <name>FastCrawler
</name>
376 <spider ident=
"Feed24.com">
378 <url>http://www.feed24.com
</url>
380 <spider ident=
"FeedBlitz">
381 <name>FeedBlitz
</name>
382 <url>http://www.feedblitz.com
</url>
384 <spider ident=
"FeedBurner">
385 <name>FeedBurner
</name>
386 <url>http://www.FeedBurner.com
</url>
388 <spider ident=
"Feedly">
390 <url>http://www.feedly.com/fetcher.html
</url>
392 <spider ident=
"Feedfetcher-Google">
393 <name>FeedFetcher-Google
</name>
394 <url>http://www.google.com/feedfetcher.html
</url>
396 <spider ident=
"UniversalFeedParser">
397 <name>FeedParser
</name>
398 <url>http://www.feedparser.org
</url>
400 <spider ident=
"Feedster Crawler">
401 <name>Feedster
</name>
402 <url>http://www.feedstermedia.com/
</url>
404 <spider ident=
"FEHLSTART Superspider">
405 <name>FEHLSTART
</name>
407 <spider ident=
"FelixIDE">
408 <name>Felix IDE
</name>
410 <spider ident=
"ESIRover">
411 <name>FetchRover
</name>
413 <spider ident=
"fido">
416 <spider ident=
"findlinks">
417 <name>FindLinks
</name>
418 <url>http://wortschatz.uni-leipzig.de/findlinks/
</url>
420 <spider ident=
"FindoryBot">
422 <url>http://www.findory.com
</url>
424 <spider ident=
"Fish-Search-Robot">
425 <name>Fish search
</name>
427 <spider ident=
"Mozilla/4.0 (compatible: FDSE robot)">
428 <name>Fluid Dynamics
</name>
430 <spider ident=
"fouineur.9bit.qc.ca">
431 <name>Fouineur
</name>
433 <spider ident=
"Freecrawl">
434 <name>Freecrawl
</name>
436 <spider ident=
"FunnelWeb">
437 <name>FunnelWeb
</name>
439 <spider ident=
"GaisBot">
441 <url>http://gais.cs.ccu.edu.tw/robot.php
</url>
443 <spider ident=
"gamekitbot">
445 <url>http://www.uchoose.de/crawler/gamekitbot/
</url>
447 <spider ident=
"gammaSpider">
448 <name>gammaSpider
</name>
450 <spider ident=
"gazz">
453 <spider ident=
"gcreep">
456 <spider ident=
"genieBot">
457 <name>genieBot
</name>
458 <url>http://
64.5.245.11/faq/faq.html
</url>
460 <spider ident=
"geourl">
462 <url>http://geourl.org/bot.html
</url>
464 <spider ident=
"GetterroboPlus">
465 <name>GetterroboPlus Puu
</name>
467 <spider ident=
"GetURL.rexx">
470 <spider ident=
"Gigabot">
472 <url>http://www.gigablast.com/spider.html
</url>
474 <spider ident=
"Girafabot">
475 <name>Girafabot
</name>
476 <url>http://www.girafa.com/
</url>
478 <spider ident=
"Goku">
480 <url>http://goku.ru/bot.htm; bot@goku.ru
</url>
482 <spider ident=
"Golem">
485 <spider ident=
"gonzo">
488 <spider ident=
"Googlebot/">
490 <url>http://www.google.com/bot.html
</url>
492 <spider ident=
"Mediapartners-Google">
493 <name>Google AdSense
</name>
494 <url>https://www.google.com/adsense/faq
</url>
496 <spider ident=
"Googlebot-Image">
497 <name>Googlebot-Image
</name>
498 <url>http://www.googlebot.com/bot.html
</url>
500 <spider ident=
"Googlebot-Mobile">
501 <name>Googlebot-Mobile
</name>
502 <url>http://www.google.com/bot.html
</url>
504 <spider ident=
"GoogleStackdriverMonitoring-UptimeChecks">
505 <name>Google Stackdriver Monitoring
</name>
506 <url>https://cloud.google.com/monitoring/alerts/uptime-checks
</url>
508 <spider ident=
"Google-Ads-Creatives-Assistant">
509 <name>Google-Ads-Creatives-Assistant
</name>
511 <spider ident=
"Google-AdWords-Express">
512 <name>Google-AdWords-Express
</name>
514 <spider ident=
"AdsBot-Google">
515 <name>Google Ads-Bot
</name>
516 <url>http://www.google.com/adsbot.html
</url>
518 <spider ident=
"Gpostbot">
519 <name>Gpostbot
</name>
520 <url>http://www.gpost.info/help.php?c=bot
</url>
522 <spider ident=
"griffon">
525 <spider ident=
"Gromit">
528 <spider ident=
"http://grub.org">
529 <name>Grub Client
</name>
531 <spider ident=
"Gulper Web Bot">
532 <name>Gulper Bot
</name>
534 <spider ident=
"havIndex">
535 <name>havIndex
</name>
537 <spider ident=
"HeinrichderMiragoRobot">
538 <name>HeinrichderMiragoRobot
</name>
540 <spider ident=
"HenryTheMiragoRobot">
541 <name>HenryTheMiragoRobot
</name>
543 <spider ident=
"heritrix">
544 <name>Heritrix
</name>
545 <url>http://www.worio.com
</url>
547 <spider ident=
"HKU WWW Robot">
548 <name>HKU WWW Octopus
</name>
550 <spider ident=
"HolyCowDude">
551 <name>HolyCowDude
</name>
552 <url>http://www.holycowdude.com/spider.htm
</url>
554 <spider ident=
"HomeTags">
555 <name>HomeTags
</name>
556 <url>http://www.hometags.nl/bot
</url>
558 <spider ident=
"Hometown">
559 <name>Hometown
</name>
561 <spider ident=
"htdig">
562 <name>ht://Dig
</name>
564 <spider ident=
"AITCSRobot">
565 <name>HTML Index
</name>
567 <spider ident=
"HTMLgobble">
568 <name>HTMLgobble
</name>
570 <spider ident=
"I Robot">
571 <name>I, Robot
</name>
573 <spider ident=
"iajaBot">
576 <spider ident=
"IBM_Planetwide">
577 <name>IBM_Planetwide
</name>
579 <spider ident=
"+http://www.icerocket.com/">
580 <name>IceRocket
</name>
581 <url>http://www.icerocket.com/
</url>
583 <spider ident=
"ichiro">
586 <spider ident=
"IlTrovatore-Setaccio">
587 <name>IlTrovatore-Setaccio
</name>
588 <url>http://www.iltrovatore.it/aiuto/faq.html
</url>
590 <spider ident=
"image.kapsi.net">
591 <name>image.kapsi.net
</name>
593 <spider ident=
"Mozilla 3.01 PBWF (Win95)">
594 <name>Imagelock
</name>
596 <spider ident=
"IncyWincy">
597 <name>IncyWincy
</name>
599 <spider ident=
"Informant">
600 <name>Informant
</name>
602 <spider ident=
"InfoSeek Robot">
603 <name>InfoSeek Robot
1.0</name>
605 <spider ident=
"Infoseek Sidewinder">
606 <name>Infoseek Sidewinder
</name>
608 <spider ident=
"InfoSpiders">
609 <name>InfoSpiders
</name>
611 <spider ident=
"INGRID">
614 <spider ident=
"slurp@inktomi">
617 <spider ident=
"Insitor">
619 <url>http://www.insitor.de/
</url>
621 <spider ident=
"inspectorwww">
622 <name>Inspector Web
</name>
624 <spider ident=
"IAGENT">
625 <name>IntelliAgent
</name>
627 <spider ident=
"Intelliseek">
628 <name>Intelliseek
</name>
629 <url>http://www.intelliseek.com/
</url>
631 <spider ident=
"Internet Cruiser Robot">
632 <name>Internet Cruiser
</name>
634 <spider ident=
"internetseer">
635 <name>Internet Seer
</name>
637 <spider ident=
"sharp-info-agent">
638 <name>Internet Shinchakubin
</name>
640 <spider ident=
"InternetLinkAgent">
641 <name>InternetLinkAgent
</name>
643 <spider ident=
"IRLbot">
644 <name>IRL Crawler
</name>
645 <url>http://irl.cs.tamu.edu/crawler
</url>
647 <spider ident=
"Iron33">
650 <spider ident=
"IsraeliSearch">
651 <name>Israeli-search
</name>
653 <spider ident=
"itchBot">
656 <spider ident=
"JavaBee">
659 <spider ident=
"JBot">
662 <spider ident=
"JCrawler">
663 <name>JCrawler
</name>
665 <spider ident=
"JetBot">
667 <url>http://www.jeteye.com/jetbot.html
</url>
669 <spider ident=
"JoBo">
672 <spider ident=
"Jobot">
675 <spider ident=
"jobs.de">
677 <url>http://www.jobs.de/
</url>
679 <spider ident=
"JoeBot">
682 <spider ident=
"jumpstation">
683 <name>JumpStation
</name>
685 <spider ident=
"Katipo">
688 <spider ident=
"KDD-Explorer">
689 <name>KDD-Explorer
</name>
691 <spider ident=
"KIT-Fireball">
692 <name>KIT-Fireball
</name>
694 <spider ident=
"KO_Yappo_Robot">
695 <name>KO_Yappo_Robot
</name>
697 <spider ident=
"LabelGrab">
698 <name>LabelGrabber
</name>
700 <spider ident=
"larbin">
703 <spider ident=
"legs">
706 <spider ident=
"linkdexbot">
708 <url>http://www.linkdex.com/bots/
</url>
710 <spider ident=
"LinkScan Server">
711 <name>LinkScan
</name>
713 <spider ident=
"LinkWalker">
714 <name>LinkWalker
</name>
716 <spider ident=
"Linguee Bot">
718 <url>http://www.linguee.com/bot
</url>
720 <spider ident=
"livedoorCheckers/">
721 <name>livedoorCheckers
</name>
723 <spider ident=
"Lockon">
726 <spider ident=
"logo.gif crawler">
727 <name>logo.gif
</name>
729 <spider ident=
"Lycos">
732 <spider ident=
"Magpie">
735 <spider ident=
"MJ12bot">
736 <name>Majestics MJ12bot
</name>
738 <spider ident=
"Mammoth">
740 <url>http://www.sli-systems.com
</url>
742 <spider ident=
"Marvin">
745 <spider ident=
"marvin/infoseek">
746 <name>marvin/infoseek
</name>
748 <spider ident=
"M/3.8">
751 <spider ident=
"MediaFox">
752 <name>MediaFox
</name>
754 <spider ident=
"memorybot">
755 <name>Memorybot
</name>
756 <url>http://archivethe.net/en/index.php/about/internet_memory1
</url>
758 <spider ident=
"mercator">
759 <name>Mercator
</name>
760 <url>http://research.compaq.com/SRC/mercator/
</url>
762 <spider ident=
"MerzScope">
763 <name>MerzScope
</name>
765 <spider ident=
"METASpider">
767 <url>http://www.meta.com.ua/
</url>
769 <spider ident=
"MetaGer-LinkChecker">
772 <spider ident=
"MindCrawler">
773 <name>MindCrawler
</name>
775 <spider ident=
"Miva">
778 <spider ident=
"UdmSearch">
779 <name>mnoGoSearch
</name>
781 <spider ident=
"moget">
784 <spider ident=
"MOMspider">
785 <name>MOMspider
</name>
787 <spider ident=
"Monster">
790 <spider ident=
"Moreoverbot">
791 <name>Moreover
</name>
792 <url>http://www.moreover.com
</url>
794 <spider ident=
"msnbot">
796 <url>http://search.msn.com/msnbot.htm
</url>
798 <spider ident=
"MSRBOT">
800 <url>http://research.microsoft.com/research/sv/msrbot/
</url>
802 <spider ident=
"MuscatFerret">
803 <name>Muscat Ferret
</name>
805 <spider ident=
"MwdSearch">
806 <name>Mwd.Search
</name>
808 <spider ident=
"NPBot">
809 <name>NameProtect
</name>
811 <spider ident=
"NaverBot">
812 <name>NaverBot
</name>
813 <url>http://www.spidermatic.com/en/robot-spider/
20</url>
815 <spider ident=
"NDSpider">
816 <name>NDSpider
</name>
818 <spider ident=
"NEC-MeshExplorer">
819 <name>NEC-MeshExplorer
</name>
821 <spider ident=
"Nederland.zoek">
822 <name>Nederland.zoek
</name>
824 <spider ident=
"NerdyBot">
825 <name>NerdyBot
</name>
826 <url>http://nerdybot.com/
</url>
828 <spider ident=
"NetCarta CyberPilot Pro">
829 <name>NetCarta WebMap
</name>
831 <spider ident=
"Netcraft">
832 <name>Netcraft Web Server Survey
</name>
833 <url>http://news.netcraft.com/
</url>
835 <spider ident=
"NetMechanic">
836 <name>NetMechanic
</name>
838 <spider ident=
"NetScoop">
839 <name>NetScoop
</name>
841 <spider ident=
"newscan-online">
842 <name>newscan-online
</name>
844 <spider ident=
"NextGenSearchBot 1">
845 <name>NextGenSearchBot
</name>
846 <url>http://www.zoominfo.com/NextGenSearchBot
</url>
848 <spider ident=
"NHSEWalker">
849 <name>NHSE Web Forager
</name>
853 <url>http://www.newsisfree.com/robot.php
</url>
855 <spider ident=
"NimbleCrawler">
856 <name>NimbleCrawler
</name>
857 <url>http://www.healthline.com/aboutus.jsp
</url>
859 <spider ident=
"Nomad">
862 <spider ident=
"Norbert the Spider">
864 <url>http://www.Burf.com
</url>
866 <spider ident=
"Gulliver">
867 <name>Northern Light
</name>
869 <spider ident=
"explorersearch">
870 <name>nzexplorer
</name>
872 <spider ident=
"Occam">
875 <spider ident=
"Ocelli">
877 <url>http://www.globalspec.com/Ocelli
</url>
879 <spider ident=
"Online24-Bot">
880 <name>Online24-Bot
</name>
882 <spider ident=
"Openbot">
884 <url>http://www.openfind.com.tw/robot.html
</url>
886 <spider ident=
"Openfind">
887 <name>Openfind data gatherer
</name>
889 <spider ident=
"Orbsearch">
890 <name>Orb Search
</name>
892 <spider ident=
"PackRat">
893 <name>Pack Rat
</name>
895 <spider ident=
"PageBoy">
898 <spider ident=
"Pandalytics">
899 <name>Pandalytics
</name>
900 <url>https://domainsbot.com/pandalytics/
</url>
902 <spider ident=
"ParaSite">
903 <name>ParaSite
</name>
905 <spider ident=
"Patric">
908 <spider ident=
"PEGASUS">
911 <spider ident=
"PerlCrawler/1.0 Xavatoria/2.0">
912 <name>PerlCrawler
1.0</name>
914 <spider ident=
"PGP-KA">
915 <name>PGP Key Agent
</name>
917 <spider ident=
"Duppies">
920 <spider ident=
"phpdig">
923 <spider ident=
"PiltdownMan">
924 <name>PiltdownMan
</name>
926 <spider ident=
"Pimptrain's robot">
927 <name>Pimptrain.com's
</name>
929 <spider ident=
"pingalink">
930 <name>PingALink
</name>
932 <spider ident=
"Pioneer">
935 <spider ident=
"PluckFeedCrawler">
937 <url>http://www.pluck.com
</url>
939 <spider ident=
"PlumtreeWebAccessor">
940 <name>PlumtreeWebAccessor
</name>
942 <spider ident=
"PodNova">
944 <url>http://www.podnova.com
</url>
946 <spider ident=
"Pompos">
948 <url>http://dir.com/pompos.html
</url>
950 <spider ident=
"Poppi">
953 <spider ident=
"publiclibraryarchive.org">
954 <name>publiclibraryarchive.org
</name>
956 <spider ident=
"gestaltIconoclast">
957 <name>Popular Iconoclast
</name>
959 <spider ident=
"PortalJuice.com">
960 <name>Portal Juice
</name>
962 <spider ident=
"PortalBSpider">
963 <name>PortalB Spider
</name>
965 <spider ident=
"Qualidator">
966 <name>Qualidator
</name>
967 <url>www.qualidator.com/Web/de/Support/FAQ_OnlineTestStatistiken.htm
</url>
969 <spider ident=
"www.kolinka.com">
970 <name>Project Kolinka Forum Search
</name>
971 <url>http://www.kolinka.com/
</url>
973 <spider ident=
"psbot">
976 <spider ident=
"Qango.com Web Directory">
978 <url>http://www.qango.com
</url>
980 <spider ident=
"SBSearch">
981 <name>SecretSearchEngineLabs.com
</name>
982 <url>http://www.secretsearchenginelabs.com/secret-web-crawler.php
</url>
984 <spider ident=
"SemrushBot">
985 <name>SemrushBot
</name>
986 <url>http://semrush.com/bot/
</url>
988 <spider ident=
"StackRambler">
990 <url>http://www.rambler.ru/
</url>
992 <spider ident=
"Raven">
993 <name>Raven Search
</name>
995 <spider ident=
"Resume Robot">
996 <name>Resume Robot
</name>
998 <spider ident=
"Road Runner: ImageScape Robot">
999 <name>Road Runner: The ImageScape Robot
</name>
1001 <spider ident=
"RHCS">
1002 <name>RoadHouse Crawling System
</name>
1004 <spider ident=
"Robbie">
1005 <name>Robbie the Robot
</name>
1007 <spider ident=
"RoboCrawl">
1008 <name>RoboCrawl
</name>
1010 <spider ident=
"Robofox">
1011 <name>RoboFox
</name>
1013 <spider ident=
"Robot du CRIM 1.0a">
1014 <name>Robot Francoroute
</name>
1016 <spider ident=
"Robozilla">
1017 <name>Robozilla
</name>
1019 <spider ident=
"Roverbot">
1020 <name>Roverbot
</name>
1022 <spider ident=
"RSS-SPIDER">
1023 <name>RSS Feed Seeker
</name>
1024 <url>http://www.rss-spider.com/fsb.php
</url>
1026 <spider ident=
"RuLeS">
1029 <spider ident=
"RyzeCrawler">
1030 <name>RyzeCrawler
</name>
1031 <url>http://www.domain2day.nl/crawler/
</url>
1033 <spider ident=
"SafetyNet Robot">
1034 <name>SafetyNet
</name>
1036 <spider ident=
"SBIder">
1037 <name>SBIder.
</name>
1038 <url>http://www.sitesell.com/sbider.html
</url>
1040 <spider ident=
"Scharia">
1041 <name>Scharia
</name>
1043 <spider ident=
"Science-Index">
1044 <name>Science-Index
</name>
1046 <spider ident=
"Scooter">
1047 <name>Scooter
</name>
1049 <spider ident=
"SearchNZ">
1050 <name>SearchNZ
</name>
1051 <url>http://www.searchnz.co.nz/
</url>
1053 <spider ident=
"searchprocess">
1054 <name>SearchProcess
</name>
1056 <spider ident=
"SearchmetricsBot">
1057 <name>SearchmetricsBot
</name>
1058 <url>http://www.searchmetrics.com/en/searchmetrics-bot/
</url>
1060 <spider ident=
"Seekbot">
1061 <name>Seekbot
</name>
1062 <url>http://www.seekbot.net/bot.html
</url>
1064 <spider ident=
"Senrigan">
1065 <name>Senrigan
</name>
1067 <spider ident=
"Sensis Web Crawler">
1068 <name>Sensis Web Crawler
</name>
1069 <url>http://www.sensis.com.au/help.do
</url>
1071 <spider ident=
"SentiBot">
1072 <name>SentiBot
</name>
1073 <url>http://www.sentibot.eu
</url>
1075 <spider ident=
"SeznamBot/3.2">
1076 <name>Seznam Bot
</name>
1077 <url>http://napoveda.seznam.cz/en/seznambot-intro/
</url>
1079 <spider ident=
"SG-Scout">
1080 <name>SG-Scout
</name>
1082 <spider ident=
"Shagseeker">
1083 <name>ShagSeeker
</name>
1085 <spider ident=
"Shai'Hulud">
1086 <name>Shai'Hulud
</name>
1088 <spider ident=
"SimBot/1.0">
1089 <name>Simmany Robot Ver1.0
</name>
1091 <spider ident=
"SkypeUriPreview">
1092 <name>Skype Preview
</name>
1093 <url>https://www.skype.com/
</url>
1095 <spider ident=
"ssearcher100">
1096 <name>Site Searcher
</name>
1098 <spider ident=
"Site Valet">
1099 <name>Site Valet
</name>
1101 <spider ident=
"http://www.site-list.net">
1102 <name>Site-List
</name>
1103 <url>http://www.site-list.net
</url>
1105 <spider ident=
"SiteTech-Rover">
1106 <name>SiteTech-Rover
</name>
1108 <spider ident=
"+SitiDi.net/SitiDiBot/">
1109 <name>SitiDi.net/SitiDiBot
</name>
1111 <spider ident=
"aWapClient">
1112 <name>Skymob.com
</name>
1114 <spider ident=
"Slack">
1115 <name>Slackbot
</name>
1116 <url>https://api.slack.com/robots
</url>
1118 <spider ident=
"SLCrawler">
1119 <name>SLCrawler
</name>
1121 <spider ident=
"Sleek Spider">
1124 <spider ident=
"ESISmartSpider">
1125 <name>Smart Spider
</name>
1127 <spider ident=
"Snapbot">
1128 <name>Snapbot
</name>
1129 <url>http://www.snap.com/
</url>
1131 <spider ident=
"Snooper">
1132 <name>Snooper
</name>
1134 <spider ident=
"sohu-search">
1135 <name>sohu-search
</name>
1137 <spider ident=
"Solbot">
1140 <spider ident=
"Speedy Spider">
1141 <name>Speedy Spider
</name>
1142 <url>http://www.entireweb.com/about/search_tech/speedyspider/
</url>
1144 <spider ident=
"Sphere Scout">
1147 <spider ident=
"Sphider2">
1148 <name>Sphider
</name>
1150 <spider ident=
"SpiderBot">
1151 <name>SpiderBot
</name>
1153 <spider ident=
"spiderline">
1154 <name>Spiderline Crawler
</name>
1156 <spider ident=
"SpiderMan">
1157 <name>SpiderMan
</name>
1159 <spider ident=
"SpiderView">
1160 <name>SpiderView(tm)
</name>
1162 <spider ident=
"mouse.house">
1163 <name>spider_monkey
</name>
1165 <spider ident=
"suke">
1168 <spider ident=
"suntek">
1169 <name>suntek search engine
</name>
1171 <spider ident=
"Szukacz">
1172 <name>Szukacz
</name>
1173 <url>http://www.szukacz.pl/html/RobotEnglishVersion.html
</url>
1175 <spider ident=
"T-H-U-N-D-E-R-S-T-O-N-E">
1176 <name>T-H-U-N-D-E-R-S-T-O-N-E
</name>
1178 <spider ident=
"TinEye Crawler">
1180 <url>http://tineye.com/crawler.html
</url>
1182 <spider ident=
"Black Widow">
1183 <name>TACH Black Widow
</name>
1185 <spider ident=
"Tapatalk CloudSearch">
1186 <name>Tapatalk CloudSearch
</name>
1188 <spider ident=
"Tarantula">
1189 <name>Tarantula
</name>
1191 <spider ident=
"tarspider">
1192 <name>tarspider
</name>
1194 <spider ident=
"dlw3robot">
1195 <name>Tcl W3 Robot
</name>
1197 <spider ident=
"TechBOT">
1198 <name>TechBOT
</name>
1200 <spider ident=
"Technoratibot">
1201 <name>Technorati
</name>
1202 <url>http://technorati.com/about/
</url>
1204 <spider ident=
"Templeton">
1205 <name>Templeton
</name>
1207 <spider ident=
"teoma">
1208 <name>Teoma/Ask Jeeves
</name>
1209 <url>http://sp.teoma.com/docs/teoma/about/
</url>
1211 <spider ident=
"JubiiRobot">
1212 <name>The Jubii
</name>
1214 <spider ident=
"NorthStar">
1215 <name>The NorthStar Robot
</name>
1217 <spider ident=
"w3index">
1218 <name>The NWI Robot
</name>
1220 <spider ident=
"Peregrinator-Mathematics">
1221 <name>The Peregrinator
</name>
1223 <spider ident=
"Pixray-Seeker">
1225 <url>http://www.pixray.com/pixraybot/
</url>
1227 <spider ident=
"TelegramBot (like TwitterBot)">
1228 <name>TelegramBot (like TwitterBot)
</name>
1229 <url>https://telegram.org/
</url>
1231 <spider ident=
"thumbshots-de-Bot">
1232 <name>thumbshots-de-Bot
</name>
1234 <spider ident=
"TITAN">
1237 <spider ident=
"TitIn">
1240 <spider ident=
"TLSpider">
1241 <name>TLSpider
</name>
1243 <spider ident=
"TMCrawler">
1244 <name>TMCrawler
</name>
1246 <spider ident=
"trendictionbot">
1247 <name>Trendiction-Bot
</name>
1248 <url>http://www.trendiction.com/bot
</url>
1250 <spider ident=
"slysearch">
1251 <name>Turnitin.com
</name>
1252 <url>http://www.turnitin.com/static/products_services/search_engines.html
</url>
1254 <spider ident=
"TurnitinBot/">
1255 <name>TurnitinBot
</name>
1257 <spider ident=
"TurtleScanner">
1259 <url>http://www.turtle.ru/
</url>
1261 <spider ident=
"TwengaBot">
1263 <url>http://www.twenga.com/bot.html
</url>
1265 <spider ident=
"Twiceler">
1266 <name>Twiceler
</name>
1267 <url>http://www.cuill.com/twiceler/robot.html
</url>
1269 <spider ident=
"Twitterbot">
1270 <name>Twitterbot
</name>
1271 <url>https://twitter.com/
</url>
1273 <spider ident=
"UCSD-Crawler">
1274 <name>UCSD Crawl
</name>
1276 <spider ident=
"UMBC-memeta-Bot">
1279 <spider ident=
"unisterbot">
1280 <name>Unister
</name>
1282 <spider ident=
"Unpartisan">
1283 <name>Unpartisan
</name>
1284 <url>http://www.unpartisan.com
</url>
1286 <spider ident=
"UptimeRobot/2.0">
1287 <name>Uptime Robot
</name>
1288 <url>http://uptimerobot.com/
</url>
1290 <spider ident=
"urlck">
1291 <name>URL Check
</name>
1293 <spider ident=
"URL Spider Pro">
1294 <name>URL Spider Pro
</name>
1296 <spider ident=
"Valkyrie">
1297 <name>Valkyrie
</name>
1299 <spider ident=
"Verticrawl">
1300 <name>Verticrawl
</name>
1302 <spider ident=
"Victoria">
1303 <name>Victoria
</name>
1305 <spider ident=
"vision-search">
1306 <name>vision-search
</name>
1308 <spider ident=
"VoilaBot">
1309 <name>VoilaBot
</name>
1310 <url>http://www.voila.com/
</url>
1312 <spider ident=
"VisBot">
1314 <url>http://www.visvo.com/webmasters.html
</url>
1316 <spider ident=
"Voyager">
1317 <name>Voyager
</name>
1319 <spider ident=
"VWbot_K">
1322 <spider ident=
"W3M2">
1325 <spider ident=
"w3mir">
1328 <spider ident=
"w@pSpider">
1329 <name>w@pSpider
</name>
1331 <spider ident=
"appie">
1332 <name>Walhello appie
</name>
1333 <url>http://www.robotstxt.org/wc/active/html/appie.html
</url>
1335 <spider ident=
"CrawlPaper">
1336 <name>WallPaper
</name>
1338 <spider ident=
"root">
1339 <name>Web Core / Roots
</name>
1341 <spider ident=
"WBSearchBot">
1342 <name>Ware Bay
</name>
1343 <url>http://www.warebay.com/bot.html
</url>
1345 <spider ident=
"WebMoose">
1346 <name>Web Moose
</name>
1348 <spider ident=
"WebBandit">
1349 <name>WebBandit
</name>
1351 <spider ident=
"WebCatcher">
1352 <name>WebCatcher
</name>
1354 <spider ident=
"Webclipping">
1355 <name>Webclipping
</name>
1357 <spider ident=
"WebCopy">
1358 <name>WebCopy
</name>
1360 <spider ident=
"WebFetcher">
1361 <name>webfetcher
</name>
1363 <spider ident=
"weblayers">
1364 <name>weblayers
</name>
1366 <spider ident=
"WebLinker">
1367 <name>WebLinker
</name>
1369 <spider ident=
"wlm">
1370 <name>Weblog Monitor
</name>
1372 <spider ident=
"WebQuest">
1373 <name>WebQuest
</name>
1375 <spider ident=
"WebReaper">
1376 <name>WebReaper
</name>
1378 <spider ident=
"webs@recruit.co.jp">
1381 <spider ident=
"websearchbench">
1382 <name>WebSearchBench
</name>
1383 <url>http://websearchbench.cs.uni-dortmund.de/
</url>
1385 <spider ident=
"WOLP">
1386 <name>WebStolperer
</name>
1388 <spider ident=
"webvac">
1391 <spider ident=
"webwalk">
1392 <name>webwalk
</name>
1394 <spider ident=
"WebWalker">
1395 <name>WebWalker
</name>
1397 <spider ident=
"WebWatch">
1398 <name>WebWatch
</name>
1400 <spider ident=
"WebZinger">
1401 <name>WebZinger
</name>
1403 <spider ident=
"whatUseek_winona">
1404 <name>whatUseek Winona
</name>
1406 <spider ident=
"WhoWhere Robot">
1407 <name>WebWatch
</name>
1408 <url>http://www.whowhere.com
</url>
1410 <spider ident=
"SurveyBot">
1411 <name>Whois Source
</name>
1412 <url>http://www.whois.sc/info/webmasters/surveybot.html
</url>
1414 <spider ident=
"Hazel's Ferret Web hopper">
1415 <name>Wild Ferret Web Hopper
</name>
1417 <spider ident=
"HRCrawler">
1418 <name>HRCrawler
</name>
1420 <spider ident=
"WinHTTP">
1421 <name>WinHTTP
</name>
1423 <spider ident=
"wired-digital-newsbot">
1424 <name>Wired Digital
</name>
1426 <spider ident=
"zyborg">
1427 <name>WiseNut
</name>
1429 <spider ident=
"OmniExplorer_Bot">
1430 <name>WorldIndexer
</name>
1431 <url>http://www.omni-explorer.com
</url>
1433 <spider ident=
"WWWC">
1436 <spider ident=
"WWWeasel Robot">
1437 <name>WWWeasel Robot
</name>
1439 <spider ident=
"wwwster">
1440 <name>wwwster
</name>
1442 <spider ident=
"WWWWanderer">
1443 <name>WWWWanderer
</name>
1445 <spider ident=
"TECOMAC-Crawler">
1446 <name>X-Crawler
</name>
1448 <spider ident=
"XGET">
1451 <spider ident=
"cosmos">
1452 <name>XYLEME Robot
</name>
1454 <spider ident=
"yacybot">
1455 <name>YaCy-Bot
</name>
1456 <url>https://yacy.net/bot.html
</url>
1458 <spider ident=
"YahooYSMcm">
1459 <name>Yahoo Publisher Network
</name>
1460 <url>http://publisher.yahoo.com/
</url>
1462 <spider ident=
"Yahoo-Blogs">
1463 <name>Yahoo-Blogs
</name>
1464 <url>http://help.yahoo.com/help/us/ysearch/crawling/crawling-
02.html
</url>
1466 <spider ident=
"Yahoo Pipes">
1467 <name>Yahoo Pipes
</name>
1469 <spider ident=
"Yahoo! Slurp">
1470 <name>Yahoo! Slurp
</name>
1471 <url>http://help.yahoo.com/help/us/ysearch/slurp
</url>
1473 <spider ident=
"Yahoo-VerticalCrawler">
1474 <name>Yahoo-VerticalCrawler
</name>
1476 <spider ident=
"YahooFeedSeeker">
1477 <name>YahooFeedSeeker
</name>
1478 <url>http://my.yahoo.com/s/publishers.html
</url>
1480 <spider ident=
"Yandex">
1482 <url>http://www.yandex.ru/
</url>
1484 <spider ident=
"zeus">
1485 <name>Zeus Internet Marketing
</name>
1486 <url>http://www.cyber-robotics.com/
</url>
1488 <spider ident=
"http://www.zorkk.com">
1490 <url>http://www.zorkk.com
</url>
1492 <spider ident=
"Zookabot">
1493 <name>Zookabotk
</name>
1494 <url>http://zookabot.com/
</url>
1496 <spider ident=
"360Spider">
1497 <name>360Spider
</name>