projects / GitHub / WoltLab / WCF.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Update url for DuckDuckBot, Facebook and Heritrix
[GitHub/WoltLab/WCF.git] / spiderList / spiderList.xml
CommitLineData
25ab2d04 1<?xml version="1.0" encoding="UTF-8"?>
3c9bb073 2<data xmlns="http://www.woltlab.com" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.woltlab.com http://www.woltlab.com/XSD/5.4/spiderList.xsd">
25ab2d04
MW		 3 <spider ident="ABCdatos">
4 <name>ABCdatos BotLink</name>
665cc30d 5 <url>http://www.robotstxt.org/db/abcdatos.html</url>
25ab2d04 6 </spider>
9581146e
PM		 7 <spider ident="AbraveSpider">
8 <name>Abrave</name>
9 <url>http://robot.abrave.co.uk/</url>
10 </spider>
25ab2d04
MW		 11 <spider ident="abot/">
12 <name>abot</name>
25ab2d04
MW		 13 </spider>
14 <spider ident="Accelatech RSSCrawler">
15 <name>Accelatech</name>
16 </spider>
17 <spider ident="Accoona-AI-Agent">
18 <name>Accoona</name>
19 <url>http://accoona.com/about/about_accoona.jsp</url>
20 </spider>
21 <spider ident="aconon Index">
22 <name>aconon Index (raubfische.de)</name>
23 </spider>
24 <spider ident="AcoonBot">
25 <name>Acoon</name>
26 <url>http://www.acoon.de/robot.asp</url>
27 </spider>
28 <spider ident="Ahoy!">
29 <name>Ahoy!</name>
665cc30d 30 <url>http://www.robotstxt.org/db/ahoythehomepagefinder.html</url>
25ab2d04
MW		 31 </spider>
32 <spider ident="AhrefsBot">
33 <name>Ahrefs</name>
34 <url>http://ahrefs.com/robot/</url>
35 </spider>
36 <spider ident="AlkalineBOT">
37 <name>Alkaline</name>
665cc30d 38 <url>http://www.robotstxt.org/db/Alkaline.html</url>
25ab2d04 39 </spider>
f55aa1aa
P		 40 <spider ident="AlexaBOT">
41 <name>Alexa</name>
42 <url>http://www.alexa.com/help/webmasters</url>
43 </spider>
25ab2d04
MW		 44 <spider ident="http://www.almaden.ibm.com/cs/crawler">
45 <name>Almaden Crawler</name>
46 <url>http://www.almaden.ibm.com/cs/crawler/</url>
47 </spider>
2af2b30d
I		 48 <spider ident="Barkrowler">
49 <name>Barkrowler</name>
50 <url>https://www.babbar.tech/crawler</url>
51 </spider>
25ab2d04
MW		 52 <spider ident="EMC Spider">
53 <name>ananzi</name>
54 </spider>
55 <spider ident="Anthill">
56 <name>Anthill</name>
665cc30d 57 <url>http://www.robotstxt.org/db/anthill.html</url>
25ab2d04
MW		 58 </spider>
59 <spider ident="Aport">
60 <name>Aport</name>
61 <url>http://www.aport.ru/</url>
62 </spider>
63 <spider ident="AppleSyndication">
64 <name>Apple</name>
65 </spider>
6aa6c9ba 66 <spider ident="Applebot">
67 <name>Applebot</name>
68 <url>https://support.apple.com/en-gb/HT204683</url>
69 </spider>
25ab2d04
MW		 70 <spider ident="Arachnophilia">
71 <name>Arachnophilia</name>
665cc30d 72 <url>http://www.robotstxt.org/db/arachnophilia.html</url>
25ab2d04
MW		 73 </spider>
74 <spider ident="Araneo">
75 <name>Araneo</name>
665cc30d 76 <url>http://www.robotstxt.org/db/araneo.html</url>
25ab2d04
MW		 77 </spider>
78 <spider ident="ArchitextSpider">
79 <name>ArchitextSpider</name>
665cc30d 80 <url>http://www.robotstxt.org/db/architext.html</url>
25ab2d04 81 </spider>
89b52a1a 82 <spider ident="archive.org_bot">
25ab2d04 83 <name>Archive.org</name>
89b52a1a 84 <url>https://archive.org/details/archive.org_bot</url>
25ab2d04
MW		 85 </spider>
86 <spider ident="arks/1.0">
87 <name>arks</name>
665cc30d 88 <url>http://www.robotstxt.org/db/arks.html</url>
25ab2d04
MW		 89 </spider>
90 <spider ident="ASpider">
91 <name>ASpider</name>
665cc30d 92 <url>http://www.robotstxt.org/db/aspider.html</url>
25ab2d04
MW		 93 </spider>
94 <spider ident="ATN_Worldwide">
95 <name>ATN Worldwide</name>
665cc30d 96 <url>http://www.robotstxt.org/db/atn.html</url>
25ab2d04
MW		 97 </spider>
98 <spider ident="Atomz">
99 <name>Atomz.com</name>
665cc30d 100 <url>http://www.robotstxt.org/db/atomz.html</url>
25ab2d04
MW		 101 </spider>
102 <spider ident="AURESYS">
103 <name>AURESYS</name>
665cc30d 104 <url>http://www.robotstxt.org/db/auresys.html</url>
25ab2d04
MW		 105 </spider>
106 <spider ident="BackRub">
107 <name>BackRub</name>
665cc30d 108 <url>http://www.robotstxt.org/db/backrub.html</url>
25ab2d04
MW		 109 </spider>
110 <spider ident="Baiduspider">
111 <name>Baiduspider</name>
112 <url>http://www.baidu.com/search/spider.htm</url>
113 </spider>
25ab2d04
MW		 114 <spider ident="BecomeBot">
115 <name>BecomeBot</name>
116 <url>http://www.become.com/webmasters.html</url>
117 </spider>
118 <spider ident="Big Brother">
119 <name>Big Brother</name>
665cc30d 120 <url>http://www.robotstxt.org/db/bigbrother.html</url>
25ab2d04
MW		 121 </spider>
122 <spider ident="BigmirSpider">
123 <name>Bigmir</name>
124 <url>http://www.bigmir.net/</url>
125 </spider>
126 <spider ident="bingbot">
127 <name>Bing</name>
128 <url>http://www.bing.com/bingbot.htm</url>
129 </spider>
130 <spider ident="Bitacle bot">
131 <name>Bitacle</name>
132 <url>http://bitacle.org/</url>
133 </spider>
134 <spider ident="Biz360 Spider">
135 <name>Biz</name>
136 <url>http://www.biz360.com</url>
137 </spider>
138 <spider ident="Bjaaland">
139 <name>Bjaaland</name>
665cc30d 140 <url>http://www.robotstxt.org/db/bjaaland.html</url>
25ab2d04
MW		 141 </spider>
142 <spider ident="BlackWidow">
143 <name>BlackWidow</name>
665cc30d 144 <url>http://www.robotstxt.org/db/blackwidow.html</url>
25ab2d04 145 </spider>
b730b757
MW		 146 <spider ident="BLEXBot">
147 <name>BLEXBot</name>
148 <url>http://webmeup.com/crawler.html</url>
149 </spider>
25ab2d04
MW		 150 <spider ident="BlogCrawler by Xango">
151 <name>BlogCrawler</name>
152 </spider>
153 <spider ident="blogdb">
154 <name>BlogDb</name>
155 <url>http://blogdb.jp</url>
156 </spider>
157 <spider ident="blog search engine by BlogFan.ORG">
158 <name>BlogFan</name>
159 <url>http://www.blogfan.org</url>
160 </spider>
161 <spider ident="Bloglines">
162 <name>Bloglies</name>
163 <url>http://www.bloglines.com</url>
164 </spider>
165 <spider ident="BlogPulse (ISSpider-3.0)">
166 <name>BlogPulse</name>
167 </spider>
168 <spider ident="BlogSearch">
169 <name>BlogSearch</name>
170 <url>http://www.icerocket.com</url>
171 </spider>
172 <spider ident="BlogsNowBot">
173 <name>BlogsNow</name>
174 <url>http://www.blogsnow.com/</url>
175 </spider>
176 <spider ident="BlogStreetBot">
177 <name>BlogStreetBot</name>
178 <url>http://www.blogstreet.com/</url>
179 </spider>
180 <spider ident="Bulkfeeds">
181 <name>BlogStreetBot</name>
182 <url>http://bulkfeeds.net</url>
183 </spider>
184 <spider ident="BoardPulse">
185 <name>BoardPulse</name>
186 <url>http://www.boardpulse.com/</url>
187 </spider>
188 <spider ident="BoardReader">
189 <name>BoardReader</name>
190 <url>http://www.boardreader.com/aboutus.asp</url>
191 </spider>
192 <spider ident="BoardViewer">
193 <name>BoardViewer</name>
194 <url>http://www.boardviewer.com/</url>
195 </spider>
196 <spider ident="boitho.com-robot">
197 <name>Boitho</name>
198 <url>http://www.boitho.com/bot.html</url>
199 </spider>
200 <spider ident="borg-bot">
201 <name>Borg-Bot</name>
665cc30d 202 <url>http://www.robotstxt.org/db/borg-bot.html</url>
25ab2d04
MW		 203 </spider>
204 <spider ident="BSpider">
205 <name>BSpider</name>
665cc30d 206 <url>http://www.robotstxt.org/db/bspider.html</url>
25ab2d04 207 </spider>
1b4bb70a
SM		 208 <spider ident="Buck/2.2">
209 <name>Buck</name>
210 <url>https://app.hypefactors.com/media-monitoring/about.html</url>
211 </spider>
25ab2d04
MW		 212 <spider ident="CACTVS Chemistry Spider">
213 <name>CACTVS Chemistry</name>
214 </spider>
215 <spider ident="Calif">
216 <name>Calif</name>
217 </spider>
218 <spider ident="CaRP/3.6Evolution">
219 <name>CaRP</name>
220 <url>http://www.biz360.com</url>
221 </spider>
222 <spider ident="Checkbot">
223 <name>Checkbot</name>
224 </spider>
225 <spider ident="ChristCrawler.com">
226 <name>ChristCrawler.com</name>
227 </spider>
228 <spider ident="www.cienciaficcion.net">
229 <name>cIeNcIaFiCcIoN.nEt</name>
230 </spider>
231 <spider ident="CipinetBot">
232 <name>Cipinet</name>
233 <url>http://www.cipinet.com/bot.html</url>
234 </spider>
235 <spider ident="CJNetworkQuality">
236 <name>CJNetworkQuality</name>
237 <url>http://www.cj.com/networkquality/</url>
238 </spider>
239 <spider ident="CMC/0.01">
240 <name>CMC/0.01</name>
241 </spider>
242 <spider ident="ColdFusion">
243 <name>ColdFusion</name>
244 </spider>
245 <spider ident="combine">
246 <name>Combine System</name>
247 </spider>
248 <spider ident="Crawler (cometsearch@cometsystems.com)">
249 <name>cometsystems.com</name>
250 </spider>
251 <spider ident="ComputingSite Robi/1.0">
252 <name>ComputingSite Robi/1.0</name>
253 </spider>
254 <spider ident="conceptbot">
255 <name>Conceptbot</name>
256 </spider>
cbfe141b
I		 257 <spider ident="Cookiebot">
258 <name>Cookiebot</name>
259 <url>https://www.cookiebot.com/</url>
260 </spider>
25ab2d04
MW		 261 <spider ident="Cooby.de Crawler">
262 <name>Cooby.de Crawler</name>
263 </spider>
264 <spider ident="CoolBot">
265 <name>CoolBot</name>
266 </spider>
267 <spider ident="Cusco">
268 <name>Cusco</name>
269 </spider>
270 <spider ident="CyberSpyder">
271 <name>CyberSpyder</name>
272 </spider>
273 <spider ident="daypopbot">
274 <name>daypop</name>
275 </spider>
276 <spider ident="DesertRealm.com">
277 <name>Desert Realm</name>
278 </spider>
279 <spider ident="Deweb">
280 <name>DeWeb(c)</name>
281 </spider>
282 <spider ident="Die Blinde Kuh">
283 <name>Die Blinde Kuh</name>
665cc30d 284 <url>http://www.robotstxt.org/db/blindekuh.html</url>
25ab2d04
MW		 285 </spider>
286 <spider ident="dienstspider">
287 <name>DienstSpider</name>
288 </spider>
289 <spider ident="Digger/1.0 JDK/1.3.0">
290 <name>Digger</name>
291 </spider>
292 <spider ident="Digimarc WebReader">
293 <name>Digimarc MarcSpider</name>
294 </spider>
295 <spider ident="Digimarc CGIReader">
296 <name>Digimarc Marcspider/CGI</name>
297 </spider>
298 <spider ident="DIIbot">
299 <name>Digital Integrity Robot</name>
300 </spider>
301 <spider ident="grabber">
302 <name>Direct Hit Grabber</name>
303 </spider>
304 <spider ident="discobot">
305 <name>Discovery</name>
306 <url>http://discoveryengine.com/discobot.html</url>
307 </spider>
a370eb7d 308 <spider ident="Discordbot">
309 <name>Discord</name>
310 <url>https://discordapp.com/</url>
311 </spider>
25ab2d04
MW		 312 <spider ident="DNAbot/1.0">
313 <name>DNAbot</name>
314 </spider>
f51b8b91 315 <spider ident="DotBot">
fdaca223
C		 316 <name>Moz DotBot</name>
317 <url>http://www.opensiteexplorer.org/dotbot</url>
318 </spider>
25ab2d04
MW		 319 <spider ident="DragonBot/1.0 libwww/5.0">
320 <name>DragonBot</name>
321 </spider>
a370eb7d 322 <spider ident="DuckDuckBot">
323 <name>DuckDuckGo</name>
05e2373e 324 <url>https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/</url>
a370eb7d 325 </spider>
25ab2d04
MW		 326 <spider ident="DWCP/2.0">
327 <name>DWCP (Dridus' Web Cataloging Project)</name>
328 </spider>
329 <spider ident="e-SocietyRobot">
330 <name>e-Society</name>
331 <url>http://www.yama.info.waseda.ac.jp/~yamana/es/index_eng.htm</url>
332 </spider>
333 <spider ident="exactseek-pagereaper">
334 <name>eaxactseek-page</name>
335 </spider>
336 <spider ident="EbiNess/0.01a">
337 <name>EbiNess</name>
338 </spider>
339 <spider ident="edgeio-retriever">
340 <name>Edgeio</name>
341 <url>http://www.edgeio.com</url>
342 </spider>
343 <spider ident="EIT-Link-Verifier-Robot/0.2">
344 <name>EIT Link Verifier Robot</name>
345 </spider>
346 <spider ident="elfinbot">
347 <name>ELFINBOT</name>
348 </spider>
349 <spider ident="Emacs-w3/v[0-9\.]+">
350 <name>Emacs-w3 Search Engine</name>
351 </spider>
352 <spider ident="esther">
353 <name>Esther</name>
354 </spider>
355 <spider ident="EuripBot/">
356 <name>EuripBot</name>
357 </spider>
30de0ef8
I		 358 <spider ident="ev-crawler">
359 <name>Headline Web Crawler</name>
360 <url>https://headline.com/legal/crawler</url>
361 </spider>
25ab2d04
MW		 362 <spider ident="Evliya Celebi">
363 <name>Evliya Celebi</name>
364 </spider>
365 <spider ident="ExactSeek_Spider">
366 <name>ExactSeek_Spider</name>
367 <url>http://www.askjeevs.com</url>
368 </spider>
369 <spider ident="NG/2.0">
370 <name>ExaLead</name>
371 <url>http://botspotter.net/bs-389.html</url>
372 </spider>
373 <spider ident="ExaBot">
374 <name>ExaLead Beta</name>
375 <url>http://beta.exalead.com/search/C=0/2p=Help.7</url>
376 </spider>
f55aa1aa
P		 377 <spider ident="ExaLead">
378 <name>ExaLead</name>
379 <url>http://www.exalead.com/search</url>
380 </spider>
25ab2d04
MW		 381 <spider ident="facebookexternalhit">
382 <name>Facebook</name>
05e2373e 383 <url>https://developers.facebook.com/docs/sharing/webmasters/crawler/</url>
25ab2d04
MW		 384 </spider>
385 <spider ident="fast-webcrawler">
386 <name>FAST / AlltheWeb</name>
387 <url>http://help.yahoo.com/help/us/ysearch/slurp/index.html</url>
388 </spider>
389 <spider ident="FastCrawler">
390 <name>FastCrawler</name>
391 </spider>
392 <spider ident="Feed24.com">
393 <name>Feed24</name>
394 <url>http://www.feed24.com</url>
395 </spider>
396 <spider ident="FeedBlitz">
397 <name>FeedBlitz</name>
398 <url>http://www.feedblitz.com</url>
399 </spider>
400 <spider ident="FeedBurner">
401 <name>FeedBurner</name>
402 <url>http://www.FeedBurner.com</url>
403 </spider>
7a944c3f
MW		 404 <spider ident="Feedly">
405 <name>Feedly</name>
406 <url>http://www.feedly.com/fetcher.html</url>
407 </spider>
25ab2d04
MW		 408 <spider ident="Feedfetcher-Google">
409 <name>FeedFetcher-Google</name>
410 <url>http://www.google.com/feedfetcher.html</url>
411 </spider>
412 <spider ident="UniversalFeedParser">
413 <name>FeedParser</name>
414 <url>http://www.feedparser.org</url>
415 </spider>
416 <spider ident="Feedster Crawler">
417 <name>Feedster</name>
418 <url>http://www.feedstermedia.com/</url>
419 </spider>
420 <spider ident="FEHLSTART Superspider">
421 <name>FEHLSTART</name>
422 </spider>
423 <spider ident="FelixIDE">
424 <name>Felix IDE</name>
425 </spider>
426 <spider ident="ESIRover">
427 <name>FetchRover</name>
428 </spider>
429 <spider ident="fido">
430 <name>fido</name>
431 </spider>
432 <spider ident="findlinks">
433 <name>FindLinks</name>
434 <url>http://wortschatz.uni-leipzig.de/findlinks/</url>
435 </spider>
436 <spider ident="FindoryBot">
437 <name>Findroy</name>
438 <url>http://www.findory.com</url>
439 </spider>
440 <spider ident="Fish-Search-Robot">
441 <name>Fish search</name>
442 </spider>
443 <spider ident="Mozilla/4.0 (compatible: FDSE robot)">
444 <name>Fluid Dynamics</name>
445 </spider>
446 <spider ident="fouineur.9bit.qc.ca">
447 <name>Fouineur</name>
448 </spider>
449 <spider ident="Freecrawl">
450 <name>Freecrawl</name>
451 </spider>
2f86459f
SM		 452 <spider ident="FreshpingBot">
453 <name>Freshping</name>
454 <url>https://freshping.io/</url>
455 </spider>
25ab2d04
MW		 456 <spider ident="FunnelWeb">
457 <name>FunnelWeb</name>
458 </spider>
459 <spider ident="GaisBot">
460 <name>Gais</name>
461 <url>http://gais.cs.ccu.edu.tw/robot.php</url>
462 </spider>
463 <spider ident="gamekitbot">
464 <name>GAMEKIT</name>
465 <url>http://www.uchoose.de/crawler/gamekitbot/</url>
466 </spider>
467 <spider ident="gammaSpider">
468 <name>gammaSpider</name>
469 </spider>
470 <spider ident="gazz">
471 <name>gazz</name>
472 </spider>
473 <spider ident="gcreep">
474 <name>GCreep</name>
475 </spider>
476 <spider ident="genieBot">
477 <name>genieBot</name>
478 <url>http://64.5.245.11/faq/faq.html</url>
479 </spider>
480 <spider ident="geourl">
481 <name>GeoURL</name>
482 <url>http://geourl.org/bot.html</url>
483 </spider>
484 <spider ident="GetterroboPlus">
485 <name>GetterroboPlus Puu</name>
486 </spider>
487 <spider ident="GetURL.rexx">
488 <name>GetURL</name>
489 </spider>
490 <spider ident="Gigabot">
491 <name>Gigabot</name>
492 <url>http://www.gigablast.com/spider.html</url>
493 </spider>
494 <spider ident="Girafabot">
495 <name>Girafabot</name>
496 <url>http://www.girafa.com/</url>
497 </spider>
498 <spider ident="Goku">
499 <name>Goku</name>
500 <url>http://goku.ru/bot.htm; bot@goku.ru</url>
501 </spider>
502 <spider ident="Golem">
503 <name>Golem</name>
504 </spider>
505 <spider ident="gonzo">
506 <name>Gonzo</name>
507 </spider>
508 <spider ident="Googlebot/">
509 <name>Google</name>
510 <url>http://www.google.com/bot.html</url>
511 </spider>
512 <spider ident="Mediapartners-Google">
513 <name>Google AdSense</name>
514 <url>https://www.google.com/adsense/faq</url>
515 </spider>
516 <spider ident="Googlebot-Image">
517 <name>Googlebot-Image</name>
518 <url>http://www.googlebot.com/bot.html</url>
519 </spider>
520 <spider ident="Googlebot-Mobile">
521 <name>Googlebot-Mobile</name>
522 <url>http://www.google.com/bot.html</url>
523 </spider>
2f4ab6f1 524 <spider ident="Googlebot-Video/1.0">
525 <name>Googlebot-Video</name>
5ef610cb 526 <url>https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers</url>
2f4ab6f1 527 </spider>
5c30f4c0
AE		 528 <spider ident="GoogleStackdriverMonitoring-UptimeChecks">
529 <name>Google Stackdriver Monitoring</name>
530 <url>https://cloud.google.com/monitoring/alerts/uptime-checks</url>
531 </spider>
fdaca223
C		 532 <spider ident="Google-Ads-Creatives-Assistant">
533 <name>Google-Ads-Creatives-Assistant</name>
534 </spider>
535 <spider ident="Google-AdWords-Express">
536 <name>Google-AdWords-Express</name>
537 </spider>
538 <spider ident="AdsBot-Google">
539 <name>Google Ads-Bot</name>
540 <url>http://www.google.com/adsbot.html</url>
541 </spider>
25ab2d04
MW		 542 <spider ident="Gpostbot">
543 <name>Gpostbot</name>
544 <url>http://www.gpost.info/help.php?c=bot</url>
545 </spider>
546 <spider ident="griffon">
547 <name>Griffon</name>
548 </spider>
549 <spider ident="Gromit">
550 <name>Gromit</name>
551 </spider>
552 <spider ident="http://grub.org">
553 <name>Grub Client</name>
554 </spider>
555 <spider ident="Gulper Web Bot">
556 <name>Gulper Bot</name>
557 </spider>
558 <spider ident="havIndex">
559 <name>havIndex</name>
560 </spider>
561 <spider ident="HeinrichderMiragoRobot">
562 <name>HeinrichderMiragoRobot</name>
563 </spider>
564 <spider ident="HenryTheMiragoRobot">
565 <name>HenryTheMiragoRobot</name>
566 </spider>
567 <spider ident="heritrix">
568 <name>Heritrix</name>
05e2373e 569 <url>https://github.com/internetarchive/heritrix3/wiki</url>
25ab2d04
MW		 570 </spider>
571 <spider ident="HKU WWW Robot">
572 <name>HKU WWW Octopus</name>
573 </spider>
574 <spider ident="HolyCowDude">
575 <name>HolyCowDude</name>
576 <url>http://www.holycowdude.com/spider.htm</url>
577 </spider>
9581146e
PM		 578 <spider ident="HomeTags">
579 <name>HomeTags</name>
580 <url>http://www.hometags.nl/bot</url>
581 </spider>
25ab2d04
MW		 582 <spider ident="Hometown">
583 <name>Hometown</name>
584 </spider>
585 <spider ident="htdig">
586 <name>ht://Dig</name>
587 </spider>
588 <spider ident="AITCSRobot">
589 <name>HTML Index</name>
590 </spider>
591 <spider ident="HTMLgobble">
592 <name>HTMLgobble</name>
593 </spider>
594 <spider ident="I Robot">
595 <name>I, Robot</name>
596 </spider>
597 <spider ident="iajaBot">
598 <name>iajaBot</name>
599 </spider>
600 <spider ident="IBM_Planetwide">
601 <name>IBM_Planetwide</name>
602 </spider>
603 <spider ident="+http://www.icerocket.com/">
604 <name>IceRocket</name>
605 <url>http://www.icerocket.com/</url>
606 </spider>
607 <spider ident="ichiro">
608 <name>ichiro</name>
609 </spider>
610 <spider ident="IlTrovatore-Setaccio">
611 <name>IlTrovatore-Setaccio</name>
612 <url>http://www.iltrovatore.it/aiuto/faq.html</url>
613 </spider>
614 <spider ident="image.kapsi.net">
615 <name>image.kapsi.net</name>
616 </spider>
617 <spider ident="Mozilla 3.01 PBWF (Win95)">
618 <name>Imagelock</name>
619 </spider>
620 <spider ident="IncyWincy">
621 <name>IncyWincy</name>
622 </spider>
623 <spider ident="Informant">
624 <name>Informant</name>
625 </spider>
626 <spider ident="InfoSeek Robot">
627 <name>InfoSeek Robot 1.0</name>
628 </spider>
629 <spider ident="Infoseek Sidewinder">
630 <name>Infoseek Sidewinder</name>
631 </spider>
632 <spider ident="InfoSpiders">
633 <name>InfoSpiders</name>
634 </spider>
635 <spider ident="INGRID">
636 <name>Ingrid</name>
637 </spider>
638 <spider ident="slurp@inktomi">
639 <name>Inktomi</name>
640 </spider>
641 <spider ident="Insitor">
642 <name>Insitor</name>
643 <url>http://www.insitor.de/</url>
644 </spider>
645 <spider ident="inspectorwww">
646 <name>Inspector Web</name>
647 </spider>
648 <spider ident="IAGENT">
649 <name>IntelliAgent</name>
650 </spider>
651 <spider ident="Intelliseek">
652 <name>Intelliseek</name>
653 <url>http://www.intelliseek.com/</url>
654 </spider>
655 <spider ident="Internet Cruiser Robot">
656 <name>Internet Cruiser</name>
657 </spider>
658 <spider ident="internetseer">
659 <name>Internet Seer</name>
660 </spider>
661 <spider ident="sharp-info-agent">
662 <name>Internet Shinchakubin</name>
663 </spider>
664 <spider ident="InternetLinkAgent">
665 <name>InternetLinkAgent</name>
666 </spider>
667 <spider ident="IRLbot">
668 <name>IRL Crawler</name>
669 <url>http://irl.cs.tamu.edu/crawler</url>
670 </spider>
671 <spider ident="Iron33">
672 <name>Iron33</name>
673 </spider>
674 <spider ident="IsraeliSearch">
675 <name>Israeli-search</name>
676 </spider>
677 <spider ident="itchBot">
678 <name>itch</name>
25ab2d04
MW		 679 </spider>
680 <spider ident="JavaBee">
681 <name>JavaBee</name>
682 </spider>
683 <spider ident="JBot">
684 <name>JBot</name>
685 </spider>
686 <spider ident="JCrawler">
687 <name>JCrawler</name>
688 </spider>
689 <spider ident="JetBot">
690 <name>JetEye</name>
691 <url>http://www.jeteye.com/jetbot.html</url>
692 </spider>
693 <spider ident="JoBo">
694 <name>JoBo</name>
695 </spider>
696 <spider ident="Jobot">
697 <name>Jobot</name>
698 </spider>
699 <spider ident="jobs.de">
700 <name>Jobs.de</name>
701 <url>http://www.jobs.de/</url>
702 </spider>
703 <spider ident="JoeBot">
704 <name>JoeBot</name>
705 </spider>
706 <spider ident="jumpstation">
707 <name>JumpStation</name>
708 </spider>
709 <spider ident="Katipo">
710 <name>Katipo</name>
711 </spider>
712 <spider ident="KDD-Explorer">
713 <name>KDD-Explorer</name>
714 </spider>
715 <spider ident="KIT-Fireball">
716 <name>KIT-Fireball</name>
717 </spider>
718 <spider ident="KO_Yappo_Robot">
719 <name>KO_Yappo_Robot</name>
720 </spider>
721 <spider ident="LabelGrab">
722 <name>LabelGrabber</name>
723 </spider>
724 <spider ident="larbin">
725 <name>larbin</name>
726 </spider>
727 <spider ident="legs">
728 <name>legs</name>
729 </spider>
0cf0a5c1
C		 730 <spider ident="linkdexbot">
731 <name>Linkdex</name>
732 <url>http://www.linkdex.com/bots/</url>
733 </spider>
25ab2d04
MW		 734 <spider ident="LinkScan Server">
735 <name>LinkScan</name>
736 </spider>
737 <spider ident="LinkWalker">
738 <name>LinkWalker</name>
739 </spider>
f55aa1aa
P		 740 <spider ident="Linguee Bot">
741 <name>Linguee</name>
742 <url>http://www.linguee.com/bot</url>
743 </spider>
25ab2d04
MW		 744 <spider ident="livedoorCheckers/">
745 <name>livedoorCheckers</name>
746 </spider>
747 <spider ident="Lockon">
748 <name>Lockon</name>
749 </spider>
750 <spider ident="logo.gif crawler">
751 <name>logo.gif</name>
752 </spider>
753 <spider ident="Lycos">
754 <name>Lycos</name>
755 </spider>
756 <spider ident="Magpie">
757 <name>Magpie</name>
758 </spider>
759 <spider ident="MJ12bot">
760 <name>Majestics MJ12bot</name>
761 </spider>
762 <spider ident="Mammoth">
763 <name>Mammoth</name>
764 <url>http://www.sli-systems.com</url>
765 </spider>
766 <spider ident="Marvin">
767 <name>Marvin</name>
768 </spider>
769 <spider ident="marvin/infoseek">
770 <name>marvin/infoseek</name>
771 </spider>
772 <spider ident="M/3.8">
773 <name>Mattie</name>
774 </spider>
775 <spider ident="MediaFox">
776 <name>MediaFox</name>
777 </spider>
d9cc31f0
SG		 778 <spider ident="memorybot">
779 <name>Memorybot</name>
780 <url>http://archivethe.net/en/index.php/about/internet_memory1</url>
781 </spider>
25ab2d04
MW		 782 <spider ident="mercator">
783 <name>Mercator</name>
784 <url>http://research.compaq.com/SRC/mercator/</url>
785 </spider>
786 <spider ident="MerzScope">
787 <name>MerzScope</name>
788 </spider>
789 <spider ident="METASpider">
790 <name>META</name>
791 <url>http://www.meta.com.ua/</url>
792 </spider>
793 <spider ident="MetaGer-LinkChecker">
794 <name>MetaGer</name>
795 </spider>
796 <spider ident="MindCrawler">
797 <name>MindCrawler</name>
798 </spider>
799 <spider ident="Miva">
800 <name>Miva</name>
801 </spider>
802 <spider ident="UdmSearch">
803 <name>mnoGoSearch</name>
804 </spider>
805 <spider ident="moget">
806 <name>moget</name>
807 </spider>
808 <spider ident="MOMspider">
809 <name>MOMspider</name>
810 </spider>
811 <spider ident="Monster">
812 <name>Monster</name>
813 </spider>
814 <spider ident="Moreoverbot">
815 <name>Moreover</name>
816 <url>http://www.moreover.com</url>
817 </spider>
818 <spider ident="msnbot">
819 <name>MSNBot</name>
820 <url>http://search.msn.com/msnbot.htm</url>
821 </spider>
822 <spider ident="MSRBOT">
823 <name>MSRBOT</name>
824 <url>http://research.microsoft.com/research/sv/msrbot/</url>
825 </spider>
826 <spider ident="MuscatFerret">
827 <name>Muscat Ferret</name>
828 </spider>
829 <spider ident="MwdSearch">
830 <name>Mwd.Search</name>
831 </spider>
832 <spider ident="NPBot">
833 <name>NameProtect</name>
834 </spider>
835 <spider ident="NaverBot">
836 <name>NaverBot</name>
837 <url>http://www.spidermatic.com/en/robot-spider/20</url>
52bb0ca4 838 </spider>
6a3fbb36 839 <spider ident="NDSpider">
52bb0ca4 840 <name>NDSpider</name>
25ab2d04
MW		 841 </spider>
842 <spider ident="NEC-MeshExplorer">
843 <name>NEC-MeshExplorer</name>
844 </spider>
845 <spider ident="Nederland.zoek">
846 <name>Nederland.zoek</name>
847 </spider>
89b52a1a
MW		 848 <spider ident="NerdyBot">
849 <name>NerdyBot</name>
850 <url>http://nerdybot.com/</url>
851 </spider>
25ab2d04
MW		 852 <spider ident="NetCarta CyberPilot Pro">
853 <name>NetCarta WebMap</name>
854 </spider>
855 <spider ident="Netcraft">
856 <name>Netcraft Web Server Survey</name>
857 <url>http://news.netcraft.com/</url>
858 </spider>
859 <spider ident="NetMechanic">
860 <name>NetMechanic</name>
861 </spider>
862 <spider ident="NetScoop">
863 <name>NetScoop</name>
864 </spider>
865 <spider ident="newscan-online">
866 <name>newscan-online</name>
867 </spider>
868 <spider ident="NextGenSearchBot 1">
869 <name>NextGenSearchBot</name>
870 <url>http://www.zoominfo.com/NextGenSearchBot</url>
871 </spider>
872 <spider ident="NHSEWalker">
873 <name>NHSE Web Forager</name>
874 </spider>
875 <spider ident="NIF">
876 <name>NIF</name>
d04e330e 877 <url>http://www.newsisfree.com/robot.php</url>
25ab2d04
MW		 878 </spider>
879 <spider ident="NimbleCrawler">
880 <name>NimbleCrawler</name>
881 <url>http://www.healthline.com/aboutus.jsp</url>
882 </spider>
883 <spider ident="Nomad">
884 <name>Nomad</name>
885 </spider>
886 <spider ident="Norbert the Spider">
887 <name>Norbert</name>
888 <url>http://www.Burf.com</url>
889 </spider>
890 <spider ident="Gulliver">
891 <name>Northern Light</name>
892 </spider>
893 <spider ident="explorersearch">
894 <name>nzexplorer</name>
895 </spider>
896 <spider ident="Occam">
897 <name>Occam</name>
898 </spider>
899 <spider ident="Ocelli">
900 <name>Ocelli</name>
901 <url>http://www.globalspec.com/Ocelli</url>
902 </spider>
903 <spider ident="Online24-Bot">
904 <name>Online24-Bot</name>
905 </spider>
906 <spider ident="Openbot">
907 <name>Openbot</name>
908 <url>http://www.openfind.com.tw/robot.html</url>
909 </spider>
910 <spider ident="Openfind">
911 <name>Openfind data gatherer</name>
912 </spider>
913 <spider ident="Orbsearch">
914 <name>Orb Search</name>
915 </spider>
916 <spider ident="PackRat">
917 <name>Pack Rat</name>
918 </spider>
919 <spider ident="PageBoy">
920 <name>PageBoy</name>
921 </spider>
5657c6b8
N		 922 <spider ident="Pandalytics">
923 <name>Pandalytics</name>
924 <url>https://domainsbot.com/pandalytics/</url>
925 </spider>
25ab2d04
MW		 926 <spider ident="ParaSite">
927 <name>ParaSite</name>
928 </spider>
929 <spider ident="Patric">
930 <name>Patric</name>
931 </spider>
932 <spider ident="PEGASUS">
933 <name>pegasus</name>
934 </spider>
935 <spider ident="PerlCrawler/1.0 Xavatoria/2.0">
936 <name>PerlCrawler 1.0</name>
937 </spider>
6ca2411e 938 <spider ident="PetalBot">
939 <name>PetalBot</name>
940 <url>https://aspiegel.com/petalbot</url>
941 </spider>
25ab2d04
MW		 942 <spider ident="PGP-KA">
943 <name>PGP Key Agent</name>
944 </spider>
945 <spider ident="Duppies">
946 <name>Phantom</name>
947 </spider>
948 <spider ident="phpdig">
949 <name>PhpDig</name>
950 </spider>
951 <spider ident="PiltdownMan">
952 <name>PiltdownMan</name>
953 </spider>
954 <spider ident="Pimptrain's robot">
955 <name>Pimptrain.com's</name>
956 </spider>
957 <spider ident="pingalink">
958 <name>PingALink</name>
959 </spider>
960 <spider ident="Pioneer">
961 <name>Pioneer</name>
962 </spider>
963 <spider ident="PluckFeedCrawler">
964 <name>Pluck</name>
965 <url>http://www.pluck.com</url>
966 </spider>
967 <spider ident="PlumtreeWebAccessor">
968 <name>PlumtreeWebAccessor</name>
969 </spider>
970 <spider ident="PodNova">
971 <name>PodNova</name>
972 <url>http://www.podnova.com</url>
973 </spider>
974 <spider ident="Pompos">
975 <name>Pompos</name>
976 <url>http://dir.com/pompos.html</url>
977 </spider>
978 <spider ident="Poppi">
979 <name>Poppi</name>
980 </spider>
0cf0a5c1
C		 981 <spider ident="publiclibraryarchive.org">
982 <name>publiclibraryarchive.org</name>
983 </spider>
25ab2d04
MW		 984 <spider ident="gestaltIconoclast">
985 <name>Popular Iconoclast</name>
986 </spider>
987 <spider ident="PortalJuice.com">
988 <name>Portal Juice</name>
989 </spider>
990 <spider ident="PortalBSpider">
991 <name>PortalB Spider</name>
992 </spider>
f55aa1aa
P		 993 <spider ident="Qualidator">
994 <name>Qualidator</name>
995 <url>www.qualidator.com/Web/de/Support/FAQ_OnlineTestStatistiken.htm</url>
996 </spider>
25ab2d04
MW		 997 <spider ident="www.kolinka.com">
998 <name>Project Kolinka Forum Search</name>
999 <url>http://www.kolinka.com/</url>
1000 </spider>
1001 <spider ident="psbot">
1002 <name>psbot</name>
1003 </spider>
1004 <spider ident="Qango.com Web Directory">
1005 <name>Qango</name>
1006 <url>http://www.qango.com</url>
1007 </spider>
71a53c20
SM		 1008 <spider ident="Qwant">
1009 <name>Qwant</name>
1010 <url>https://help.qwant.com/bot/</url>
1011 </spider>
f55aa1aa
P		 1012 <spider ident="SBSearch">
1013 <name>SecretSearchEngineLabs.com</name>
1014 <url>http://www.secretsearchenginelabs.com/secret-web-crawler.php</url>
1015 </spider>
307ef0a2 1016 <spider ident="SemrushBot">
1017 <name>SemrushBot</name>
1018 <url>http://semrush.com/bot/</url>
1019 </spider>
25ab2d04
MW		 1020 <spider ident="StackRambler">
1021 <name>Rambler</name>
1022 <url>http://www.rambler.ru/</url>
1023 </spider>
1024 <spider ident="Raven">
1025 <name>Raven Search</name>
1026 </spider>
1027 <spider ident="Resume Robot">
1028 <name>Resume Robot</name>
1029 </spider>
1030 <spider ident="Road Runner: ImageScape Robot">
1031 <name>Road Runner: The ImageScape Robot</name>
1032 </spider>
1033 <spider ident="RHCS">
1034 <name>RoadHouse Crawling System</name>
1035 </spider>
1036 <spider ident="Robbie">
1037 <name>Robbie the Robot</name>
1038 </spider>
1039 <spider ident="RoboCrawl">
1040 <name>RoboCrawl</name>
1041 </spider>
1042 <spider ident="Robofox">
1043 <name>RoboFox</name>
1044 </spider>
1045 <spider ident="Robot du CRIM 1.0a">
1046 <name>Robot Francoroute</name>
1047 </spider>
1048 <spider ident="Robozilla">
1049 <name>Robozilla</name>
1050 </spider>
1051 <spider ident="Roverbot">
1052 <name>Roverbot</name>
1053 </spider>
1054 <spider ident="RSS-SPIDER">
1055 <name>RSS Feed Seeker</name>
1056 <url>http://www.rss-spider.com/fsb.php</url>
1057 </spider>
1058 <spider ident="RuLeS">
1059 <name>RuLeS</name>
1060 </spider>
9581146e
PM		 1061 <spider ident="RyzeCrawler">
1062 <name>RyzeCrawler</name>
1063 <url>http://www.domain2day.nl/crawler/</url>
1064 </spider>
25ab2d04
MW		 1065 <spider ident="SafetyNet Robot">
1066 <name>SafetyNet</name>
1067 </spider>
1068 <spider ident="SBIder">
1069 <name>SBIder.</name>
1070 <url>http://www.sitesell.com/sbider.html</url>
1071 </spider>
1072 <spider ident="Scharia">
1073 <name>Scharia</name>
1074 </spider>
1075 <spider ident="Science-Index">
1076 <name>Science-Index</name>
1077 </spider>
1078 <spider ident="Scooter">
1079 <name>Scooter</name>
1080 </spider>
1081 <spider ident="SearchNZ">
1082 <name>SearchNZ</name>
1083 <url>http://www.searchnz.co.nz/</url>
1084 </spider>
1085 <spider ident="searchprocess">
1086 <name>SearchProcess</name>
1087 </spider>
1088 <spider ident="SearchmetricsBot">
1089 <name>SearchmetricsBot</name>
1090 <url>http://www.searchmetrics.com/en/searchmetrics-bot/</url>
1091 </spider>
1092 <spider ident="Seekbot">
1093 <name>Seekbot</name>
1094 <url>http://www.seekbot.net/bot.html</url>
1095 </spider>
4cba12bf
I		 1096 <spider ident="SeekportBot">
1097 <name>Seekport Bot</name>
1098 <url>https://bot.seekport.com</url>
1099 </spider>
25ab2d04
MW		 1100 <spider ident="Senrigan">
1101 <name>Senrigan</name>
1102 </spider>
1103 <spider ident="Sensis Web Crawler">
1104 <name>Sensis Web Crawler</name>
1105 <url>http://www.sensis.com.au/help.do</url>
1106 </spider>
16240335 1107 <spider ident="SentiBot">
4c8c4953 1108 <name>SentiBot</name>
16240335
C		 1109 <url>http://www.sentibot.eu</url>
1110 </spider>
548371c0
I		 1111 <spider ident="SEO Scanner">
1112 <name>SEO Scanner</name>
1113 </spider>
abafa2ec
SM		 1114 <spider ident="SEOkicks">
1115 <name>SEOkicks</name>
1116 <url>https://www.seokicks.de/robot.html</url>
1117 </spider>
08bf1baa
SM		 1118 <spider ident="serpstatbot">
1119 <name>serpstatbot</name>
1120 <url>http://serpstatbot.com/</url>
1121 </spider>
a2b39a59
C		 1122 <spider ident="SeznamBot/3.2">
1123 <name>Seznam Bot</name>
1124 <url>http://napoveda.seznam.cz/en/seznambot-intro/</url>
1125 </spider>
25ab2d04
MW		 1126 <spider ident="SG-Scout">
1127 <name>SG-Scout</name>
1128 </spider>
1129 <spider ident="Shagseeker">
1130 <name>ShagSeeker</name>
1131 </spider>
1132 <spider ident="Shai'Hulud">
1133 <name>Shai'Hulud</name>
1134 </spider>
1135 <spider ident="SimBot/1.0">
1136 <name>Simmany Robot Ver1.0</name>
1137 </spider>
a370eb7d 1138 <spider ident="SkypeUriPreview">
1139 <name>Skype Preview</name>
1140 <url>https://www.skype.com/</url>
1141 </spider>
25ab2d04
MW		 1142 <spider ident="ssearcher100">
1143 <name>Site Searcher</name>
1144 </spider>
1145 <spider ident="Site Valet">
1146 <name>Site Valet</name>
1147 </spider>
1148 <spider ident="http://www.site-list.net">
1149 <name>Site-List</name>
1150 <url>http://www.site-list.net</url>
1151 </spider>
1152 <spider ident="SiteTech-Rover">
1153 <name>SiteTech-Rover</name>
1154 </spider>
1155 <spider ident="+SitiDi.net/SitiDiBot/">
1156 <name>SitiDi.net/SitiDiBot</name>
1157 </spider>
1158 <spider ident="aWapClient">
1159 <name>Skymob.com</name>
1160 </spider>
677899c2
AE		 1161 <spider ident="Slack">
1162 <name>Slackbot</name>
1163 <url>https://api.slack.com/robots</url>
1164 </spider>
25ab2d04
MW		 1165 <spider ident="SLCrawler">
1166 <name>SLCrawler</name>
1167 </spider>
1168 <spider ident="Sleek Spider">
1169 <name>Sleek</name>
1170 </spider>
1171 <spider ident="ESISmartSpider">
1172 <name>Smart Spider</name>
1173 </spider>
1174 <spider ident="Snapbot">
1175 <name>Snapbot</name>
1176 <url>http://www.snap.com/</url>
1177 </spider>
1178 <spider ident="Snooper">
1179 <name>Snooper</name>
1180 </spider>
1181 <spider ident="sohu-search">
1182 <name>sohu-search</name>
1183 </spider>
1184 <spider ident="Solbot">
1185 <name>Solbot</name>
1186 </spider>
1187 <spider ident="Speedy Spider">
1188 <name>Speedy Spider</name>
1189 <url>http://www.entireweb.com/about/search_tech/speedyspider/</url>
1190 </spider>
1191 <spider ident="Sphere Scout">
1192 <name>Sphere</name>
1193 </spider>
1194 <spider ident="Sphider2">
1195 <name>Sphider</name>
1196 </spider>
1197 <spider ident="SpiderBot">
1198 <name>SpiderBot</name>
1199 </spider>
1200 <spider ident="spiderline">
1201 <name>Spiderline Crawler</name>
1202 </spider>
1203 <spider ident="SpiderMan">
1204 <name>SpiderMan</name>
1205 </spider>
1206 <spider ident="SpiderView">
1207 <name>SpiderView(tm)</name>
1208 </spider>
1209 <spider ident="mouse.house">
1210 <name>spider_monkey</name>
1211 </spider>
1212 <spider ident="suke">
1213 <name>Suke</name>
1214 </spider>
1215 <spider ident="suntek">
1216 <name>suntek search engine</name>
1217 </spider>
1218 <spider ident="Szukacz">
1219 <name>Szukacz</name>
1220 <url>http://www.szukacz.pl/html/RobotEnglishVersion.html</url>
1221 </spider>
1222 <spider ident="T-H-U-N-D-E-R-S-T-O-N-E">
1223 <name>T-H-U-N-D-E-R-S-T-O-N-E</name>
1224 </spider>
f55aa1aa
P		 1225 <spider ident="TinEye Crawler">
1226 <name>TinEye</name>
1227 <url>http://tineye.com/crawler.html</url>
1228 </spider>
25ab2d04
MW		 1229 <spider ident="Black Widow">
1230 <name>TACH Black Widow</name>
1231 </spider>
e7d8bd51 1232 <spider ident="Tapatalk CloudSearch">
0cf0a5c1
C		 1233 <name>Tapatalk CloudSearch</name>
1234 </spider>
25ab2d04
MW		 1235 <spider ident="Tarantula">
1236 <name>Tarantula</name>
1237 </spider>
1238 <spider ident="tarspider">
1239 <name>tarspider</name>
1240 </spider>
1241 <spider ident="dlw3robot">
1242 <name>Tcl W3 Robot</name>
1243 </spider>
1244 <spider ident="TechBOT">
1245 <name>TechBOT</name>
1246 </spider>
1247 <spider ident="Technoratibot">
1248 <name>Technorati</name>
1249 <url>http://technorati.com/about/</url>
1250 </spider>
1251 <spider ident="Templeton">
1252 <name>Templeton</name>
1253 </spider>
1254 <spider ident="teoma">
1255 <name>Teoma/Ask Jeeves</name>
1256 <url>http://sp.teoma.com/docs/teoma/about/</url>
1257 </spider>
1258 <spider ident="JubiiRobot">
1259 <name>The Jubii</name>
1260 </spider>
1261 <spider ident="NorthStar">
1262 <name>The NorthStar Robot</name>
1263 </spider>
1264 <spider ident="w3index">
1265 <name>The NWI Robot</name>
1266 </spider>
1267 <spider ident="Peregrinator-Mathematics">
1268 <name>The Peregrinator</name>
1269 </spider>
0cf0a5c1
C		 1270 <spider ident="Pixray-Seeker">
1271 <name>Pixray</name>
1272 <url>http://www.pixray.com/pixraybot/</url>
1273 </spider>
a370eb7d 1274 <spider ident="TelegramBot (like TwitterBot)">
1275 <name>TelegramBot (like TwitterBot)</name>
1276 <url>https://telegram.org/</url>
1277 </spider>
25ab2d04
MW		 1278 <spider ident="thumbshots-de-Bot">
1279 <name>thumbshots-de-Bot</name>
1280 </spider>
1281 <spider ident="TITAN">
1282 <name>TITAN</name>
1283 </spider>
1284 <spider ident="TitIn">
1285 <name>TitIn</name>
1286 </spider>
1287 <spider ident="TLSpider">
1288 <name>TLSpider</name>
1289 </spider>
1290 <spider ident="TMCrawler">
1291 <name>TMCrawler</name>
1292 </spider>
1293 <spider ident="trendictionbot">
1294 <name>Trendiction-Bot</name>
1295 <url>http://www.trendiction.com/bot</url>
1296 </spider>
1297 <spider ident="slysearch">
1298 <name>Turnitin.com</name>
1299 <url>http://www.turnitin.com/static/products_services/search_engines.html</url>
1300 </spider>
1301 <spider ident="TurnitinBot/">
1302 <name>TurnitinBot</name>
1303 </spider>
1304 <spider ident="TurtleScanner">
1305 <name>Turtle</name>
1306 <url>http://www.turtle.ru/</url>
1307 </spider>
9581146e
PM		 1308 <spider ident="TwengaBot">
1309 <name>Twenga</name>
1310 <url>http://www.twenga.com/bot.html</url>
1311 </spider>
25ab2d04
MW		 1312 <spider ident="Twiceler">
1313 <name>Twiceler</name>
1314 <url>http://www.cuill.com/twiceler/robot.html</url>
1315 </spider>
a370eb7d 1316 <spider ident="Twitterbot">
1317 <name>Twitterbot</name>
1318 <url>https://twitter.com/</url>
1319 </spider>
25ab2d04
MW		 1320 <spider ident="UCSD-Crawler">
1321 <name>UCSD Crawl</name>
1322 </spider>
1323 <spider ident="UMBC-memeta-Bot">
1324 <name>UMBC</name>
1325 </spider>
1326 <spider ident="unisterbot">
1327 <name>Unister</name>
1328 </spider>
1329 <spider ident="Unpartisan">
1330 <name>Unpartisan</name>
1331 <url>http://www.unpartisan.com</url>
1332 </spider>
4465d786
I		 1333 <spider ident="Uptime-Kuma">
1334 <name>Uptime-Kuma</name>
1335 <url>https://uptime.kuma.pet/</url>
1336 </spider>
a602e267
SK		 1337 <spider ident="UptimeRobot/2.0">
1338 <name>Uptime Robot</name>
1339 <url>http://uptimerobot.com/</url>
1340 </spider>
25ab2d04
MW		 1341 <spider ident="urlck">
1342 <name>URL Check</name>
1343 </spider>
1344 <spider ident="URL Spider Pro">
1345 <name>URL Spider Pro</name>
1346 </spider>
1347 <spider ident="Valkyrie">
1348 <name>Valkyrie</name>
1349 </spider>
37b2fc3c
I		 1350 <spider ident="VelenPublicWebCrawler">
1351 <name>Velen Crawler</name>
1352 <url>https://velen.io</url>
1353 </spider>
25ab2d04
MW		 1354 <spider ident="Verticrawl">
1355 <name>Verticrawl</name>
1356 </spider>
1357 <spider ident="Victoria">
1358 <name>Victoria</name>
1359 </spider>
1360 <spider ident="vision-search">
1361 <name>vision-search</name>
1362 </spider>
1363 <spider ident="VoilaBot">
1364 <name>VoilaBot</name>
1365 <url>http://www.voila.com/</url>
1366 </spider>
9581146e
PM		 1367 <spider ident="VisBot">
1368 <name>VisBot</name>
1369 <url>http://www.visvo.com/webmasters.html</url>
1370 </spider>
25ab2d04
MW		 1371 <spider ident="Voyager">
1372 <name>Voyager</name>
1373 </spider>
1374 <spider ident="VWbot_K">
1375 <name>VWbot</name>
1376 </spider>
1377 <spider ident="W3M2">
1378 <name>W3M2</name>
1379 </spider>
1380 <spider ident="w3mir">
1381 <name>w3mir</name>
1382 </spider>
1383 <spider ident="w@pSpider">
1384 <name>w@pSpider</name>
1385 </spider>
1386 <spider ident="appie">
1387 <name>Walhello appie</name>
665cc30d 1388 <url>http://www.robotstxt.org/db/appie.html</url>
25ab2d04
MW		 1389 </spider>
1390 <spider ident="CrawlPaper">
1391 <name>WallPaper</name>
1392 </spider>
1393 <spider ident="root">
1394 <name>Web Core / Roots</name>
1395 </spider>
f55aa1aa
P		 1396 <spider ident="WBSearchBot">
1397 <name>Ware Bay</name>
1398 <url>http://www.warebay.com/bot.html</url>
1399 </spider>
25ab2d04
MW		 1400 <spider ident="WebMoose">
1401 <name>Web Moose</name>
1402 </spider>
1403 <spider ident="WebBandit">
1404 <name>WebBandit</name>
1405 </spider>
1406 <spider ident="WebCatcher">
1407 <name>WebCatcher</name>
1408 </spider>
1409 <spider ident="Webclipping">
1410 <name>Webclipping</name>
1411 </spider>
1412 <spider ident="WebCopy">
1413 <name>WebCopy</name>
1414 </spider>
1415 <spider ident="WebFetcher">
1416 <name>webfetcher</name>
1417 </spider>
1418 <spider ident="weblayers">
1419 <name>weblayers</name>
1420 </spider>
1421 <spider ident="WebLinker">
1422 <name>WebLinker</name>
1423 </spider>
1424 <spider ident="wlm">
1425 <name>Weblog Monitor</name>
1426 </spider>
1427 <spider ident="WebQuest">
1428 <name>WebQuest</name>
1429 </spider>
1430 <spider ident="WebReaper">
1431 <name>WebReaper</name>
1432 </spider>
1433 <spider ident="webs@recruit.co.jp">
1434 <name>webs</name>
1435 </spider>
1436 <spider ident="websearchbench">
1437 <name>WebSearchBench</name>
1438 <url>http://websearchbench.cs.uni-dortmund.de/</url>
1439 </spider>
1440 <spider ident="WOLP">
1441 <name>WebStolperer</name>
1442 </spider>
1443 <spider ident="webvac">
1444 <name>WebVac</name>
1445 </spider>
1446 <spider ident="webwalk">
1447 <name>webwalk</name>
1448 </spider>
1449 <spider ident="WebWalker">
1450 <name>WebWalker</name>
1451 </spider>
1452 <spider ident="WebWatch">
1453 <name>WebWatch</name>
52bb0ca4 1454 </spider>
6a3fbb36 1455 <spider ident="WebZinger">
52bb0ca4 1456 <name>WebZinger</name>
25ab2d04
MW		 1457 </spider>
1458 <spider ident="whatUseek_winona">
1459 <name>whatUseek Winona</name>
52bb0ca4 1460 </spider>
6a3fbb36 1461 <spider ident="WhoWhere Robot">
52bb0ca4
P		 1462 <name>WebWatch</name>
1463 <url>http://www.whowhere.com</url>
25ab2d04
MW		 1464 </spider>
1465 <spider ident="SurveyBot">
1466 <name>Whois Source</name>
1467 <url>http://www.whois.sc/info/webmasters/surveybot.html</url>
1468 </spider>
1469 <spider ident="Hazel's Ferret Web hopper">
1470 <name>Wild Ferret Web Hopper</name>
1471 </spider>
0cf0a5c1
C		 1472 <spider ident="HRCrawler">
1473 <name>HRCrawler</name>
1474 </spider>
25ab2d04
MW		 1475 <spider ident="WinHTTP">
1476 <name>WinHTTP</name>
1477 </spider>
1478 <spider ident="wired-digital-newsbot">
1479 <name>Wired Digital</name>
1480 </spider>
1481 <spider ident="zyborg">
1482 <name>WiseNut</name>
1483 </spider>
c3fe17fb 1484 <spider ident="WoltLabSuite">
1485 <name>WoltLab Suite</name>
1486 </spider>
25ab2d04
MW		 1487 <spider ident="OmniExplorer_Bot">
1488 <name>WorldIndexer</name>
1489 <url>http://www.omni-explorer.com</url>
1490 </spider>
1491 <spider ident="WWWC">
1492 <name>WWWC</name>
1493 </spider>
1494 <spider ident="WWWeasel Robot">
1495 <name>WWWeasel Robot</name>
1496 </spider>
1497 <spider ident="wwwster">
1498 <name>wwwster</name>
1499 </spider>
1500 <spider ident="WWWWanderer">
1501 <name>WWWWanderer</name>
1502 </spider>
1503 <spider ident="TECOMAC-Crawler">
1504 <name>X-Crawler</name>
1505 </spider>
1506 <spider ident="XGET">
1507 <name>XGET</name>
1508 </spider>
1509 <spider ident="cosmos">
1510 <name>XYLEME Robot</name>
1511 </spider>
1512 <spider ident="yacybot">
1513 <name>YaCy-Bot</name>
f0cbc030 1514 <url>https://yacy.net/bot.html</url>
25ab2d04
MW		 1515 </spider>
1516 <spider ident="YahooYSMcm">
1517 <name>Yahoo Publisher Network</name>
1518 <url>http://publisher.yahoo.com/</url>
1519 </spider>
1520 <spider ident="Yahoo-Blogs">
1521 <name>Yahoo-Blogs</name>
1522 <url>http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html</url>
1523 </spider>
1524 <spider ident="Yahoo Pipes">
1525 <name>Yahoo Pipes</name>
1526 </spider>
1527 <spider ident="Yahoo! Slurp">
1528 <name>Yahoo! Slurp</name>
1529 <url>http://help.yahoo.com/help/us/ysearch/slurp</url>
1530 </spider>
1531 <spider ident="Yahoo-VerticalCrawler">
1532 <name>Yahoo-VerticalCrawler</name>
1533 </spider>
1534 <spider ident="YahooFeedSeeker">
1535 <name>YahooFeedSeeker</name>
1536 <url>http://my.yahoo.com/s/publishers.html</url>
1537 </spider>
1538 <spider ident="Yandex">
1539 <name>Yandex</name>
1540 <url>http://www.yandex.ru/</url>
1541 </spider>
1542 <spider ident="zeus">
1543 <name>Zeus Internet Marketing</name>
1544 <url>http://www.cyber-robotics.com/</url>
1545 </spider>
1546 <spider ident="http://www.zorkk.com">
1547 <name>Zork</name>
1548 <url>http://www.zorkk.com</url>
1549 </spider>
f55aa1aa
P		 1550 <spider ident="Zookabot">
1551 <name>Zookabotk</name>
1552 <url>http://zookabot.com/</url>
1553 </spider>
faf45819
SM		 1554 <spider ident="ZoominfoBot">
1555 <name>ZoominfoBot</name>
1556 <url>https://www.zoominfo.com/about-zoominfo/zoominfobot</url>
1557 </spider>
0cf0a5c1
C		 1558 <spider ident="360Spider">
1559 <name>360Spider</name>
1560 </spider>
52bb0ca4 1561</data>
WoltLab Suite Core (previously WoltLab Community Framework)