projects / GitHub / WoltLab / WCF.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'origin/6.0'
[GitHub/WoltLab/WCF.git] / spiderList / spiderList.xml
CommitLineData
25ab2d04 1<?xml version="1.0" encoding="UTF-8"?>
99951f50 2<data xmlns="http://www.woltlab.com" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.woltlab.com http://www.woltlab.com/XSD/6.0/spiderList.xsd">
25ab2d04
MW		 3 <spider ident="ABCdatos">
4 <name>ABCdatos BotLink</name>
665cc30d 5 <url>http://www.robotstxt.org/db/abcdatos.html</url>
25ab2d04 6 </spider>
9581146e
PM		 7 <spider ident="AbraveSpider">
8 <name>Abrave</name>
9 <url>http://robot.abrave.co.uk/</url>
10 </spider>
25ab2d04
MW		 11 <spider ident="Accelatech RSSCrawler">
12 <name>Accelatech</name>
13 </spider>
14 <spider ident="Accoona-AI-Agent">
15 <name>Accoona</name>
cdfa27f5 16 <url>https://www.accoona.com/about/</url>
25ab2d04
MW		 17 </spider>
18 <spider ident="aconon Index">
19 <name>aconon Index (raubfische.de)</name>
20 </spider>
21 <spider ident="AcoonBot">
22 <name>Acoon</name>
23 <url>http://www.acoon.de/robot.asp</url>
24 </spider>
25 <spider ident="Ahoy!">
26 <name>Ahoy!</name>
665cc30d 27 <url>http://www.robotstxt.org/db/ahoythehomepagefinder.html</url>
25ab2d04
MW		 28 </spider>
29 <spider ident="AhrefsBot">
30 <name>Ahrefs</name>
31 <url>http://ahrefs.com/robot/</url>
32 </spider>
33 <spider ident="AlkalineBOT">
34 <name>Alkaline</name>
665cc30d 35 <url>http://www.robotstxt.org/db/Alkaline.html</url>
25ab2d04 36 </spider>
3018aace
I		 37 <spider ident="Amazonbot">
38 <name>Amazonbot</name>
39 <url>https://developer.amazon.com/support/amazonbot</url>
40 </spider>
f55aa1aa
P		 41 <spider ident="AlexaBOT">
42 <name>Alexa</name>
43 <url>http://www.alexa.com/help/webmasters</url>
44 </spider>
25ab2d04
MW		 45 <spider ident="http://www.almaden.ibm.com/cs/crawler">
46 <name>Almaden Crawler</name>
47 <url>http://www.almaden.ibm.com/cs/crawler/</url>
48 </spider>
2af2b30d
I		 49 <spider ident="Barkrowler">
50 <name>Barkrowler</name>
51 <url>https://www.babbar.tech/crawler</url>
52 </spider>
25ab2d04
MW		 53 <spider ident="EMC Spider">
54 <name>ananzi</name>
55 </spider>
56 <spider ident="Anthill">
57 <name>Anthill</name>
665cc30d 58 <url>http://www.robotstxt.org/db/anthill.html</url>
25ab2d04
MW		 59 </spider>
60 <spider ident="Aport">
61 <name>Aport</name>
62 <url>http://www.aport.ru/</url>
63 </spider>
64 <spider ident="AppleSyndication">
65 <name>Apple</name>
66 </spider>
6aa6c9ba 67 <spider ident="Applebot">
68 <name>Applebot</name>
69 <url>https://support.apple.com/en-gb/HT204683</url>
70 </spider>
25ab2d04
MW		 71 <spider ident="Arachnophilia">
72 <name>Arachnophilia</name>
665cc30d 73 <url>http://www.robotstxt.org/db/arachnophilia.html</url>
25ab2d04
MW		 74 </spider>
75 <spider ident="Araneo">
76 <name>Araneo</name>
665cc30d 77 <url>http://www.robotstxt.org/db/araneo.html</url>
25ab2d04
MW		 78 </spider>
79 <spider ident="ArchitextSpider">
80 <name>ArchitextSpider</name>
665cc30d 81 <url>http://www.robotstxt.org/db/architext.html</url>
25ab2d04 82 </spider>
89b52a1a 83 <spider ident="archive.org_bot">
25ab2d04 84 <name>Archive.org</name>
89b52a1a 85 <url>https://archive.org/details/archive.org_bot</url>
25ab2d04
MW		 86 </spider>
87 <spider ident="arks/1.0">
88 <name>arks</name>
665cc30d 89 <url>http://www.robotstxt.org/db/arks.html</url>
25ab2d04
MW		 90 </spider>
91 <spider ident="ASpider">
92 <name>ASpider</name>
665cc30d 93 <url>http://www.robotstxt.org/db/aspider.html</url>
25ab2d04
MW		 94 </spider>
95 <spider ident="ATN_Worldwide">
96 <name>ATN Worldwide</name>
665cc30d 97 <url>http://www.robotstxt.org/db/atn.html</url>
25ab2d04
MW		 98 </spider>
99 <spider ident="Atomz">
100 <name>Atomz.com</name>
665cc30d 101 <url>http://www.robotstxt.org/db/atomz.html</url>
25ab2d04
MW		 102 </spider>
103 <spider ident="AURESYS">
104 <name>AURESYS</name>
665cc30d 105 <url>http://www.robotstxt.org/db/auresys.html</url>
25ab2d04 106 </spider>
94a6867f
I		 107 <spider ident="AwarioSmartBot">
108 <name>AwarioSmartBot</name>
109 <url>https://awario.com/bots.html</url>
110 </spider>
111 <spider ident="AwarioRssBot">
112 <name>AwarioRssBot</name>
113 <url>https://awario.com/bots.html</url>
114 </spider>
25ab2d04
MW		 115 <spider ident="BackRub">
116 <name>BackRub</name>
665cc30d 117 <url>http://www.robotstxt.org/db/backrub.html</url>
25ab2d04
MW		 118 </spider>
119 <spider ident="Baiduspider">
120 <name>Baiduspider</name>
121 <url>http://www.baidu.com/search/spider.htm</url>
122 </spider>
25ab2d04
MW		 123 <spider ident="BecomeBot">
124 <name>BecomeBot</name>
125 <url>http://www.become.com/webmasters.html</url>
126 </spider>
127 <spider ident="Big Brother">
128 <name>Big Brother</name>
665cc30d 129 <url>http://www.robotstxt.org/db/bigbrother.html</url>
25ab2d04
MW		 130 </spider>
131 <spider ident="BigmirSpider">
132 <name>Bigmir</name>
133 <url>http://www.bigmir.net/</url>
134 </spider>
135 <spider ident="bingbot">
136 <name>Bing</name>
137 <url>http://www.bing.com/bingbot.htm</url>
138 </spider>
139 <spider ident="Bitacle bot">
140 <name>Bitacle</name>
141 <url>http://bitacle.org/</url>
142 </spider>
d73de6dc
I		 143 <spider ident="BitBot">
144 <name>BitBot</name>
145 <url>https://bitbot.dev</url>
146 </spider>
25ab2d04
MW		 147 <spider ident="Biz360 Spider">
148 <name>Biz</name>
149 <url>http://www.biz360.com</url>
150 </spider>
151 <spider ident="Bjaaland">
152 <name>Bjaaland</name>
665cc30d 153 <url>http://www.robotstxt.org/db/bjaaland.html</url>
25ab2d04
MW		 154 </spider>
155 <spider ident="BlackWidow">
156 <name>BlackWidow</name>
665cc30d 157 <url>http://www.robotstxt.org/db/blackwidow.html</url>
25ab2d04 158 </spider>
b730b757
MW		 159 <spider ident="BLEXBot">
160 <name>BLEXBot</name>
f2a434c0 161 <url>http://webmeup-crawler.com/</url>
b730b757 162 </spider>
25ab2d04
MW		 163 <spider ident="BlogCrawler by Xango">
164 <name>BlogCrawler</name>
165 </spider>
166 <spider ident="blogdb">
167 <name>BlogDb</name>
168 <url>http://blogdb.jp</url>
169 </spider>
170 <spider ident="blog search engine by BlogFan.ORG">
171 <name>BlogFan</name>
172 <url>http://www.blogfan.org</url>
173 </spider>
174 <spider ident="Bloglines">
175 <name>Bloglies</name>
176 <url>http://www.bloglines.com</url>
177 </spider>
178 <spider ident="BlogPulse (ISSpider-3.0)">
179 <name>BlogPulse</name>
180 </spider>
181 <spider ident="BlogSearch">
182 <name>BlogSearch</name>
183 <url>http://www.icerocket.com</url>
184 </spider>
185 <spider ident="BlogsNowBot">
186 <name>BlogsNow</name>
187 <url>http://www.blogsnow.com/</url>
188 </spider>
189 <spider ident="BlogStreetBot">
190 <name>BlogStreetBot</name>
191 <url>http://www.blogstreet.com/</url>
192 </spider>
193 <spider ident="Bulkfeeds">
194 <name>BlogStreetBot</name>
195 <url>http://bulkfeeds.net</url>
196 </spider>
197 <spider ident="BoardPulse">
198 <name>BoardPulse</name>
199 <url>http://www.boardpulse.com/</url>
200 </spider>
201 <spider ident="BoardReader">
202 <name>BoardReader</name>
203 <url>http://www.boardreader.com/aboutus.asp</url>
204 </spider>
205 <spider ident="BoardViewer">
206 <name>BoardViewer</name>
207 <url>http://www.boardviewer.com/</url>
208 </spider>
209 <spider ident="boitho.com-robot">
210 <name>Boitho</name>
211 <url>http://www.boitho.com/bot.html</url>
212 </spider>
213 <spider ident="borg-bot">
214 <name>Borg-Bot</name>
665cc30d 215 <url>http://www.robotstxt.org/db/borg-bot.html</url>
25ab2d04
MW		 216 </spider>
217 <spider ident="BSpider">
218 <name>BSpider</name>
665cc30d 219 <url>http://www.robotstxt.org/db/bspider.html</url>
25ab2d04 220 </spider>
1ac02f65
I		 221 <spider ident="BublupBot">
222 <name>BublupBot</name>
223 <url>https://www.bublup.com/bublup-bot.html</url>
224 </spider>
1b4bb70a
SM		 225 <spider ident="Buck/2.2">
226 <name>Buck</name>
227 <url>https://app.hypefactors.com/media-monitoring/about.html</url>
228 </spider>
36fb7271
I		 229 <spider ident="Bytespider">
230 <name>Bytespider</name>
231 <url>https://www.bytedance.com/</url>
232 </spider>
25ab2d04
MW		 233 <spider ident="CACTVS Chemistry Spider">
234 <name>CACTVS Chemistry</name>
235 </spider>
236 <spider ident="Calif">
237 <name>Calif</name>
238 </spider>
239 <spider ident="CaRP/3.6Evolution">
240 <name>CaRP</name>
241 <url>http://www.biz360.com</url>
242 </spider>
71bfd426
I		 243 <spider ident="CensysInspect">
244 <name>Censys</name>
245 <url>https://about.censys.io/</url>
246 </spider>
25ab2d04
MW		 247 <spider ident="Checkbot">
248 <name>Checkbot</name>
249 </spider>
250 <spider ident="ChristCrawler.com">
251 <name>ChristCrawler.com</name>
252 </spider>
253 <spider ident="www.cienciaficcion.net">
254 <name>cIeNcIaFiCcIoN.nEt</name>
255 </spider>
fc42a807
I		 256 <spider ident="Cincraw">
257 <name>Cincraw</name>
258 <url>http://cincrawdata.net/bot/</url>
259 </spider>
25ab2d04
MW		 260 <spider ident="CipinetBot">
261 <name>Cipinet</name>
262 <url>http://www.cipinet.com/bot.html</url>
263 </spider>
264 <spider ident="CJNetworkQuality">
265 <name>CJNetworkQuality</name>
266 <url>http://www.cj.com/networkquality/</url>
267 </spider>
268 <spider ident="CMC/0.01">
269 <name>CMC/0.01</name>
270 </spider>
271 <spider ident="ColdFusion">
272 <name>ColdFusion</name>
273 </spider>
274 <spider ident="combine">
275 <name>Combine System</name>
276 </spider>
277 <spider ident="Crawler (cometsearch@cometsystems.com)">
278 <name>cometsystems.com</name>
279 </spider>
280 <spider ident="ComputingSite Robi/1.0">
281 <name>ComputingSite Robi/1.0</name>
282 </spider>
283 <spider ident="conceptbot">
284 <name>Conceptbot</name>
285 </spider>
cbfe141b
I		 286 <spider ident="Cookiebot">
287 <name>Cookiebot</name>
288 <url>https://www.cookiebot.com/</url>
289 </spider>
25ab2d04
MW		 290 <spider ident="Cooby.de Crawler">
291 <name>Cooby.de Crawler</name>
292 </spider>
293 <spider ident="CoolBot">
294 <name>CoolBot</name>
295 </spider>
36fb7271
I		 296 <spider ident="CriteoBot">
297 <name>Criteo</name>
298 <url>https://www.criteo.com/criteo-crawler/</url>
299 </spider>
25ab2d04
MW		 300 <spider ident="Cusco">
301 <name>Cusco</name>
302 </spider>
303 <spider ident="CyberSpyder">
304 <name>CyberSpyder</name>
305 </spider>
36fb7271 306 <spider ident="DataForSeoBot">
eef85e19 307 <name>DataForSEO</name>
36fb7271
I		 308 <url>https://dataforseo.com/dataforseo-bot</url>
309 </spider>
25ab2d04
MW		 310 <spider ident="daypopbot">
311 <name>daypop</name>
312 </spider>
313 <spider ident="DesertRealm.com">
314 <name>Desert Realm</name>
315 </spider>
316 <spider ident="Deweb">
317 <name>DeWeb(c)</name>
318 </spider>
319 <spider ident="Die Blinde Kuh">
320 <name>Die Blinde Kuh</name>
665cc30d 321 <url>http://www.robotstxt.org/db/blindekuh.html</url>
25ab2d04
MW		 322 </spider>
323 <spider ident="dienstspider">
324 <name>DienstSpider</name>
325 </spider>
326 <spider ident="Digger/1.0 JDK/1.3.0">
327 <name>Digger</name>
328 </spider>
329 <spider ident="Digimarc WebReader">
330 <name>Digimarc MarcSpider</name>
331 </spider>
332 <spider ident="Digimarc CGIReader">
333 <name>Digimarc Marcspider/CGI</name>
334 </spider>
335 <spider ident="DIIbot">
336 <name>Digital Integrity Robot</name>
337 </spider>
338 <spider ident="grabber">
339 <name>Direct Hit Grabber</name>
340 </spider>
341 <spider ident="discobot">
342 <name>Discovery</name>
343 <url>http://discoveryengine.com/discobot.html</url>
344 </spider>
a370eb7d 345 <spider ident="Discordbot">
346 <name>Discord</name>
347 <url>https://discordapp.com/</url>
348 </spider>
25ab2d04
MW		 349 <spider ident="DNAbot/1.0">
350 <name>DNAbot</name>
351 </spider>
40154603
I		 352 <spider ident="Domains Project">
353 <name>Domains Project</name>
354 <url>https://domainsproject.org/</url>
355 </spider>
f51b8b91 356 <spider ident="DotBot">
fdaca223
C		 357 <name>Moz DotBot</name>
358 <url>http://www.opensiteexplorer.org/dotbot</url>
359 </spider>
25ab2d04
MW		 360 <spider ident="DragonBot/1.0 libwww/5.0">
361 <name>DragonBot</name>
362 </spider>
b8e6d8fb
I		 363 <spider ident="dragonmetrics">
364 <name>Dragon Metrics</name>
365 <url>https://help.dragonmetrics.com/en/articles/213883-about-dragonbot</url>
366 </spider>
3533d304
I		 367 <spider ident="Dubbotbot">
368 <name>DubBot</name>
369 <url>https://help.dubbot.com/en/articles/2923576-dubbot-s-crawler-ip-address-and-user-agent</url>
370 </spider>
a370eb7d 371 <spider ident="DuckDuckBot">
372 <name>DuckDuckGo</name>
05e2373e 373 <url>https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/</url>
a370eb7d 374 </spider>
25ab2d04
MW		 375 <spider ident="DWCP/2.0">
376 <name>DWCP (Dridus' Web Cataloging Project)</name>
377 </spider>
378 <spider ident="e-SocietyRobot">
379 <name>e-Society</name>
380 <url>http://www.yama.info.waseda.ac.jp/~yamana/es/index_eng.htm</url>
381 </spider>
382 <spider ident="exactseek-pagereaper">
383 <name>eaxactseek-page</name>
384 </spider>
385 <spider ident="EbiNess/0.01a">
386 <name>EbiNess</name>
387 </spider>
388 <spider ident="edgeio-retriever">
389 <name>Edgeio</name>
390 <url>http://www.edgeio.com</url>
391 </spider>
392 <spider ident="EIT-Link-Verifier-Robot/0.2">
393 <name>EIT Link Verifier Robot</name>
394 </spider>
395 <spider ident="elfinbot">
396 <name>ELFINBOT</name>
397 </spider>
398 <spider ident="Emacs-w3/v[0-9\.]+">
399 <name>Emacs-w3 Search Engine</name>
400 </spider>
401 <spider ident="esther">
402 <name>Esther</name>
403 </spider>
404 <spider ident="EuripBot/">
405 <name>EuripBot</name>
406 </spider>
30de0ef8
I		 407 <spider ident="ev-crawler">
408 <name>Headline Web Crawler</name>
409 <url>https://headline.com/legal/crawler</url>
410 </spider>
25ab2d04
MW		 411 <spider ident="Evliya Celebi">
412 <name>Evliya Celebi</name>
413 </spider>
414 <spider ident="ExactSeek_Spider">
415 <name>ExactSeek_Spider</name>
416 <url>http://www.askjeevs.com</url>
417 </spider>
418 <spider ident="NG/2.0">
419 <name>ExaLead</name>
420 <url>http://botspotter.net/bs-389.html</url>
421 </spider>
422 <spider ident="ExaBot">
423 <name>ExaLead Beta</name>
424 <url>http://beta.exalead.com/search/C=0/2p=Help.7</url>
425 </spider>
f55aa1aa
P		 426 <spider ident="ExaLead">
427 <name>ExaLead</name>
428 <url>http://www.exalead.com/search</url>
429 </spider>
25ab2d04
MW		 430 <spider ident="facebookexternalhit">
431 <name>Facebook</name>
05e2373e 432 <url>https://developers.facebook.com/docs/sharing/webmasters/crawler/</url>
25ab2d04
MW		 433 </spider>
434 <spider ident="fast-webcrawler">
435 <name>FAST / AlltheWeb</name>
436 <url>http://help.yahoo.com/help/us/ysearch/slurp/index.html</url>
437 </spider>
438 <spider ident="FastCrawler">
439 <name>FastCrawler</name>
440 </spider>
441 <spider ident="Feed24.com">
442 <name>Feed24</name>
443 <url>http://www.feed24.com</url>
444 </spider>
445 <spider ident="FeedBlitz">
446 <name>FeedBlitz</name>
447 <url>http://www.feedblitz.com</url>
448 </spider>
449 <spider ident="FeedBurner">
450 <name>FeedBurner</name>
451 <url>http://www.FeedBurner.com</url>
452 </spider>
7a944c3f
MW		 453 <spider ident="Feedly">
454 <name>Feedly</name>
455 <url>http://www.feedly.com/fetcher.html</url>
456 </spider>
d84d7011 457 <spider ident="FeedFetcher-Google">
25ab2d04
MW		 458 <name>FeedFetcher-Google</name>
459 <url>http://www.google.com/feedfetcher.html</url>
460 </spider>
d84d7011
SG		 461 <spider ident="Google-Read-Aloud">
462 <name>Google Read Aloud</name>
463 <url>https://developers.google.com/search/docs/crawling-indexing/read-aloud-user-agent</url>
464 </spider>
465 <spider ident="Google-Site-Verification/1.0">
466 <name>Google Site Verifier</name>
467 <url>https://support.google.com/webmasters/answer/9008080</url>
468 </spider>
469 <spider ident="Google-Extended">
470 <name>Google-Extended</name>
471 </spider>
25ab2d04
MW		 472 <spider ident="UniversalFeedParser">
473 <name>FeedParser</name>
474 <url>http://www.feedparser.org</url>
475 </spider>
476 <spider ident="Feedster Crawler">
477 <name>Feedster</name>
478 <url>http://www.feedstermedia.com/</url>
479 </spider>
480 <spider ident="FEHLSTART Superspider">
481 <name>FEHLSTART</name>
482 </spider>
483 <spider ident="FelixIDE">
484 <name>Felix IDE</name>
485 </spider>
486 <spider ident="ESIRover">
487 <name>FetchRover</name>
488 </spider>
489 <spider ident="fido">
490 <name>fido</name>
491 </spider>
492 <spider ident="findlinks">
493 <name>FindLinks</name>
494 <url>http://wortschatz.uni-leipzig.de/findlinks/</url>
495 </spider>
496 <spider ident="FindoryBot">
497 <name>Findroy</name>
498 <url>http://www.findory.com</url>
499 </spider>
500 <spider ident="Fish-Search-Robot">
501 <name>Fish search</name>
502 </spider>
503 <spider ident="Mozilla/4.0 (compatible: FDSE robot)">
504 <name>Fluid Dynamics</name>
505 </spider>
a153da52
I		 506 <spider ident="my-tiny-bot">
507 <name>My-Tiny-Bot</name>
508 </spider>
25ab2d04
MW		 509 <spider ident="fouineur.9bit.qc.ca">
510 <name>Fouineur</name>
511 </spider>
512 <spider ident="Freecrawl">
513 <name>Freecrawl</name>
514 </spider>
a62fd64b
I		 515 <spider ident="FreeWebMonitoring SiteChecker">
516 <name>FreeWebMonitoring SiteChecker</name>
517 <url>https://www.freewebmonitoring.com/bot.html</url>
518 </spider>
2f86459f
SM		 519 <spider ident="FreshpingBot">
520 <name>Freshping</name>
521 <url>https://freshping.io/</url>
522 </spider>
cd1dc17e
I		 523 <spider ident="FreshRSS">
524 <name>FreshRSS</name>
525 <url>https://freshrss.org</url>
526 </spider>
25ab2d04
MW		 527 <spider ident="FunnelWeb">
528 <name>FunnelWeb</name>
529 </spider>
530 <spider ident="GaisBot">
531 <name>Gais</name>
532 <url>http://gais.cs.ccu.edu.tw/robot.php</url>
533 </spider>
534 <spider ident="gamekitbot">
535 <name>GAMEKIT</name>
536 <url>http://www.uchoose.de/crawler/gamekitbot/</url>
537 </spider>
538 <spider ident="gammaSpider">
539 <name>gammaSpider</name>
540 </spider>
541 <spider ident="gazz">
542 <name>gazz</name>
543 </spider>
544 <spider ident="gcreep">
545 <name>GCreep</name>
546 </spider>
547 <spider ident="genieBot">
548 <name>genieBot</name>
549 <url>http://64.5.245.11/faq/faq.html</url>
550 </spider>
551 <spider ident="geourl">
552 <name>GeoURL</name>
553 <url>http://geourl.org/bot.html</url>
554 </spider>
555 <spider ident="GetterroboPlus">
556 <name>GetterroboPlus Puu</name>
557 </spider>
558 <spider ident="GetURL.rexx">
559 <name>GetURL</name>
560 </spider>
561 <spider ident="Gigabot">
562 <name>Gigabot</name>
563 <url>http://www.gigablast.com/spider.html</url>
564 </spider>
565 <spider ident="Girafabot">
566 <name>Girafabot</name>
567 <url>http://www.girafa.com/</url>
568 </spider>
569 <spider ident="Goku">
570 <name>Goku</name>
571 <url>http://goku.ru/bot.htm; bot@goku.ru</url>
572 </spider>
573 <spider ident="Golem">
574 <name>Golem</name>
575 </spider>
576 <spider ident="gonzo">
577 <name>Gonzo</name>
578 </spider>
579 <spider ident="Googlebot/">
580 <name>Google</name>
581 <url>http://www.google.com/bot.html</url>
582 </spider>
583 <spider ident="Mediapartners-Google">
584 <name>Google AdSense</name>
585 <url>https://www.google.com/adsense/faq</url>
586 </spider>
587 <spider ident="Googlebot-Image">
588 <name>Googlebot-Image</name>
589 <url>http://www.googlebot.com/bot.html</url>
590 </spider>
591 <spider ident="Googlebot-Mobile">
592 <name>Googlebot-Mobile</name>
593 <url>http://www.google.com/bot.html</url>
594 </spider>
2f4ab6f1 595 <spider ident="Googlebot-Video/1.0">
596 <name>Googlebot-Video</name>
5ef610cb 597 <url>https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers</url>
2f4ab6f1 598 </spider>
d84d7011
SG		 599 <spider ident="APIs-Google">
600 <name>APIs-Google</name>
601 <url>https://developers.google.com/search/docs/crawling-indexing/apis-user-agent</url>
602 </spider>
603 <spider ident="Google Favicon">
604 <name>Google Favicon</name>
605 <url>https://developers.google.com/search/docs/appearance/favicon-in-search</url>
606 </spider>
607 <spider ident="Storebot-Google/1.0">
608 <name>Google StoreBot</name>
609 <url>https://support.google.com/merchants/answer/13294660</url>
610 </spider>
611 <spider ident="Google-InspectionTool/1.0">
612 <name>Google-InspectionTool</name>
613 <url>https://support.google.com/webmasters/answer/9012289</url>
614 </spider>
5c30f4c0
AE		 615 <spider ident="GoogleStackdriverMonitoring-UptimeChecks">
616 <name>Google Stackdriver Monitoring</name>
617 <url>https://cloud.google.com/monitoring/alerts/uptime-checks</url>
618 </spider>
fdaca223
C		 619 <spider ident="Google-Ads-Creatives-Assistant">
620 <name>Google-Ads-Creatives-Assistant</name>
621 </spider>
622 <spider ident="Google-AdWords-Express">
623 <name>Google-AdWords-Express</name>
624 </spider>
625 <spider ident="AdsBot-Google">
626 <name>Google Ads-Bot</name>
627 <url>http://www.google.com/adsbot.html</url>
628 </spider>
d84d7011
SG		 629 <spider ident="AdsBot-Google-Mobile">
630 <name>Google Ads-Bot Mobile</name>
631 <url>http://www.google.com/adsbot.html</url>
632 </spider>
25ab2d04
MW		 633 <spider ident="Gpostbot">
634 <name>Gpostbot</name>
635 <url>http://www.gpost.info/help.php?c=bot</url>
636 </spider>
637 <spider ident="griffon">
638 <name>Griffon</name>
639 </spider>
640 <spider ident="Gromit">
641 <name>Gromit</name>
642 </spider>
643 <spider ident="http://grub.org">
644 <name>Grub Client</name>
645 </spider>
646 <spider ident="Gulper Web Bot">
647 <name>Gulper Bot</name>
648 </spider>
649 <spider ident="havIndex">
650 <name>havIndex</name>
651 </spider>
652 <spider ident="HeinrichderMiragoRobot">
653 <name>HeinrichderMiragoRobot</name>
654 </spider>
655 <spider ident="HenryTheMiragoRobot">
656 <name>HenryTheMiragoRobot</name>
657 </spider>
658 <spider ident="heritrix">
659 <name>Heritrix</name>
05e2373e 660 <url>https://github.com/internetarchive/heritrix3/wiki</url>
25ab2d04
MW		 661 </spider>
662 <spider ident="HKU WWW Robot">
663 <name>HKU WWW Octopus</name>
664 </spider>
665 <spider ident="HolyCowDude">
666 <name>HolyCowDude</name>
667 <url>http://www.holycowdude.com/spider.htm</url>
668 </spider>
9581146e
PM		 669 <spider ident="HomeTags">
670 <name>HomeTags</name>
671 <url>http://www.hometags.nl/bot</url>
672 </spider>
25ab2d04
MW		 673 <spider ident="Hometown">
674 <name>Hometown</name>
675 </spider>
676 <spider ident="htdig">
677 <name>ht://Dig</name>
678 </spider>
679 <spider ident="AITCSRobot">
680 <name>HTML Index</name>
681 </spider>
682 <spider ident="HTMLgobble">
683 <name>HTMLgobble</name>
684 </spider>
685 <spider ident="I Robot">
686 <name>I, Robot</name>
687 </spider>
688 <spider ident="iajaBot">
689 <name>iajaBot</name>
690 </spider>
691 <spider ident="IBM_Planetwide">
692 <name>IBM_Planetwide</name>
693 </spider>
694 <spider ident="+http://www.icerocket.com/">
695 <name>IceRocket</name>
696 <url>http://www.icerocket.com/</url>
697 </spider>
698 <spider ident="ichiro">
699 <name>ichiro</name>
700 </spider>
701 <spider ident="IlTrovatore-Setaccio">
702 <name>IlTrovatore-Setaccio</name>
703 <url>http://www.iltrovatore.it/aiuto/faq.html</url>
704 </spider>
705 <spider ident="image.kapsi.net">
706 <name>image.kapsi.net</name>
707 </spider>
708 <spider ident="Mozilla 3.01 PBWF (Win95)">
709 <name>Imagelock</name>
710 </spider>
711 <spider ident="IncyWincy">
712 <name>IncyWincy</name>
713 </spider>
d1430201
I		 714 <spider ident="infoobot">
715 <name>infoobot</name>
716 <url>https://www.infoo.nl/bot.html</url>
717 </spider>
25ab2d04
MW		 718 <spider ident="Informant">
719 <name>Informant</name>
720 </spider>
721 <spider ident="InfoSeek Robot">
722 <name>InfoSeek Robot 1.0</name>
723 </spider>
724 <spider ident="Infoseek Sidewinder">
725 <name>Infoseek Sidewinder</name>
726 </spider>
727 <spider ident="InfoSpiders">
728 <name>InfoSpiders</name>
729 </spider>
730 <spider ident="INGRID">
731 <name>Ingrid</name>
732 </spider>
733 <spider ident="slurp@inktomi">
734 <name>Inktomi</name>
735 </spider>
736 <spider ident="Insitor">
737 <name>Insitor</name>
738 <url>http://www.insitor.de/</url>
739 </spider>
740 <spider ident="inspectorwww">
741 <name>Inspector Web</name>
742 </spider>
743 <spider ident="IAGENT">
744 <name>IntelliAgent</name>
745 </spider>
746 <spider ident="Intelliseek">
747 <name>Intelliseek</name>
748 <url>http://www.intelliseek.com/</url>
749 </spider>
750 <spider ident="Internet Cruiser Robot">
751 <name>Internet Cruiser</name>
752 </spider>
753 <spider ident="internetseer">
754 <name>Internet Seer</name>
755 </spider>
756 <spider ident="sharp-info-agent">
757 <name>Internet Shinchakubin</name>
758 </spider>
759 <spider ident="InternetLinkAgent">
760 <name>InternetLinkAgent</name>
761 </spider>
762 <spider ident="IRLbot">
763 <name>IRL Crawler</name>
764 <url>http://irl.cs.tamu.edu/crawler</url>
765 </spider>
de9232b3
I		 766 <spider ident="IonCrawl">
767 <name>IonCrawl</name>
768 <url>https://www.ionos.de/terms-gtc/faq-crawler-en</url>
769 </spider>
25ab2d04
MW		 770 <spider ident="Iron33">
771 <name>Iron33</name>
772 </spider>
773 <spider ident="IsraeliSearch">
774 <name>Israeli-search</name>
775 </spider>
776 <spider ident="itchBot">
777 <name>itch</name>
25ab2d04
MW		 778 </spider>
779 <spider ident="JavaBee">
780 <name>JavaBee</name>
781 </spider>
782 <spider ident="JBot">
783 <name>JBot</name>
784 </spider>
785 <spider ident="JCrawler">
786 <name>JCrawler</name>
787 </spider>
788 <spider ident="JetBot">
789 <name>JetEye</name>
790 <url>http://www.jeteye.com/jetbot.html</url>
791 </spider>
792 <spider ident="JoBo">
793 <name>JoBo</name>
794 </spider>
795 <spider ident="Jobot">
796 <name>Jobot</name>
797 </spider>
798 <spider ident="jobs.de">
799 <name>Jobs.de</name>
800 <url>http://www.jobs.de/</url>
801 </spider>
802 <spider ident="JoeBot">
803 <name>JoeBot</name>
804 </spider>
805 <spider ident="jumpstation">
806 <name>JumpStation</name>
807 </spider>
808 <spider ident="Katipo">
809 <name>Katipo</name>
810 </spider>
811 <spider ident="KDD-Explorer">
812 <name>KDD-Explorer</name>
813 </spider>
814 <spider ident="KIT-Fireball">
815 <name>KIT-Fireball</name>
816 </spider>
817 <spider ident="KO_Yappo_Robot">
818 <name>KO_Yappo_Robot</name>
819 </spider>
820 <spider ident="LabelGrab">
821 <name>LabelGrabber</name>
822 </spider>
823 <spider ident="larbin">
824 <name>larbin</name>
825 </spider>
826 <spider ident="legs">
827 <name>legs</name>
828 </spider>
0cf0a5c1
C		 829 <spider ident="linkdexbot">
830 <name>Linkdex</name>
831 <url>http://www.linkdex.com/bots/</url>
832 </spider>
25ab2d04
MW		 833 <spider ident="LinkScan Server">
834 <name>LinkScan</name>
835 </spider>
836 <spider ident="LinkWalker">
837 <name>LinkWalker</name>
838 </spider>
f55aa1aa
P		 839 <spider ident="Linguee Bot">
840 <name>Linguee</name>
841 <url>http://www.linguee.com/bot</url>
842 </spider>
25ab2d04
MW		 843 <spider ident="livedoorCheckers/">
844 <name>livedoorCheckers</name>
845 </spider>
846 <spider ident="Lockon">
847 <name>Lockon</name>
848 </spider>
849 <spider ident="logo.gif crawler">
850 <name>logo.gif</name>
851 </spider>
852 <spider ident="Lycos">
853 <name>Lycos</name>
854 </spider>
855 <spider ident="Magpie">
856 <name>Magpie</name>
857 </spider>
858 <spider ident="MJ12bot">
859 <name>Majestics MJ12bot</name>
449482a3 860 <url>https://mj12bot.com/</url>
25ab2d04
MW		 861 </spider>
862 <spider ident="Mammoth">
863 <name>Mammoth</name>
864 <url>http://www.sli-systems.com</url>
865 </spider>
866 <spider ident="Marvin">
867 <name>Marvin</name>
868 </spider>
869 <spider ident="marvin/infoseek">
870 <name>marvin/infoseek</name>
871 </spider>
872 <spider ident="M/3.8">
873 <name>Mattie</name>
874 </spider>
875 <spider ident="MediaFox">
876 <name>MediaFox</name>
877 </spider>
d9cc31f0
SG		 878 <spider ident="memorybot">
879 <name>Memorybot</name>
880 <url>http://archivethe.net/en/index.php/about/internet_memory1</url>
881 </spider>
25ab2d04
MW		 882 <spider ident="mercator">
883 <name>Mercator</name>
884 <url>http://research.compaq.com/SRC/mercator/</url>
885 </spider>
886 <spider ident="MerzScope">
887 <name>MerzScope</name>
888 </spider>
889 <spider ident="METASpider">
890 <name>META</name>
891 <url>http://www.meta.com.ua/</url>
892 </spider>
893 <spider ident="MetaGer-LinkChecker">
894 <name>MetaGer</name>
895 </spider>
896 <spider ident="MindCrawler">
897 <name>MindCrawler</name>
898 </spider>
899 <spider ident="Miva">
900 <name>Miva</name>
901 </spider>
902 <spider ident="UdmSearch">
903 <name>mnoGoSearch</name>
904 </spider>
905 <spider ident="moget">
906 <name>moget</name>
907 </spider>
36fb7271
I		 908 <spider ident="MojeekBot">
909 <name>Mojeek</name>
910 <url>https://www.mojeek.com/bot.html</url>
911 </spider>
25ab2d04
MW		 912 <spider ident="MOMspider">
913 <name>MOMspider</name>
914 </spider>
915 <spider ident="Monster">
916 <name>Monster</name>
917 </spider>
918 <spider ident="Moreoverbot">
919 <name>Moreover</name>
920 <url>http://www.moreover.com</url>
921 </spider>
922 <spider ident="msnbot">
923 <name>MSNBot</name>
924 <url>http://search.msn.com/msnbot.htm</url>
925 </spider>
926 <spider ident="MSRBOT">
927 <name>MSRBOT</name>
928 <url>http://research.microsoft.com/research/sv/msrbot/</url>
929 </spider>
930 <spider ident="MuscatFerret">
931 <name>Muscat Ferret</name>
932 </spider>
933 <spider ident="MwdSearch">
934 <name>Mwd.Search</name>
935 </spider>
936 <spider ident="NPBot">
937 <name>NameProtect</name>
938 </spider>
939 <spider ident="NaverBot">
940 <name>NaverBot</name>
941 <url>http://www.spidermatic.com/en/robot-spider/20</url>
52bb0ca4 942 </spider>
6a3fbb36 943 <spider ident="NDSpider">
52bb0ca4 944 <name>NDSpider</name>
25ab2d04
MW		 945 </spider>
946 <spider ident="NEC-MeshExplorer">
947 <name>NEC-MeshExplorer</name>
948 </spider>
949 <spider ident="Nederland.zoek">
950 <name>Nederland.zoek</name>
951 </spider>
c1398cdd
I		 952 <spider ident="Neevabot">
953 <name>Neeva</name>
954 <url>https://neeva.com/neevabot</url>
955 </spider>
89b52a1a
MW		 956 <spider ident="NerdyBot">
957 <name>NerdyBot</name>
958 <url>http://nerdybot.com/</url>
959 </spider>
25ab2d04
MW		 960 <spider ident="NetCarta CyberPilot Pro">
961 <name>NetCarta WebMap</name>
962 </spider>
963 <spider ident="Netcraft">
964 <name>Netcraft Web Server Survey</name>
965 <url>http://news.netcraft.com/</url>
966 </spider>
9f4d894d
I		 967 <spider ident="Neticle Crawler">
968 <name>Neticle Crawler</name>
969 <url>https://neticle.com/bot/en/</url>
970 </spider>
25ab2d04
MW		 971 <spider ident="NetMechanic">
972 <name>NetMechanic</name>
973 </spider>
974 <spider ident="NetScoop">
975 <name>NetScoop</name>
976 </spider>
977 <spider ident="newscan-online">
978 <name>newscan-online</name>
979 </spider>
980 <spider ident="NextGenSearchBot 1">
981 <name>NextGenSearchBot</name>
982 <url>http://www.zoominfo.com/NextGenSearchBot</url>
983 </spider>
984 <spider ident="NHSEWalker">
985 <name>NHSE Web Forager</name>
986 </spider>
987 <spider ident="NIF">
988 <name>NIF</name>
d04e330e 989 <url>http://www.newsisfree.com/robot.php</url>
25ab2d04
MW		 990 </spider>
991 <spider ident="NimbleCrawler">
992 <name>NimbleCrawler</name>
993 <url>http://www.healthline.com/aboutus.jsp</url>
994 </spider>
995 <spider ident="Nomad">
996 <name>Nomad</name>
997 </spider>
998 <spider ident="Norbert the Spider">
999 <name>Norbert</name>
1000 <url>http://www.Burf.com</url>
1001 </spider>
1002 <spider ident="Gulliver">
1003 <name>Northern Light</name>
1004 </spider>
1005 <spider ident="explorersearch">
1006 <name>nzexplorer</name>
1007 </spider>
1008 <spider ident="Occam">
1009 <name>Occam</name>
1010 </spider>
1011 <spider ident="Ocelli">
1012 <name>Ocelli</name>
1013 <url>http://www.globalspec.com/Ocelli</url>
1014 </spider>
1015 <spider ident="Online24-Bot">
1016 <name>Online24-Bot</name>
1017 </spider>
1018 <spider ident="Openbot">
1019 <name>Openbot</name>
1020 <url>http://www.openfind.com.tw/robot.html</url>
1021 </spider>
1022 <spider ident="Openfind">
1023 <name>Openfind data gatherer</name>
1024 </spider>
1025 <spider ident="Orbsearch">
1026 <name>Orb Search</name>
1027 </spider>
1028 <spider ident="PackRat">
1029 <name>Pack Rat</name>
1030 </spider>
1031 <spider ident="PageBoy">
1032 <name>PageBoy</name>
1033 </spider>
a7331d8e
I		 1034 <spider ident="PagePeeker">
1035 <name>PagePeeker</name>
1036 <url>https://pagepeeker.com/robots/</url>
1037 </spider>
5657c6b8
N		 1038 <spider ident="Pandalytics">
1039 <name>Pandalytics</name>
1040 <url>https://domainsbot.com/pandalytics/</url>
1041 </spider>
25ab2d04
MW		 1042 <spider ident="ParaSite">
1043 <name>ParaSite</name>
1044 </spider>
1045 <spider ident="Patric">
1046 <name>Patric</name>
1047 </spider>
1048 <spider ident="PEGASUS">
1049 <name>pegasus</name>
1050 </spider>
1051 <spider ident="PerlCrawler/1.0 Xavatoria/2.0">
1052 <name>PerlCrawler 1.0</name>
1053 </spider>
6ca2411e 1054 <spider ident="PetalBot">
1055 <name>PetalBot</name>
1056 <url>https://aspiegel.com/petalbot</url>
1057 </spider>
25ab2d04
MW		 1058 <spider ident="PGP-KA">
1059 <name>PGP Key Agent</name>
1060 </spider>
1061 <spider ident="Duppies">
1062 <name>Phantom</name>
1063 </spider>
1064 <spider ident="phpdig">
1065 <name>PhpDig</name>
1066 </spider>
1067 <spider ident="PiltdownMan">
1068 <name>PiltdownMan</name>
1069 </spider>
1070 <spider ident="Pimptrain's robot">
1071 <name>Pimptrain.com's</name>
1072 </spider>
1073 <spider ident="pingalink">
1074 <name>PingALink</name>
1075 </spider>
1076 <spider ident="Pioneer">
1077 <name>Pioneer</name>
1078 </spider>
1079 <spider ident="PluckFeedCrawler">
1080 <name>Pluck</name>
1081 <url>http://www.pluck.com</url>
1082 </spider>
1083 <spider ident="PlumtreeWebAccessor">
1084 <name>PlumtreeWebAccessor</name>
1085 </spider>
1086 <spider ident="PodNova">
1087 <name>PodNova</name>
1088 <url>http://www.podnova.com</url>
1089 </spider>
1090 <spider ident="Pompos">
1091 <name>Pompos</name>
1092 <url>http://dir.com/pompos.html</url>
1093 </spider>
1094 <spider ident="Poppi">
1095 <name>Poppi</name>
1096 </spider>
0cf0a5c1
C		 1097 <spider ident="publiclibraryarchive.org">
1098 <name>publiclibraryarchive.org</name>
1099 </spider>
25ab2d04
MW		 1100 <spider ident="gestaltIconoclast">
1101 <name>Popular Iconoclast</name>
1102 </spider>
1103 <spider ident="PortalJuice.com">
1104 <name>Portal Juice</name>
1105 </spider>
1106 <spider ident="PortalBSpider">
1107 <name>PortalB Spider</name>
1108 </spider>
f55aa1aa
P		 1109 <spider ident="Qualidator">
1110 <name>Qualidator</name>
1111 <url>www.qualidator.com/Web/de/Support/FAQ_OnlineTestStatistiken.htm</url>
1112 </spider>
25ab2d04
MW		 1113 <spider ident="www.kolinka.com">
1114 <name>Project Kolinka Forum Search</name>
1115 <url>http://www.kolinka.com/</url>
1116 </spider>
1117 <spider ident="psbot">
1118 <name>psbot</name>
1119 </spider>
1120 <spider ident="Qango.com Web Directory">
1121 <name>Qango</name>
1122 <url>http://www.qango.com</url>
1123 </spider>
71a53c20
SM		 1124 <spider ident="Qwant">
1125 <name>Qwant</name>
1126 <url>https://help.qwant.com/bot/</url>
1127 </spider>
f55aa1aa
P		 1128 <spider ident="SBSearch">
1129 <name>SecretSearchEngineLabs.com</name>
1130 <url>http://www.secretsearchenginelabs.com/secret-web-crawler.php</url>
1131 </spider>
307ef0a2 1132 <spider ident="SemrushBot">
1133 <name>SemrushBot</name>
1134 <url>http://semrush.com/bot/</url>
1135 </spider>
25ab2d04
MW		 1136 <spider ident="StackRambler">
1137 <name>Rambler</name>
1138 <url>http://www.rambler.ru/</url>
1139 </spider>
1140 <spider ident="Raven">
1141 <name>Raven Search</name>
1142 </spider>
1143 <spider ident="Resume Robot">
1144 <name>Resume Robot</name>
1145 </spider>
1146 <spider ident="Road Runner: ImageScape Robot">
1147 <name>Road Runner: The ImageScape Robot</name>
1148 </spider>
1149 <spider ident="RHCS">
1150 <name>RoadHouse Crawling System</name>
1151 </spider>
1152 <spider ident="Robbie">
1153 <name>Robbie the Robot</name>
1154 </spider>
1155 <spider ident="RoboCrawl">
1156 <name>RoboCrawl</name>
1157 </spider>
1158 <spider ident="Robofox">
1159 <name>RoboFox</name>
1160 </spider>
1161 <spider ident="Robot du CRIM 1.0a">
1162 <name>Robot Francoroute</name>
1163 </spider>
1164 <spider ident="Robozilla">
1165 <name>Robozilla</name>
1166 </spider>
1167 <spider ident="Roverbot">
1168 <name>Roverbot</name>
1169 </spider>
1170 <spider ident="RSS-SPIDER">
1171 <name>RSS Feed Seeker</name>
1172 <url>http://www.rss-spider.com/fsb.php</url>
1173 </spider>
1174 <spider ident="RuLeS">
1175 <name>RuLeS</name>
1176 </spider>
9581146e
PM		 1177 <spider ident="RyzeCrawler">
1178 <name>RyzeCrawler</name>
1179 <url>http://www.domain2day.nl/crawler/</url>
1180 </spider>
25ab2d04
MW		 1181 <spider ident="SafetyNet Robot">
1182 <name>SafetyNet</name>
1183 </spider>
1184 <spider ident="SBIder">
1185 <name>SBIder.</name>
1186 <url>http://www.sitesell.com/sbider.html</url>
1187 </spider>
1188 <spider ident="Scharia">
1189 <name>Scharia</name>
1190 </spider>
1191 <spider ident="Science-Index">
1192 <name>Science-Index</name>
1193 </spider>
1194 <spider ident="Scooter">
1195 <name>Scooter</name>
1196 </spider>
e034e5a9
I		 1197 <spider ident="SearchAtlas">
1198 <name>SearchAtlas</name>
1199 <url>https://searchatlas.com/</url>
1200 </spider>
25ab2d04
MW		 1201 <spider ident="SearchNZ">
1202 <name>SearchNZ</name>
1203 <url>http://www.searchnz.co.nz/</url>
1204 </spider>
1205 <spider ident="searchprocess">
1206 <name>SearchProcess</name>
1207 </spider>
1208 <spider ident="SearchmetricsBot">
1209 <name>SearchmetricsBot</name>
1210 <url>http://www.searchmetrics.com/en/searchmetrics-bot/</url>
1211 </spider>
1212 <spider ident="Seekbot">
1213 <name>Seekbot</name>
1214 <url>http://www.seekbot.net/bot.html</url>
1215 </spider>
4cba12bf
I		 1216 <spider ident="SeekportBot">
1217 <name>Seekport Bot</name>
1218 <url>https://bot.seekport.com</url>
1219 </spider>
25ab2d04
MW		 1220 <spider ident="Senrigan">
1221 <name>Senrigan</name>
1222 </spider>
1223 <spider ident="Sensis Web Crawler">
1224 <name>Sensis Web Crawler</name>
1225 <url>http://www.sensis.com.au/help.do</url>
1226 </spider>
16240335 1227 <spider ident="SentiBot">
4c8c4953 1228 <name>SentiBot</name>
16240335
C		 1229 <url>http://www.sentibot.eu</url>
1230 </spider>
548371c0
I		 1231 <spider ident="SEO Scanner">
1232 <name>SEO Scanner</name>
1233 </spider>
8cbe75a7
I		 1234 <spider ident="SeobilityBot">
1235 <name>SeobilityBot</name>
1236 <url>https://www.seobility.net/sites/bot.html</url>
1237 </spider>
abafa2ec
SM		 1238 <spider ident="SEOkicks">
1239 <name>SEOkicks</name>
1240 <url>https://www.seokicks.de/robot.html</url>
1241 </spider>
f9dfae3d
I		 1242 <spider ident="seostar.co">
1243 <name>Seostar</name>
1244 <url>https://seostar.co/robot/</url>
1245 </spider>
2f45b485
I		 1246 <spider ident="SerendeputyBot">
1247 <name>SerendeputyBot</name>
1248 <url>http://serendeputy.com/about/serendeputy-bot</url>
1249 </spider>
08bf1baa
SM		 1250 <spider ident="serpstatbot">
1251 <name>serpstatbot</name>
1252 <url>http://serpstatbot.com/</url>
1253 </spider>
a2b39a59
C		 1254 <spider ident="SeznamBot/3.2">
1255 <name>Seznam Bot</name>
1256 <url>http://napoveda.seznam.cz/en/seznambot-intro/</url>
1257 </spider>
25ab2d04
MW		 1258 <spider ident="SG-Scout">
1259 <name>SG-Scout</name>
1260 </spider>
1261 <spider ident="Shagseeker">
1262 <name>ShagSeeker</name>
1263 </spider>
1264 <spider ident="Shai'Hulud">
1265 <name>Shai'Hulud</name>
1266 </spider>
1267 <spider ident="SimBot/1.0">
1268 <name>Simmany Robot Ver1.0</name>
1269 </spider>
34ae61a1
I		 1270 <spider ident="SimplePie">
1271 <name>SimplePie</name>
1272 <url>https://simplepie.org</url>
1273 </spider>
a370eb7d 1274 <spider ident="SkypeUriPreview">
1275 <name>Skype Preview</name>
1276 <url>https://www.skype.com/</url>
1277 </spider>
25ab2d04
MW		 1278 <spider ident="ssearcher100">
1279 <name>Site Searcher</name>
1280 </spider>
1281 <spider ident="Site Valet">
1282 <name>Site Valet</name>
1283 </spider>
1284 <spider ident="http://www.site-list.net">
1285 <name>Site-List</name>
1286 <url>http://www.site-list.net</url>
1287 </spider>
1288 <spider ident="SiteTech-Rover">
1289 <name>SiteTech-Rover</name>
1290 </spider>
1291 <spider ident="+SitiDi.net/SitiDiBot/">
1292 <name>SitiDi.net/SitiDiBot</name>
1293 </spider>
1294 <spider ident="aWapClient">
1295 <name>Skymob.com</name>
1296 </spider>
677899c2
AE		 1297 <spider ident="Slack">
1298 <name>Slackbot</name>
1299 <url>https://api.slack.com/robots</url>
1300 </spider>
25ab2d04
MW		 1301 <spider ident="SLCrawler">
1302 <name>SLCrawler</name>
1303 </spider>
1304 <spider ident="Sleek Spider">
1305 <name>Sleek</name>
1306 </spider>
1307 <spider ident="ESISmartSpider">
1308 <name>Smart Spider</name>
1309 </spider>
1310 <spider ident="Snapbot">
1311 <name>Snapbot</name>
1312 <url>http://www.snap.com/</url>
1313 </spider>
1314 <spider ident="Snooper">
1315 <name>Snooper</name>
1316 </spider>
1317 <spider ident="sohu-search">
1318 <name>sohu-search</name>
1319 </spider>
1320 <spider ident="Solbot">
1321 <name>Solbot</name>
1322 </spider>
1323 <spider ident="Speedy Spider">
1324 <name>Speedy Spider</name>
1325 <url>http://www.entireweb.com/about/search_tech/speedyspider/</url>
1326 </spider>
1327 <spider ident="Sphere Scout">
1328 <name>Sphere</name>
1329 </spider>
1330 <spider ident="Sphider2">
1331 <name>Sphider</name>
1332 </spider>
1333 <spider ident="SpiderBot">
1334 <name>SpiderBot</name>
1335 </spider>
1336 <spider ident="spiderline">
1337 <name>Spiderline Crawler</name>
1338 </spider>
1339 <spider ident="SpiderMan">
1340 <name>SpiderMan</name>
1341 </spider>
1342 <spider ident="SpiderView">
1343 <name>SpiderView(tm)</name>
1344 </spider>
1345 <spider ident="mouse.house">
1346 <name>spider_monkey</name>
1347 </spider>
1348 <spider ident="suke">
1349 <name>Suke</name>
1350 </spider>
1351 <spider ident="suntek">
1352 <name>suntek search engine</name>
1353 </spider>
ea6ec8d3
I		 1354 <spider ident="Superfeedr">
1355 <name>Superfeedr</name>
1356 <url>http://superfeedr.com</url>
1357 </spider>
59192e17
I		 1358 <spider ident="SurdotlyBot">
1359 <name>SurdotlyBot</name>
1360 <url>http://sur.ly/bot.html</url>
1361 </spider>
25ab2d04
MW		 1362 <spider ident="Szukacz">
1363 <name>Szukacz</name>
1364 <url>http://www.szukacz.pl/html/RobotEnglishVersion.html</url>
1365 </spider>
1366 <spider ident="T-H-U-N-D-E-R-S-T-O-N-E">
1367 <name>T-H-U-N-D-E-R-S-T-O-N-E</name>
1368 </spider>
f55aa1aa
P		 1369 <spider ident="TinEye Crawler">
1370 <name>TinEye</name>
1371 <url>http://tineye.com/crawler.html</url>
1372 </spider>
25ab2d04
MW		 1373 <spider ident="Black Widow">
1374 <name>TACH Black Widow</name>
1375 </spider>
e7d8bd51 1376 <spider ident="Tapatalk CloudSearch">
0cf0a5c1
C		 1377 <name>Tapatalk CloudSearch</name>
1378 </spider>
25ab2d04
MW		 1379 <spider ident="Tarantula">
1380 <name>Tarantula</name>
1381 </spider>
1382 <spider ident="tarspider">
1383 <name>tarspider</name>
1384 </spider>
1385 <spider ident="dlw3robot">
1386 <name>Tcl W3 Robot</name>
1387 </spider>
1388 <spider ident="TechBOT">
1389 <name>TechBOT</name>
1390 </spider>
1391 <spider ident="Technoratibot">
1392 <name>Technorati</name>
1393 <url>http://technorati.com/about/</url>
1394 </spider>
1395 <spider ident="Templeton">
1396 <name>Templeton</name>
1397 </spider>
1398 <spider ident="teoma">
1399 <name>Teoma/Ask Jeeves</name>
1400 <url>http://sp.teoma.com/docs/teoma/about/</url>
1401 </spider>
340a1799
I		 1402 <spider ident="trovitBot">
1403 <name>trovitBot</name>
1404 <url>http://www.trovit.com/bot.html</url>
1405 </spider>
25ab2d04
MW		 1406 <spider ident="JubiiRobot">
1407 <name>The Jubii</name>
1408 </spider>
1409 <spider ident="NorthStar">
1410 <name>The NorthStar Robot</name>
1411 </spider>
1412 <spider ident="w3index">
1413 <name>The NWI Robot</name>
1414 </spider>
1415 <spider ident="Peregrinator-Mathematics">
1416 <name>The Peregrinator</name>
1417 </spider>
0cf0a5c1
C		 1418 <spider ident="Pixray-Seeker">
1419 <name>Pixray</name>
1420 <url>http://www.pixray.com/pixraybot/</url>
1421 </spider>
a370eb7d 1422 <spider ident="TelegramBot (like TwitterBot)">
1423 <name>TelegramBot (like TwitterBot)</name>
1424 <url>https://telegram.org/</url>
1425 </spider>
84b3ca1c
I		 1426 <spider ident="Testomatobot">
1427 <name>TestomatoBot</name>
1428 <url>https://www.testomato.com/bot</url>
1429 </spider>
25ab2d04
MW		 1430 <spider ident="thumbshots-de-Bot">
1431 <name>thumbshots-de-Bot</name>
1432 </spider>
1433 <spider ident="TITAN">
1434 <name>TITAN</name>
1435 </spider>
1436 <spider ident="TitIn">
1437 <name>TitIn</name>
1438 </spider>
1439 <spider ident="TLSpider">
1440 <name>TLSpider</name>
1441 </spider>
1442 <spider ident="TMCrawler">
1443 <name>TMCrawler</name>
1444 </spider>
1445 <spider ident="trendictionbot">
1446 <name>Trendiction-Bot</name>
1447 <url>http://www.trendiction.com/bot</url>
1448 </spider>
1449 <spider ident="slysearch">
1450 <name>Turnitin.com</name>
1451 <url>http://www.turnitin.com/static/products_services/search_engines.html</url>
1452 </spider>
d2c93ade 1453 <spider ident="TurnitinBot">
25ab2d04 1454 <name>TurnitinBot</name>
d2c93ade 1455 <url>http://www.turnitin.com/robot/crawlerinfo.html</url>
25ab2d04
MW		 1456 </spider>
1457 <spider ident="TurtleScanner">
1458 <name>Turtle</name>
1459 <url>http://www.turtle.ru/</url>
1460 </spider>
9581146e
PM		 1461 <spider ident="TwengaBot">
1462 <name>Twenga</name>
1463 <url>http://www.twenga.com/bot.html</url>
1464 </spider>
25ab2d04
MW		 1465 <spider ident="Twiceler">
1466 <name>Twiceler</name>
1467 <url>http://www.cuill.com/twiceler/robot.html</url>
1468 </spider>
a370eb7d 1469 <spider ident="Twitterbot">
1470 <name>Twitterbot</name>
1471 <url>https://twitter.com/</url>
1472 </spider>
25ab2d04
MW		 1473 <spider ident="UCSD-Crawler">
1474 <name>UCSD Crawl</name>
1475 </spider>
1476 <spider ident="UMBC-memeta-Bot">
1477 <name>UMBC</name>
1478 </spider>
1479 <spider ident="unisterbot">
1480 <name>Unister</name>
1481 </spider>
1482 <spider ident="Unpartisan">
1483 <name>Unpartisan</name>
1484 <url>http://www.unpartisan.com</url>
1485 </spider>
4465d786
I		 1486 <spider ident="Uptime-Kuma">
1487 <name>Uptime-Kuma</name>
1488 <url>https://uptime.kuma.pet/</url>
1489 </spider>
a602e267
SK		 1490 <spider ident="UptimeRobot/2.0">
1491 <name>Uptime Robot</name>
1492 <url>http://uptimerobot.com/</url>
1493 </spider>
25ab2d04
MW		 1494 <spider ident="urlck">
1495 <name>URL Check</name>
1496 </spider>
1497 <spider ident="URL Spider Pro">
1498 <name>URL Spider Pro</name>
1499 </spider>
1500 <spider ident="Valkyrie">
1501 <name>Valkyrie</name>
1502 </spider>
37b2fc3c
I		 1503 <spider ident="VelenPublicWebCrawler">
1504 <name>Velen Crawler</name>
1505 <url>https://velen.io</url>
1506 </spider>
25ab2d04
MW		 1507 <spider ident="Verticrawl">
1508 <name>Verticrawl</name>
1509 </spider>
1510 <spider ident="Victoria">
1511 <name>Victoria</name>
1512 </spider>
1513 <spider ident="vision-search">
1514 <name>vision-search</name>
1515 </spider>
1516 <spider ident="VoilaBot">
1517 <name>VoilaBot</name>
1518 <url>http://www.voila.com/</url>
1519 </spider>
9581146e
PM		 1520 <spider ident="VisBot">
1521 <name>VisBot</name>
1522 <url>http://www.visvo.com/webmasters.html</url>
1523 </spider>
25ab2d04
MW		 1524 <spider ident="Voyager">
1525 <name>Voyager</name>
1526 </spider>
1527 <spider ident="VWbot_K">
1528 <name>VWbot</name>
1529 </spider>
1530 <spider ident="W3M2">
1531 <name>W3M2</name>
1532 </spider>
1533 <spider ident="w3mir">
1534 <name>w3mir</name>
1535 </spider>
1536 <spider ident="w@pSpider">
1537 <name>w@pSpider</name>
1538 </spider>
1539 <spider ident="appie">
1540 <name>Walhello appie</name>
665cc30d 1541 <url>http://www.robotstxt.org/db/appie.html</url>
25ab2d04
MW		 1542 </spider>
1543 <spider ident="CrawlPaper">
1544 <name>WallPaper</name>
1545 </spider>
1546 <spider ident="root">
1547 <name>Web Core / Roots</name>
1548 </spider>
f55aa1aa
P		 1549 <spider ident="WBSearchBot">
1550 <name>Ware Bay</name>
1551 <url>http://www.warebay.com/bot.html</url>
1552 </spider>
25ab2d04
MW		 1553 <spider ident="WebMoose">
1554 <name>Web Moose</name>
1555 </spider>
1556 <spider ident="WebBandit">
1557 <name>WebBandit</name>
1558 </spider>
1559 <spider ident="WebCatcher">
1560 <name>WebCatcher</name>
1561 </spider>
1562 <spider ident="Webclipping">
1563 <name>Webclipping</name>
1564 </spider>
1565 <spider ident="WebCopy">
1566 <name>WebCopy</name>
1567 </spider>
1568 <spider ident="WebFetcher">
1569 <name>webfetcher</name>
1570 </spider>
1571 <spider ident="weblayers">
1572 <name>weblayers</name>
1573 </spider>
1574 <spider ident="WebLinker">
1575 <name>WebLinker</name>
1576 </spider>
1577 <spider ident="wlm">
1578 <name>Weblog Monitor</name>
1579 </spider>
1580 <spider ident="WebQuest">
1581 <name>WebQuest</name>
1582 </spider>
1583 <spider ident="WebReaper">
1584 <name>WebReaper</name>
1585 </spider>
1586 <spider ident="webs@recruit.co.jp">
1587 <name>webs</name>
1588 </spider>
1589 <spider ident="websearchbench">
1590 <name>WebSearchBench</name>
1591 <url>http://websearchbench.cs.uni-dortmund.de/</url>
1592 </spider>
1593 <spider ident="WOLP">
1594 <name>WebStolperer</name>
1595 </spider>
1596 <spider ident="webvac">
1597 <name>WebVac</name>
1598 </spider>
1599 <spider ident="webwalk">
1600 <name>webwalk</name>
1601 </spider>
1602 <spider ident="WebWalker">
1603 <name>WebWalker</name>
1604 </spider>
1605 <spider ident="WebWatch">
1606 <name>WebWatch</name>
52bb0ca4 1607 </spider>
6a3fbb36 1608 <spider ident="WebZinger">
52bb0ca4 1609 <name>WebZinger</name>
25ab2d04
MW		 1610 </spider>
1611 <spider ident="whatUseek_winona">
1612 <name>whatUseek Winona</name>
52bb0ca4 1613 </spider>
6a3fbb36 1614 <spider ident="WhoWhere Robot">
52bb0ca4
P		 1615 <name>WebWatch</name>
1616 <url>http://www.whowhere.com</url>
25ab2d04
MW		 1617 </spider>
1618 <spider ident="SurveyBot">
1619 <name>Whois Source</name>
1620 <url>http://www.whois.sc/info/webmasters/surveybot.html</url>
1621 </spider>
1622 <spider ident="Hazel's Ferret Web hopper">
1623 <name>Wild Ferret Web Hopper</name>
1624 </spider>
0cf0a5c1
C		 1625 <spider ident="HRCrawler">
1626 <name>HRCrawler</name>
1627 </spider>
25ab2d04
MW		 1628 <spider ident="WinHTTP">
1629 <name>WinHTTP</name>
1630 </spider>
1631 <spider ident="wired-digital-newsbot">
1632 <name>Wired Digital</name>
1633 </spider>
1634 <spider ident="zyborg">
1635 <name>WiseNut</name>
1636 </spider>
c3fe17fb 1637 <spider ident="WoltLabSuite">
1638 <name>WoltLab Suite</name>
1639 </spider>
25ab2d04
MW		 1640 <spider ident="OmniExplorer_Bot">
1641 <name>WorldIndexer</name>
1642 <url>http://www.omni-explorer.com</url>
1643 </spider>
1644 <spider ident="WWWC">
1645 <name>WWWC</name>
1646 </spider>
1647 <spider ident="WWWeasel Robot">
1648 <name>WWWeasel Robot</name>
1649 </spider>
1650 <spider ident="wwwster">
1651 <name>wwwster</name>
1652 </spider>
1653 <spider ident="WWWWanderer">
1654 <name>WWWWanderer</name>
1655 </spider>
1656 <spider ident="TECOMAC-Crawler">
1657 <name>X-Crawler</name>
1658 </spider>
1659 <spider ident="XGET">
1660 <name>XGET</name>
1661 </spider>
1662 <spider ident="cosmos">
1663 <name>XYLEME Robot</name>
1664 </spider>
1665 <spider ident="yacybot">
1666 <name>YaCy-Bot</name>
f0cbc030 1667 <url>https://yacy.net/bot.html</url>
25ab2d04
MW		 1668 </spider>
1669 <spider ident="YahooYSMcm">
1670 <name>Yahoo Publisher Network</name>
1671 <url>http://publisher.yahoo.com/</url>
1672 </spider>
1673 <spider ident="Yahoo-Blogs">
1674 <name>Yahoo-Blogs</name>
1675 <url>http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html</url>
1676 </spider>
1677 <spider ident="Yahoo Pipes">
1678 <name>Yahoo Pipes</name>
1679 </spider>
1680 <spider ident="Yahoo! Slurp">
1681 <name>Yahoo! Slurp</name>
1682 <url>http://help.yahoo.com/help/us/ysearch/slurp</url>
1683 </spider>
1684 <spider ident="Yahoo-VerticalCrawler">
1685 <name>Yahoo-VerticalCrawler</name>
1686 </spider>
1687 <spider ident="YahooFeedSeeker">
1688 <name>YahooFeedSeeker</name>
1689 <url>http://my.yahoo.com/s/publishers.html</url>
1690 </spider>
1691 <spider ident="Yandex">
1692 <name>Yandex</name>
b16b1f57 1693 <url>https://yandex.com/bots</url>
25ab2d04
MW		 1694 </spider>
1695 <spider ident="zeus">
1696 <name>Zeus Internet Marketing</name>
1697 <url>http://www.cyber-robotics.com/</url>
1698 </spider>
1699 <spider ident="http://www.zorkk.com">
1700 <name>Zork</name>
1701 <url>http://www.zorkk.com</url>
1702 </spider>
f55aa1aa
P		 1703 <spider ident="Zookabot">
1704 <name>Zookabotk</name>
1705 <url>http://zookabot.com/</url>
1706 </spider>
faf45819
SM		 1707 <spider ident="ZoominfoBot">
1708 <name>ZoominfoBot</name>
1709 <url>https://www.zoominfo.com/about-zoominfo/zoominfobot</url>
1710 </spider>
0cf0a5c1
C		 1711 <spider ident="360Spider">
1712 <name>360Spider</name>
1713 </spider>
186a373f
SG		 1714 <spider ident="GPTBot/">
1715 <name>GPTBot</name>
1716 <url>https://openai.com/gptbot</url>
1717 </spider>
f1945886
SG		 1718 <spider ident="PerplexityBot/">
1719 <name>PerplexityBot</name>
1720 <url>https://perplexity.ai/perplexitybot</url>
1721 </spider>
52bb0ca4 1722</data>
WoltLab Suite Core (previously WoltLab Community Framework)