From: Marcel Werk <burntime@woltlab.com>
Date: Mon, 21 Jul 2014 20:54:28 +0000 (+0200)
Subject: Added missing bots
X-Git-Tag: 2.1.0_Alpha_1~549
X-Git-Url: https://git.stricted.de/?a=commitdiff_plain;h=89b52a1aeafcc530a6e66dd61ea22e8d1bac8fd2;p=GitHub%2FWoltLab%2FWCF.git

Added missing bots

Closes #1779
Closes #1684
---

diff --git a/spiderList/spiderList.xml b/spiderList/spiderList.xml
index 94a5ac4337..c37684c8de 100644
--- a/spiderList/spiderList.xml
+++ b/spiderList/spiderList.xml
@@ -72,9 +72,9 @@
 		<name>ArchitextSpider</name>
 		<url>http://www.robotstxt.org/wc/active/html/architext.html</url>
 	</spider>
-	<spider ident="ia_archiver">
+	<spider ident="archive.org_bot">
 		<name>Archive.org</name>
-		<url>http://www.archive.org/about/exclude.php</url>
+		<url>https://archive.org/details/archive.org_bot</url>
 	</spider>
 	<spider ident="arks/1.0">
 		<name>arks</name>
@@ -789,6 +789,10 @@
 	<spider ident="Nederland.zoek">
 		<name>Nederland.zoek</name>
 	</spider>
+	<spider ident="NerdyBot">
+		<name>NerdyBot</name>
+		<url>http://nerdybot.com/</url>
+	</spider>
 	<spider ident="NetCarta CyberPilot Pro">
 		<name>NetCarta WebMap</name>
 	</spider>