Added missing bots

Closes #1779
Closes #1684

diff --git a/spiderList/spiderList.xml b/spiderList/spiderList.xml
index 94a5ac433751332e82ab9db764de8b960d8ecab6..c37684c8de5264715b306ffb867225f37d85e664 100644 (file)
--- a/spiderList/spiderList.xml
+++ b/spiderList/spiderList.xml
@@ -72,9 +72,9 @@
                <name>ArchitextSpider</name>
                <url>http://www.robotstxt.org/wc/active/html/architext.html</url>
        </spider>
-       <spider ident="ia_archiver">
+       <spider ident="archive.org_bot">
                <name>Archive.org</name>
-               <url>http://www.archive.org/about/exclude.php</url>
+               <url>https://archive.org/details/archive.org_bot</url>
        </spider>
        <spider ident="arks/1.0">
                <name>arks</name>
@@ -789,6 +789,10 @@
        <spider ident="Nederland.zoek">
                <name>Nederland.zoek</name>
        </spider>
+       <spider ident="NerdyBot">
+               <name>NerdyBot</name>
+               <url>http://nerdybot.com/</url>
+       </spider>
        <spider ident="NetCarta CyberPilot Pro">
                <name>NetCarta WebMap</name>
        </spider>