Added several Google Crawlers & Fetchers

Added

- Google Read Aloud
- Google Site Verifier
- Google-Extended (Bard & Vertex AI)
- APIs-Google
- Google Favicon
- Google StoreBot
- Google-InspectionTool
- AdsBot-Google-Mobile

to the spider list.

diff --git a/spiderList/spiderList.xml b/spiderList/spiderList.xml
index a1217218dbccd494180b14d8eb3ca782f0a8558a..ce31244e2f348a5bdf82bc7700e18a771e159da7 100644 (file)
--- a/spiderList/spiderList.xml
+++ b/spiderList/spiderList.xml
@@ -442,10 +442,21 @@
                <name>Feedly</name>
                <url>http://www.feedly.com/fetcher.html</url>
        </spider>
-       <spider ident="Feedfetcher-Google">
+       <spider ident="FeedFetcher-Google">
                <name>FeedFetcher-Google</name>
                <url>http://www.google.com/feedfetcher.html</url>
        </spider>
+       <spider ident="Google-Read-Aloud">
+               <name>Google Read Aloud</name>
+               <url>https://developers.google.com/search/docs/crawling-indexing/read-aloud-user-agent</url>
+       </spider>
+       <spider ident="Google-Site-Verification/1.0">
+               <name>Google Site Verifier</name>
+               <url>https://support.google.com/webmasters/answer/9008080</url>
+       </spider>
+       <spider ident="Google-Extended">
+               <name>Google-Extended</name>
+       </spider>
        <spider ident="UniversalFeedParser">
                <name>FeedParser</name>
                <url>http://www.feedparser.org</url>
@@ -570,6 +581,22 @@
                <name>Googlebot-Video</name>
                <url>https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers</url>
        </spider>
+       <spider ident="APIs-Google">
+               <name>APIs-Google</name>
+               <url>https://developers.google.com/search/docs/crawling-indexing/apis-user-agent</url>
+       </spider>
+       <spider ident="Google Favicon">
+               <name>Google Favicon</name>
+               <url>https://developers.google.com/search/docs/appearance/favicon-in-search</url>
+       </spider>
+       <spider ident="Storebot-Google/1.0">
+               <name>Google StoreBot</name>
+               <url>https://support.google.com/merchants/answer/13294660</url>
+       </spider>
+       <spider ident="Google-InspectionTool/1.0">
+               <name>Google-InspectionTool</name>
+               <url>https://support.google.com/webmasters/answer/9012289</url>
+       </spider>
        <spider ident="GoogleStackdriverMonitoring-UptimeChecks">
                <name>Google Stackdriver Monitoring</name>
                <url>https://cloud.google.com/monitoring/alerts/uptime-checks</url>
@@ -584,6 +611,10 @@
                <name>Google Ads-Bot</name>
                <url>http://www.google.com/adsbot.html</url>
        </spider>
+       <spider ident="AdsBot-Google-Mobile">
+               <name>Google Ads-Bot Mobile</name>
+               <url>http://www.google.com/adsbot.html</url>
+       </spider>
        <spider ident="Gpostbot">
                <name>Gpostbot</name>
                <url>http://www.gpost.info/help.php?c=bot</url>