[Github Action] Automated trickest wordlists update.

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt

@@ -1,6 +1,7 @@
 $
 $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 %24rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
+&solved=
 ,
 -
 -/en$
@@ -52,6 +53,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /.git
 /?z=1
 /MANUALS/*.pdf$
+/Search/
 /_
 /_/rsrc
 /_activities
@@ -68,6 +70,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /api
 /apis
 /archive
+/base-en/
 /bin/search.py
 /blame
 /cache
@@ -78,6 +81,14 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /discovered
 /docs/product
 /download
+/download/Browse.aspx?
+/download/browse.aspx?
+/download/confirmation.aspx?
+/download/info.aspx?
+/download/registration-suggested.aspx?
+/download/results.aspx?
+/download/thankyou.aspx
+/download/thankyou.aspx?
 /download?
 /edit$
 /favorites_list
@@ -89,12 +100,17 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /format:thumbnail
 /forum-attachment
 /gcrnsts
+/layout/
 /leads/*/jsplayer_show_lead_form.html
 /leads/*/show_form.html
 /likes
 /listing/*/favoriters
 /lookup?
+/mediaapi/
 /mediaviewer/*/tr
+/music/*/Search/
+/music/*/search/
+/newsearch/
 /organize
 /people/*/circle
 /people/*/favorites
@@ -110,19 +126,30 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /scheduled-pin
 /scheduled-pins
 /search
+/search/
 /search/*?q=
 /search?q=
+/security/search-results.aspx?
 /shop/*/favoriters
 /shop/*/sold
 /stats?id=
+/store/*/layout/
+/store/*/search/
+/store/buynow?
+/store/cart
+/store/d/groove-music-pass/cfq7ttc0k5dq/0001
+/store/locations/store-locations-sitemap.xml
+/store/music/groove-music-pass/*
 /story_pins
 /tagged
 /tarball
 /tree
 /tried
 /tv-season/*/*?i=
+/unsubscribe/
 /video-
 /video_pins
+/wal/
 /wiki/Special:
 /wiki/index.php/Help
 /wiki/index.php/MediaWiki
@@ -135,6 +162,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /wp-content/themes
 /wp-includes
 /zipball
+=imgmanager
 ?
 ?*ListingID=
 ?*ProfileID=
@@ -181,6 +209,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?hl=*&
 ?hl=*&*&gws_rd=ssl
 ?hl=*&gws_rd=ssl$
+?intc=
 ?lang=
 ?null$
 ?offset=0
@@ -214,6 +243,7 @@ AccountMerge
 Achievements
 ActLikeAnOwner
 ActionRecord
+ActivityUID=
 AdRequestEventLogApi
 AddForumUser
 AddListing
@@ -382,6 +412,8 @@ GSSAjax
 GeneralSupport
 GermanwingsPromo
 GermanwingsRegistration
+Germany/kleinunternehmen/euga/detail.mspx?
+Germany/kleinunternehmen/euga/results.mspx?
 GetListedAjaxGeneral
 GetListedLightbox
 GetListedUpdateAttraction
@@ -418,6 +450,7 @@ HotelsManageListing
 HotelsManagementCenter
 HotelsProductCart
 HowTheSiteWorks
+HpOptOut=true$
 IBCommerce
 ImpressionRecord
 ImproveListing
@@ -435,6 +468,7 @@ International/questions/qa-html-language-declarations/icons
 InterstitialsWidget
 IpadInterstitial
 JSErrorRecord
+Japan/DirectX/default.asp?
 JoinPlus
 JsonServiceController
 JustForYouGuide
@@ -857,6 +891,7 @@ TAScreenSaver
 TCDestinationsWidget
 TCIBContentQualityTool
 TLMCountries
+TOCLinksForCrawlers*
 Tablet
 TaggingAjax
 TagsJSONServlet
@@ -1107,6 +1142,38 @@ accounts/force-by-email
 accounts/o8
 accounts/o8/id
 aclk
+action=accessorysearch&
+action=accessorysearch&page=2$
+action=accessorysearch&page=3$
+action=accessorysearch&page=4$
+action=accessorysearch&page=5$
+action=accessorysearch&page=6$
+action=accessorysearch&page=7$
+action=accessorysearch&page=8$
+action=accessorysearch&product=*$
+action=accessorysearch&product=*&*
+action=catalogsearch&
+action=catalogsearch&catalog_mode=grid&page=2$
+action=catalogsearch&catalog_mode=grid&page=3$
+action=catalogsearch&catalog_mode=grid&page=4$
+action=catalogsearch&catalog_mode=grid&page=5$
+action=catalogsearch&catalog_mode=grid&page=6$
+action=catalogsearch&catalog_mode=grid&page=7$
+action=catalogsearch&catalog_mode=grid&page=8$
+action=catalogsearch&catalog_mode=list&page=2$
+action=catalogsearch&catalog_mode=list&page=3$
+action=catalogsearch&catalog_mode=list&page=4$
+action=catalogsearch&catalog_mode=list&page=5$
+action=catalogsearch&catalog_mode=list&page=6$
+action=catalogsearch&catalog_mode=list&page=7$
+action=catalogsearch&catalog_mode=list&page=8$
+action=overlay&
+action=productLinkAction&
+action=productSupportSearch
+action=productcompareaction&
+action=quickSearch&
+action=siteSearch
+action=writeReview
 actions/news/rss
 ad_acknowledgment
 ad_id=
@@ -1150,7 +1217,6 @@ alkali
 allanalytics
 alllocations
 ambry
-amp/authorize
 amphtml
 amphtml/blogs/test
 amphtml/classic-apps
@@ -1449,6 +1515,7 @@ biz_redir
 biz_share
 biz_update
 bkshp?*q=
+blacklisted*
 blog/*/*/comments
 blog/*/*/feed
 blog/*/*/trackback
@@ -1637,6 +1704,7 @@ calendar/about
 call/join
 call?id=
 calls
+canada/Library/mnp/2/aspx/
 cap
 card/panel
 cart
@@ -1762,6 +1830,10 @@ commercesearch/services
 commission_2010-2014/katainen
 commits?author=
 communities
+communities/bin.aspx?
+communities/blogs/PortalResults.mspx?
+communities/eventdetails.mspx?
+communities/rss.aspx*
 communities/sitemap
 community
 community/*/category/*
@@ -1841,6 +1913,7 @@ de*/people/*/favorites
 de*/shop/*/favoriters
 de*/shop/*/sold
 de-de.www.cloudflare.com
+de-de/corporate/rechtliche-hinweise/impressum_de.aspx
 de-de/feedback
 de-de/lp
 de-en*/listing/*/favoriters
@@ -2142,6 +2215,15 @@ en-in.www.cloudflare.com
 en-in/feedback
 en-in/lp
 en-us.www.cloudflare.com
+en-us/windows/si/matrix.html
+en/microsoft-365
+en/microsoft-365/
+en/microsoft-teams
+en/microsoft-teams/
+en/microsoft-viva
+en/microsoft-viva/
+en/security
+en/security/
 endorsements
 engagement
 enterprise-jobs
@@ -2191,11 +2273,19 @@ es/market/confederate_flag
 es/market/dixie_flag
 es/market/rebel_flag
 es/messages
+es/microsoft-365
+es/microsoft-365/
+es/microsoft-teams
+es/microsoft-teams/
+es/microsoft-viva
+es/microsoft-viva/
 es/mini.php
 es/search*attr_*=
 es/search*price_bucket=
 es/search*search_type=
 es/search*ship_to=
+es/security
+es/security/
 es/share
 es/shop_name_search_service
 es/teams
@@ -2249,6 +2339,7 @@ facebook_connect
 faves-i_love_u_get_away_from_me
 favorite
 fb.html
+fe=true
 feature/sf-test
 feed
 feed$
@@ -2259,6 +2350,7 @@ feed?section=stayhome
 feed?section=stayhome$
 feedback
 feeds
+feeds/TechNet/fr-fr/screenshot/screenshot%20surface.jpg
 feeds/videos.xml
 fetch
 fi-en*/listing/*/favoriters
@@ -2380,17 +2472,28 @@ fr/market/confederate_flag
 fr/market/dixie_flag
 fr/market/rebel_flag
 fr/messages
+fr/microsoft-365
+fr/microsoft-365/
+fr/microsoft-teams
+fr/microsoft-teams/
+fr/microsoft-viva
+fr/microsoft-viva/
 fr/mini.php
 fr/search*attr_*=
 fr/search*price_bucket=
 fr/search*search_type=
 fr/search*ship_to=
+fr/security
+fr/security/
 fr/share
 fr/shop_name_search_service
 fr/teams
 fr/transactions
 fr/treasury
 fr/your
+france/formation/centres/planning.asp?
+france/ie/default.asp?
+france/mnp_utility.mspx?
 friends
 from_id=
 frommers
@@ -2406,6 +2509,9 @@ gamtarget
 gb/images
 gcc
 gdpr-consent
+genuine/
+germany/library/images/mnp/
+germany/video/de/de/related*
 getCallUsInfoData
 get_midroll_info
 get_video
@@ -2618,8 +2724,9 @@ hosted/images
 hotelfinder/rpc
 hotels/rpc
 hp/video
-http://chrome.google.com/robots.txt
-http://sites.google.com/robots.txt
+hpc/*/supported-applications.aspx?
+http://amazon.co.uk/robots.txt
+http://tinyurl.com/robots.txt
 hub?category=
 hz/contact-us/ajax/initiate-trusted-contact
 hz/help/contact/*/message/$
@@ -2663,6 +2770,7 @@ ie/conversations
 ie/guest-pass
 ie/help/sitemap
 ie/help/sitemap.php
+ie/ie40/
 ie/invites/buy
 ie/listing/*/stubs
 ie/locale/overlay
@@ -2737,6 +2845,7 @@ il-en/your
 il/cart
 im?
 images
+imaginecup/*
 imesync
 imgres
 imode/?
@@ -2797,11 +2906,14 @@ includes
 indepthtoolkit/smallprox
 indepthtoolkit/smallprox$
 index.html?
+info/customerror.htm*
+info/smart404.asp*
 info/tech-feedback
 inproduct_help/chatsupport
 install-shuffles
 internalPref
 intl/*/about/views
+intlkb/
 invite
 inviteFromProfile
 inviteMany
@@ -2822,6 +2934,7 @@ iplayer/cbeebies/search
 iplayer/search
 ireceipt/get
 ireceipt/get?
+isapi/
 it*/listing/*/favoriters
 it*/people/*/circle
 it*/people/*/favorites
@@ -2877,6 +2990,11 @@ it/your
 ja-jp.www.cloudflare.com
 ja-jp/feedback
 ja-jp/lp
+japan/directx/default.asp?
+japan/enable/textview.asp?
+japan/products/library/search.asp?
+japan/showcase/print/default.aspx?
+japan/terminology/query.asp?
 job-apply
 jobs-guest
 jobs/EU_UpdateJobEmployerCounter
@@ -3050,7 +3168,9 @@ m/?
 m/finance
 m=1$
 m?
+mac/help.mspx?
 macros
+mactopia/help.mspx?
 mail/$
 mail?
 mailinglist
@@ -3181,6 +3301,7 @@ min=
 mini.php
 missingpage
 mld?
+mnp_utility.mspx?
 mob/tracking
 mobile
 mobile/cities-transformed
@@ -3203,6 +3324,7 @@ mtb_composer
 multiproduct
 muse/identity/v2/identity.js
 music
+music/*/wal/
 music/album
 music/artist
 musica
@@ -3775,6 +3897,7 @@ register
 registration
 registry/wishlist
 remotelogout
+rep=hc
 report
 report-abuse?report=
 report=
@@ -3807,6 +3930,8 @@ row
 rss
 rss/people/*/reviews
 ru-ru.www.cloudflare.com
+ru-ru/events/platforma/materials/default.aspx?speaker*
+rus/licensing/Unilateral.aspx/*
 rw/WashingtonPost/Content/Epaper
 rweb
 s
@@ -4001,7 +4126,7 @@ shopping/reviewer
 shopping/seller
 shopping/suppliers/search
 shopping?
-showtimes/location/*http://tripadvisor.com/robots.txt
+showtimes/location/*http://ted.com/robots.txt
 sidewiki/entry
 signature
 signin/inject
@@ -4294,6 +4419,8 @@ sounds/search?
 source
 source=
 source_repo=
+spain/empresas/
+spain/medianaempresa/
 specialoffers
 specials
 spice
@@ -5075,6 +5202,13 @@ wiki/ויקיפדיה%3Aרשימת_מועמדים_למחיקה
 wiki/ויקיפדיה:דפים_לא_קיימים_ומוגנים
 wiki/ויקיפדיה:ערכים_לא_קיימים_ומוגנים
 wiki/ויקיפדיה:רשימת_מועמדים_למחיקה
+windows/404.aspx?*
+windows/campaign/features-built-in-apps.aspx
+windows/campaign/meet-apps.aspx
+windows/campaign/meet-start.aspx
+windows/compatibility/windows-7/
+windows/compatibility/windows-7/*
+windows/compatibility/windows-vista/
 wirecutter/*&xid=
 wirecutter/*.csv$
 wirecutter/*.zip$

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt

@@ -3072,7 +3072,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /newssearch
 /no-internet-heading-assigned
 /no-internet-heading-assisted
-/no-referrer-when-downgradehttp://ko-fi.com/robots.txt
+/no-referrer-when-downgradehttp://gmail.com/robots.txt
 /no-search-results
 /node
 /node/add
@@ -3641,7 +3641,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*__rmid
 ?*__rmidpage
 ?*ajs_event
-?*apikey*http://chromium.org/robots.txt
+?*apikey*http://visitazores.com/robots.txt
 ?*app
 ?*appidt=*
 ?*aspects=
@@ -3869,7 +3869,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*utm_medium=
 ?*utm_source
 ?*utm_source=
-?*utm_source=http://pixel.quantserve.com/robots.txt
+?*utm_source=http://chrome.google.com/robots.txt
 ?*utm_term=
 ?*verified=
 ?*xid
@@ -3930,7 +3930,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?br
 ?bundle=
 ?cacheBurst=
-?cacheKill=
 ?calendar
 ?cat=*
 ?categoryId=
@@ -4007,8 +4006,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?ffsputnik=*
 ?filter=
 ?filters=
-?forceCds=
-?forceXml=
 ?format=
 ?format=json
 ?forumId=
@@ -4063,7 +4060,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?last_metric=
 ?lastactivity
 ?lid=
-?live=
 ?loc=*&cvid=*
 ?location=
 ?location=home
@@ -4204,7 +4200,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?showcomment*
 ?site_version=viewfullsite
 ?sitrackingid=
-?skipCache=
 ?slogin
 ?snr
 ?solrsort
@@ -5692,6 +5687,7 @@ Korean
 LFE/Live.controls.sharedpages/error.aspx
 LICENSE.txt
 LNAjax
+LP
 LanderLogger
 LandingPage
 Landingpages.html
@@ -6927,7 +6923,7 @@ account-settings
 account-setup
 account/
 account/accomplishments
-account/accomplishments/http://business.google.com/robots.txt
+account/accomplishments/http://creativemarket.com/robots.txt
 account/ackgift
 account/deactivated
 account/emailoptout/?*token=
@@ -6970,6 +6966,7 @@ account_settings/set_your_password$
 account_settings/set_your_password?
 account_settings?
 accountaccess
+accounthelp
 accounts
 accounts/ClientAuth
 accounts/ClientAuth
@@ -7281,7 +7278,6 @@ amp$
 amp-proxy
 amp-tealium
 amp/
-amp/authorize
 amphtml
 amphtml/blogs/test
 amphtml/classic-apps
@@ -7347,12 +7343,12 @@ api.html?
 api.php
 api/
 api/?
-api/attachments/*http://feeds.feedburner.com/robots.txt
+api/attachments/*http://docs.wixstatic.com/robots.txt
 api/discovery
 api/editionurlgenerator
 api/eventlogging
 api/html
-api/http://google.ru/robots.txt
+api/http://kobo.com/robots.txt
 api/internal/*/location
 api/internal/analytics/songs
 api/jobPostings/jobs
@@ -7427,7 +7423,7 @@ appchart
 appdynamics-wem.cisco.com
 appealed-pin
 apple-app-site-association
-apple-app-site-associationhttp://google.co.in/robots.txt
+apple-app-site-associationhttp://fonts.google.com/robots.txt
 apple-news
 appliance
 applicant-eligibility
@@ -8635,7 +8631,7 @@ blog/*?text=
 blog/*sitemap.xml$
 blog/*utm_source=
 blog/.
-blog/.html.*http://onlinelibrary.wiley.com/robots.txt
+blog/.html.*http://salon.com/robots.txt
 blog/?p=
 blog/?s=
 blog/Neuer_Beitrag.html
@@ -8763,7 +8759,7 @@ bookmark_mana2
 bookmarklet
 bookmarks
 bookmarks/widget
-bookmarkshttp://whatsapp.com/robots.txt
+bookmarkshttp://ed.ted.com/robots.txt
 books
 books/NBK
 books/about
@@ -9682,7 +9678,7 @@ cdn-cgi
 cdn-cgi/
 cdn-cgi/bm/cv
 cdn-cgi/challenge-platform
-cdn-cgi/http://focus.de/robots.txt
+cdn-cgi/http://mediapost.com/robots.txt
 cdn-cgi/rum
 cdn-cgi/zaraz/s.js
 cdn-cgi/zaraz/t
@@ -9709,7 +9705,7 @@ cgi-bin/homelandsecurity
 cgi-bin/loginpage
 cgi-bin/state
 cgi-bin/wx
-cgi-binhttp://youtube.de/robots.txt
+cgi-binhttp://ifttt.com/robots.txt
 cgi-local
 cgi-perl
 cgi-registry
@@ -10054,7 +10050,7 @@ co/users
 co/ventureone
 code
 code/*
-code/http://arxiv.org/robots.txt
+code/http://w3techs.com/robots.txt
 codeCoverage
 codesamples/
 coin*mid=
@@ -10266,7 +10262,7 @@ conflagexp
 connect
 connect/@
 connect/help/search-results.html
-connect/http://imgur.com/robots.txt
+connect/http://blog.twitter.com/robots.txt
 connect/social-intranet.html
 connections
 connections/recent
@@ -10384,8 +10380,9 @@ content/dam/ircc/documents/pdf/francais/trousses/form/imm5476
 content/dam/ircc/documents/pdf/francais/trousses/form/imm5644
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0002
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0004
-content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://classroom.google.com/robots.txt
+content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://canalplus.fr/robots.txt
 content/dam/ircc/documents/pdf/francais/trousses/guides/guide-0142-avion-afghanistan-voie-acces-rp.pdf
+content/dam/redcross/lp
 content/dam/web/da_dk/www
 content/dam/web/de_ch/www
 content/dam/web/de_de/www
@@ -10422,6 +10419,7 @@ content/ipad
 content/pdf/10.1007%2F978-94-6209-446-8_8.pdf
 content/pdf/bbm
 content/pdf/bfm
+content/redcross
 content/ru_ru/
 content/search
 content/telegraph
@@ -12375,7 +12373,7 @@ events/friday-rush$
 events/fsi-cloud-symposium-nyc.html
 events/rss
 evernote/like.php
-evite/*http://avexnet.or.jp/robots.txt
+evite/*http://dw.com/robots.txt
 examples/*
 exclusive-offers
 exec/obidos/account-access-login
@@ -12661,7 +12659,7 @@ files/techabuse-welsh.pdf
 files/techabuse.pdf
 fileview
 filing/free-file-fillable-forms/free-file-fillable-forms-support
-filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://freepik.com/robots.txt
+filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://is.gd/robots.txt
 filter
 filter/tips
 filter/tips
@@ -13079,7 +13077,7 @@ fy-NL/search
 fy-NL/user
 fy-NL/users
 g
-g/4arE9Chttp://caniuse.com/robots.txt
+g/4arE9Chttp://flickr.com/robots.txt
 g/api
 g00
 g01
@@ -13264,7 +13262,9 @@ generated_files
 genre
 geo/download
 geo/tools
+geocoder
 geoip.php
+geolocation
 geooverride
 get
 get$
@@ -13305,7 +13305,7 @@ gh/news
 gh/useracquisition
 gh_jid=
 gif
-gifshttp://blog.livedoor.jp/robots.txt
+gifshttp://gist.github.com/robots.txt
 gift-redemption
 gift-terms
 gift_card=
@@ -13332,7 +13332,7 @@ glass/images/commerce
 global
 global$
 global-consumer-survey/tool
-global-infrastructure/services/get-async/http://mentalshinri.net/robots.txt
+global-infrastructure/services/get-async/http://baidu.com/robots.txt
 global-master
 global-nav-esi
 global/RU/
@@ -13657,7 +13657,7 @@ ha/search
 ha/user
 ha/users
 hackathons/reinvent-2013/
-hacker/*/followhttp://sec.gov/robots.txt
+hacker/*/followhttp://hackerone.com/robots.txt
 hacker/*/unfollow
 hair-transplant-services
 handler
@@ -14057,38 +14057,39 @@ http
 http-checker
 http2_data
 http:
-http://1.gravatar.com/robots.txt
+http://2.bp.blogspot.com/robots.txt
+http://about.google/robots.txt
+http://amazon.de/robots.txt
+http://amzn.to/robots.txt
+http://api.whatsapp.com/robots.txt
+http://app.box.com/robots.txt
+http://bbc.com/robots.txt
 http://bcn.cl/robots.txt
 http://behance.net/robots.txt
-http://bloomberg.com/robots.txt
-http://developer.apple.com/robots.txt
-http://docs.wixstatic.com/robots.txt
-http://fastcompany.com/robots.txt
-http://fb.com/robots.txt
-http://fcc.gov/robots.txt
-http://fda.gov/robots.txt
-http://gigaom.com/robots.txt
-http://google.com.br/robots.txt
-http://huffingtonpost.com/robots.txt
-http://i2.wp.com/robots.txt
-http://ipsos.com/robots.txt
-http://kobo.com/robots.txt
-http://mediafire.com/robots.txt
-http://meyerweb.com/robots.txt
-http://neilpatel.com/robots.txt
-http://nicovideo.jp/robots.txt
-http://nydailynews.com/robots.txt
-http://online.wsj.com/robots.txt
-http://php.net/robots.txt
-http://phys.org/robots.txt
-http://pkp.sfu.ca/robots.txt
-http://platform.twitter.com/robots.txt
-http://podcasts.google.com/robots.txt
-http://s3-eu-west-1.amazonaws.com/robots.txt
-http://salon.com/robots.txt
-http://sourceforge.net/robots.txt
-http://support.office.com/robots.txt
-http://video.google.com/robots.txt
+http://census.gov/robots.txt
+http://euronews.com/robots.txt
+http://extension.purdue.edu/robots.txt
+http://graphisoft.com/robots.txt
+http://hubspot.com/robots.txt
+http://ie.edu/robots.txt
+http://linkedin.com/robots.txt
+http://mrisoftware.com/robots.txt
+http://nationalgeographic.com/robots.txt
+http://news.cnet.com/robots.txt
+http://news.yahoo.com/robots.txt
+http://npmjs.com/robots.txt
+http://patreon.com/robots.txt
+http://producthunt.com/robots.txt
+http://raiplay.it/robots.txt
+http://redbubble.com/robots.txt
+http://support.apple.com/robots.txt
+http://techradar.com/robots.txt
+http://tiny.cc/robots.txt
+http://torproject.org/robots.txt
+http://translate.google.com/robots.txt
+http://upload.wikimedia.org/robots.txt
+http://vogue.com/robots.txt
+http://yelp.com/robots.txt
 hu
 hu-*/
 hu-hu/
@@ -14776,7 +14777,7 @@ intcmp=
 integrate/
 integromedb
 intelligencer/amp
-interactive*http://store.cdbaby.com/robots.txt
+interactive*http://fcc.gov/robots.txt
 interactive/reading-preference
 interface/pda
 intern/service/suche
@@ -14831,6 +14832,7 @@ inviteMany
 invited
 invites/buy
 invoice
+invoice-central
 invoice/cn/
 invoice/de/
 invoice/download
@@ -15630,6 +15632,7 @@ language?
 languages/404_en.htm
 lapi
 larbin
+lat=
 latest
 latest-playlist
 latest-talk
@@ -15660,7 +15663,7 @@ layout/set/txt_pq
 layout/set/xml
 layouts
 lb
-leadingtogether/http://icelandair.com/robots.txt
+leadingtogether/http://validator.w3.org/robots.txt
 leaf
 learn
 learn-noperf
@@ -16351,7 +16354,7 @@ media/tourism-content
 media/travelpod-roll.flv
 media/tv/channel/credits
 media/tyfo*
-media/tyfo*http://fbi.gov/robots.txt
+media/tyfo*http://statcounter.com/robots.txt
 media/unesco
 media/vr-geo-photo-j
 media/vr-geo-photo-l
@@ -17632,11 +17635,12 @@ node/*/edit
 node/*/nodequeue
 node/add
 node/add/
+node=
 noflashhtml
 nofollow=1
 nojs=
 nonConfig/api/wpa
-nonce="NHtxzoWap88jROl2H7MNyA">if(google.cis
+nonce="P3xkYXxe3DrQqkPFcGX0Lw">if(google.cis
 nonprofits/account
 nonstyleishida/videolist-*.html
 nonstyleishida/videolist.html
@@ -17670,6 +17674,7 @@ npost/
 nqrwns
 nrdapp
 ns/
+ns/apology
 ns/hydra/console/?
 ns/local_news
 nsj/analytics
@@ -17859,6 +17864,7 @@ okxbet.com
 oma
 omgilibot
 on
+on/demandware.store
 onboard
 onboarding
 one/help/search-results.html
@@ -18560,7 +18566,7 @@ pmc/publisherportal/download
 pmc/publisherportal/journalmanager
 pmc/tocrender.fcgi
 pmc/utilities
-pmc='{\x22aa\x22:{},\x22abd\x22:{\x22abd\x22:false,\x22deb\x22:false,\x22det\x22:false},\x22async\x22:{},\x22attn\x22:{},\x22bgd\x22:{\x22ac\x22:true,\x22as\x22:true,\x22at\x22:0,\x22ea\x22:true,\x22ed\x22:0,\x22ei\x22:true,\x22el\x22:true,\x22ep\x22:true,\x22er\x22:true,\x22et\x22:0,\x22eu\x22:false,\x22li\x22:false,\x22wl\x22:false},\x22cdos\x22:{\x22cdobsel\x22:false},\x22csi\x22:{},\x22d\x22:{},\x22foot\x22:{},\x22gf\x22:{\x22pid\x22:196},\x22gwc\x22:{},\x22hsm\x22:{},\x22jsa\x22:{\x22csi\x22:true,\x22csir\x22:100},\x22kyn\x22:{},\x22lli\x22:{},\x22mb4ZUb\x22:{},\x22pHXghd\x22:{},\x22sb_wiz\x22:{\x22onf\x22:\x22EAEyiAEKHAoaV2FzIG1hY2h0IGVpbmUgUm9ib3RzIHR4dD8KKAomV28gbGllZ3QgZGllIFJvYm90cyB0eHQgYmVpIFdvcmRQcmVzcz8KOwo5V2FydW0gaXN0IGVzIHdpY2h0aWcsIGVpbmUgcm9ib3RzLnR4dC1EYXRlaSB6dSBlcnN0ZWxsZW4_EOQC\x22,\x22scq\x22:\x22\x22,\x22stok\x22:\x22ZsYE9eHjT_Rt7SDyZKUroeo4c8g\x22},\x22sf\x22:{},\x22sonic\x22:{},\x22spch\x22:{\x22ae\x22:\x22\\u00dcberpr\\u00fcfe
+pmc='{\x22aa\x22:{},\x22abd\x22:{\x22abd\x22:false,\x22deb\x22:false,\x22det\x22:false},\x22async\x22:{},\x22attn\x22:{},\x22bgd\x22:{\x22ac\x22:true,\x22as\x22:true,\x22at\x22:0,\x22ea\x22:true,\x22ed\x22:0,\x22ei\x22:true,\x22el\x22:true,\x22ep\x22:true,\x22er\x22:true,\x22et\x22:0,\x22eu\x22:false,\x22li\x22:false,\x22wl\x22:false},\x22cdos\x22:{\x22cdobsel\x22:false},\x22csi\x22:{},\x22d\x22:{},\x22foot\x22:{},\x22gf\x22:{\x22pid\x22:196},\x22gwc\x22:{},\x22hsm\x22:{},\x22jsa\x22:{\x22csi\x22:true,\x22csir\x22:100},\x22kyn\x22:{},\x22lli\x22:{},\x22mb4ZUb\x22:{},\x22pHXghd\x22:{},\x22sb_wiz\x22:{\x22onf\x22:\x22EAEyYQocChpXYXMgbWFjaHQgZWluZSBSb2JvdHMgdHh0PwooCiZXbyBsaWVndCBkaWUgUm9ib3RzIHR4dCBiZWkgV29yZFByZXNzPwoUChJXYXMgaGVpw590IFJvYm90cz8Q5AI\x22,\x22scq\x22:\x22\x22,\x22stok\x22:\x22motd1GHhqY3s4OAGBmZO01V0MuE\x22},\x22sf\x22:{},\x22sonic\x22:{},\x22spch\x22:{\x22ae\x22:\x22\\u00dcberpr\\u00fcfe
 pmcomment/
 pmm-templates
 pmo/contact/en/staff
@@ -19573,6 +19579,7 @@ related
 related-books
 related_community.pl
 related_tag
+relation=
 relations.php
 releasenotes/item/
 relevance_feedback
@@ -20203,6 +20210,7 @@ search-console/scfe
 search-effectiveness.html
 search-people?
 search-results
+search-the-red-cross-site
 search.html
 search.jsp
 search.php
@@ -20384,7 +20392,7 @@ section/3
 section/36
 section/6
 section/8
-section/9http://w3c.github.io/robots.txt
+section/9http://issuu.com/robots.txt
 sectionized
 sections
 sections$
@@ -20497,7 +20505,6 @@ server-status
 server/banners
 server/body-include
 server/bottom-notes
-server/docs/search
 server/footer
 server/fs-gang
 server/generic
@@ -20756,7 +20763,7 @@ shows/
 showtag
 showtime/.requestinvite.html.swp
 showtime/help/search-results.html
-showtimes/location/*http://ad.doubleclick.net/robots.txt
+showtimes/location/*http://img.youtube.com/robots.txt
 shr*
 shutterstock
 si
@@ -22015,7 +22022,7 @@ tag
 tag-manager
 tag-manager-360
 tag/
-tag/*+http://bizjournals.com/robots.txt
+tag/*+http://discogs.com/robots.txt
 tag/*/a
 tag/*/a/
 tag/*/b
@@ -22083,6 +22090,9 @@ tags/newsletters
 tags/topic
 takaoka-saki-blog/videolist-*.html
 takaoka-saki-blog/videolist.html
+take-a-class/asspsafetytraining
+take-a-class/hidden
+take-a-class/participant-materials
 takedown
 talent/matches
 talk
@@ -22246,7 +22256,7 @@ test5
 test6
 testcases
 testfl
-testhttp://pca.st/robots.txt
+testhttp://moz.com/robots.txt
 testing
 testpage-forhomepage
 tests/*
@@ -22487,6 +22497,9 @@ tr/user/*/shoutbox?
 tr/users
 tr/v1
 tr_tr/sablonlar/M
+trace
+traces/page
+traces/tag
 track
 track.gif
 trackback
@@ -22531,7 +22544,7 @@ transmute/static-2.6
 transmute/static-2.8
 transmute/static-2.9
 transparency
-transparency/http://producthunt.com/robots.txt
+transparency/http://anchor.fm/robots.txt
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14885
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14895
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14896
@@ -22591,6 +22604,7 @@ trends/topics
 trends/viz?
 trends?
 trends_tpt
+tributes
 trips/upcoming
 trips/v1
 trivia
@@ -23287,6 +23301,7 @@ user/*/library
 user/*/library/music
 user/*/shoutbox
 user/*/shoutbox?
+user/*/traces
 user/*/track
 user/*/track?page=
 user/*?*redirect=
@@ -23720,6 +23735,7 @@ watch_fragments_ajax
 watch_popup
 watch_queue_ajax
 watson/assets/pdfs/reg
+way=
 wc
 wcs
 wcsstore
@@ -24726,9 +24742,9 @@ wp-admin
 wp-admin/
 wp-admin/admin-ajax.php
 wp-admin/admin-ajax.php
-wp-admin/admin-ajax.phphttp://dictionary.com/robots.txt
-wp-admin/admin-ajax.phphttp://jupiterbroadcasting.com/robots.txt
-wp-admin/admin-ajax.phphttp://roh.org.uk/robots.txt
+wp-admin/admin-ajax.phphttp://aljazeera.com/robots.txt
+wp-admin/admin-ajax.phphttp://fool.com/robots.txt
+wp-admin/admin-ajax.phphttp://openstreetmap.org/robots.txt
 wp-admin/load-scripts.php
 wp-admin/load-styles.php
 wp-adv
@@ -24795,7 +24811,7 @@ wp-content/themes/wired/assets/images/wired_insider_1118_1000x190.jpg
 wp-content/themes/wired/assets/images/wired_video_1000x190.jpg
 wp-content/uploads
 wp-content/uploads/
-wp-content/uploads/wp-import-export-lite/http://healthaffairs.org/robots.txt
+wp-content/uploads/wp-import-export-lite/http://rebrand.ly/robots.txt
 wp-dyn
 wp-includes
 wp-includes*.css
@@ -24929,7 +24945,7 @@ your-shop
 yourAccount
 youraccount
 youreurope/businessACC
-youreurope/businessDEV/http://un.org/robots.txt
+youreurope/businessDEV/http://europarl.europa.eu/robots.txt
 youreurope/citizensACC
 youreurope/citizensDEV
 yours
@@ -25155,6 +25171,7 @@ zz
 ~guido/orlijn
 1
 1*?
+1+1+1+*
 1/hi/education/3826037.stm
 1/hi/england/1678616.stm
 1/hi/england/beds/bucks/herts/3663494.stm
@@ -25279,7 +25296,7 @@ zz
 8618
 9742
 9744
-101612.rsshttp://explore.org/robots.txt
+101612.rsshttp://c-span.org/robots.txt
 1010203
 1014943
 1027454