[Github Action] Automated trickest wordlists update.

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt

@@ -31,6 +31,10 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 .well-known
 .well-known/assetlinks.json
 .xml
+/*/*/_tools
+/*/*/more_ideas
+/*/_tools
+/*/activity
 /*/branches
 /*/commits
 /*/compare
@@ -38,8 +42,11 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /*/edit$
 /*/forks
 /*/graphs
+/*/group
+/*/invite
 /*/issues/new
 /*/issues/search
+/*/more_ideas
 /*/network
 /*/projects
 /*/pulse
@@ -49,6 +56,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /*/watchers
 /*source=
 /.git
+/?z=1
 /B00M3E1HJY
 /B00M3E1NYI
 /B00M3E1Q5Y
@@ -67,6 +75,16 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /Search/
 /_
 /_/rsrc
+/_activities
+/_activity
+/_community
+/_created
+/_followers
+/_following
+/_quick_saves
+/_saved
+/_shop
+/_shopping_list
 /album/*/*?i=
 /api
 /apis
@@ -80,6 +98,8 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /comments/*?*sort=
 /commits/*?author
 /commits/*?path
+/communities
+/discovered
 /docs/product
 /download
 /download/Browse.aspx?
@@ -104,6 +124,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /layout/
 /leads/*/jsplayer_show_lead_form.html
 /leads/*/show_form.html
+/likes
 /listing/*/favoriters
 /lookup?
 /mediaapi/
@@ -111,15 +132,20 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /music/*/Search/
 /music/*/search/
 /newsearch/
+/organize
 /people/*/circle
 /people/*/favorites
 /pinned_flashes
+/pins
 /podcast/*/*?i=
 /print$
+/profile-about
 /raw
 /revisions
 /rg*/mediaviewer/rm*/tr
 /rss
+/scheduled-pin
+/scheduled-pins
 /search
 /search$
 /search.compact$
@@ -141,11 +167,15 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /store/d/groove-music-pass/cfq7ttc0k5dq/0001
 /store/locations/store-locations-sitemap.xml
 /store/music/groove-music-pass/*
+/story_pins
+/tagged
 /tarball
 /tree
+/tried
 /tv-season/*/*?i=
 /unsubscribe/
 /video-
+/video_pins
 /wal/
 /wiki/Special:
 /wiki/index.php/Help
@@ -214,6 +244,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?order=
 ?pt1=true$
 ?s=
+?show_error=true
 ?tab=achievements&achievement=
 ?utm_*=
 ?w=address
@@ -300,6 +331,8 @@ WebObjects/MZStoreElements.woa
 [_0-9a-zA-Z]+/status/[0-9]+/likes
 [_0-9a-zA-Z]+/status/[0-9]+/retweets
 _
+_/_
+_/_/pixel
 _/aihub_pod
 _/ain/_/js
 _/alkali-learn
@@ -361,6 +394,8 @@ _next/gtm
 _next/jwt
 _next/title
 _next/viewer
+_ngjs/resource
+_ngjs/resource/*/get
 _nimbusJson.json
 _nimbusJsonp.js
 _nimbusJsonpStatic.js
@@ -466,6 +501,7 @@ ad_spice
 ad_syndication_user_tracking
 ad_visibility
 addContacts
+addboard
 addressBookExport
 admin
 adredir?
@@ -487,6 +523,7 @@ adwords/proposal
 adx/bin
 affil
 affil/pal
+age-appeal
 aihub/tfhub
 ajax
 ajax/follow_info
@@ -524,6 +561,7 @@ api_access_key=
 app*_
 app*form_id=
 app/updates
+appealed-pin
 apple-app-site-association
 apps
 apps*&tags=
@@ -693,6 +731,7 @@ auth/verifychallenge
 author/following
 author=
 authwall
+autocomplete
 away.php
 b?*node=7454898011
 b?*node=7454917011
@@ -806,7 +845,11 @@ blogs/test
 blogsearch
 blogsearch_feeds
 bn
+board
+board_activity
+board_activity_comment
 bookmark?
+bookmarklet
 bookmarks
 books
 books/about
@@ -828,17 +871,38 @@ books?*zoom=1
 books?*zoom=5
 booksrightsholders
 boutique/sitemaps
+branded-pins
 browse/reviews/recent
+business/ad_create
 business/adshare
 business/adshare/faqs
 business/blahblah
+business/business-access
+business/business-hierarchy
+business/business-manager
+business/catalogs
+business/commerce-integrations
+business/community-information
+business/convert
+business/create
+business/create-linked-account
 business/feedback
+business/hub
+business/invite
 business/leads/jsplayer_show_lead_form.html
 business/leads/show_form.html
 business/learning/blog
+business/onboarding
+business/pin_create
 business/premium/is_payment_pending
 business/promotion
+business/referral
 business/sales/blog
+business/site
+business/switch-account
+business/top-ads
+business/update
+business/verified-merchant/status
 buttonlite.js
 buyside-e2e-monitor
 buzz/post
@@ -1029,6 +1093,7 @@ checkout
 checkout/package
 checkoutnow
 checkpoint
+christmas-spotlight
 chrome/browser/mobile/tour
 chrome/crlset
 chrome/webstore/images
@@ -1051,6 +1116,7 @@ clickthru
 client_errors
 clima/ets
 clima/sites/registry
+close-account
 cloud-launcher/sitemap
 cloudsourcetools/images
 cms_content
@@ -1071,6 +1137,7 @@ commercesearch/images
 commercesearch/services
 commission_2010-2014/katainen
 commits?author=
+communities
 communities/bin.aspx?
 communities/blogs/PortalResults.mspx?
 communities/eventdetails.mspx?
@@ -1086,25 +1153,33 @@ community/category/*
 community/comments
 community/feed
 community/trackback
+community_comment
+community_post
 community_s
 companyDir
 compare
 compare/*/apply
 compressiontest
 config.js
+connect
 connections
 connector
 consumers/archive
 contact_me
 contactus
 content
+content-claiming
 context_referrer=gig_page
 context_referrer=user_page
 control
+conversation
 conversations
+convert-business
+convert-personal
 coop/directory
 coop/manage
 copy-conductor/crawled
+copyright-reporting
 corgi$
 couchmode
 counter/*?
@@ -1112,7 +1187,13 @@ counters/client-screenview
 coupons/visit
 covid19
 covid19/mobility
+create_pin
 creative-europe/404_en.htm
+creative-studio
+creative_editor
+creator-spotlight
+creator_hub/_earn
+creator_hub/_partnerships
 creatorhub
 cse/home
 cse/manage
@@ -1121,7 +1202,9 @@ cse?
 csp
 csp_block
 csp_report_only
+csrf_error
 ct
+ct.html
 ct/log_list
 culturalinstitute/images
 culturalinstitute/sitemaps
@@ -1231,7 +1314,9 @@ de/teams
 de/transactions
 de/treasury
 de/your
+deactivate-account
 default
+delete_post
 delight
 dell/cities-transformed
 dell/cloud-future-of-medicine
@@ -1281,6 +1366,7 @@ digital-single-market/taxonomy
 digital-single-market/users
 directory
 directory/all
+discover
 discussion
 discussion/comment-permalink
 discussion/edit-profile
@@ -1378,6 +1464,8 @@ ebooks?*zoom=1
 ebooks?*zoom=5
 ece
 eclas
+edit
+editorial/*
 edu/cs4hs
 education/404_en.htm
 edurec
@@ -1389,6 +1477,7 @@ elite?
 email
 email-unsubscribe
 email_actions
+email_verification_error
 embed
 embed/feed/update
 employment_social/anticipedia/xwiki/bin/attach
@@ -1443,6 +1532,7 @@ en/microsoft-viva/
 en/security
 en/security/
 endorsements
+engagement
 enterprise-jobs
 enterprise-partner-search/sitemaps
 environment/ets
@@ -1545,6 +1635,8 @@ exec/obidos/subst/marketplace/sell-your-collection.html
 exec/obidos/subst/marketplace/sell-your-stuff.html
 exec/obidos/subst/partners/friends/access.html
 exec/obidos/tg/cm/member
+explore
+explore/*/?p=
 explore/*/lightbox
 external
 external/overture
@@ -1554,6 +1646,7 @@ facebook
 facebook_connect
 faves-i_love_u_get_away_from_me
 favorite
+fb.html
 fe=true
 feature/sf-test
 feed
@@ -1633,6 +1726,8 @@ flights
 flutter-canvaskit
 folder
 follow
+follow_board
+follow_user
 following
 food/archive
 food/favourites
@@ -1743,6 +1838,8 @@ globalmarketfinder
 globalmarketfinder/*.html
 go/awards
 gold?
+google_gdn.html
+google_search.html
 googlebooks/images
 goto
 gp
@@ -1853,6 +1950,7 @@ gview
 gwt
 h5games_gamecenter/images
 handmade
+hanukkah-spotlight
 hashtag/*?src=
 hd
 header/channels
@@ -1927,15 +2025,18 @@ hk-en/teams
 hk-en/transactions
 hk-en/treasury
 hk-en/your
+holiday-spotlight
+holidays-celebrations
 home
+homefeed
 homepage-video-test
 hosted/images
 hotelfinder/rpc
 hotels/rpc
 hp/video
 hpc/*/supported-applications.aspx?
-http://archive.org/robots.txt
-http://flickr.com/robots.txt
+http://soundcloud.com/robots.txt
+http://translate.google.com/robots.txt
 hub?category=
 hz/contact-us/ajax/initiate-trusted-contact
 hz/help/contact/*/message/$
@@ -1946,6 +2047,9 @@ i/u
 ia
 icons
 id-id.www.cloudflare.com
+idea-ads-tool
+idea-pin-builder
+idea-pin-invite
 identity/accountsettingsactivityweb
 identity/accountsettingssyaf
 ie*/listing/*/favoriters
@@ -2116,11 +2220,14 @@ info/customerror.htm*
 info/smart404.asp*
 info/tech-feedback
 inproduct_help/chatsupport
+install-shuffles
 internalPref
 intl/*/about/views
 intlkb/
+invite
 inviteFromProfile
 inviteMany
+invited
 invites/buy
 invoice/est
 invoice/estimate/buyerView
@@ -2210,6 +2317,8 @@ johnhancock
 johnhancock/in-a-lifetime-healthcare
 johnhancock/in-a-lifetime-sustainability
 johnhancock/in-a-lifetime-work
+join/?next=
+join/signup/?next=
 jp*/listing/*/favoriters
 jp*/people/*/circle
 jp*/people/*/favorites
@@ -2272,6 +2381,7 @@ knowmore
 ko-kr.www.cloudflare.com
 ko-kr/feedback
 ko-kr/lp
+kwanzaa-spotlight
 labs/popgadget/search
 lamda/images
 landing/cmsnext-root
@@ -2306,10 +2416,12 @@ learning/search?
 learning/settings
 learning/settings$
 learning/settings?
+lens-search
 li/track
 library/devices
 licensing-syndication
 limited-release
+linked
 linkedin
 list
 list/ls*/_ajax
@@ -2318,6 +2430,7 @@ listing/*/stubs
 lite
 lite.publication.PublicationRequestFulltextPromo.requestFulltext.html
 live
+live-session
 live_chat
 lm/R1XIHQVKXSKBNJ
 lm/R3HQ5WJSZK6QSO
@@ -2348,6 +2461,7 @@ location_suggest
 lochp?
 login
 login$
+login/?next=
 login?*=
 logo-maker/brief
 logo-maker/choose-variation
@@ -2526,6 +2640,7 @@ native
 nb-no.www.cloudflare.com
 nbubumblebee
 network
+new_invite
 news$
 news/$
 news/0
@@ -2537,6 +2652,7 @@ news/test
 news/test-liveblog
 news?viewArticle
 newsArticle
+news_hub
 newsletter
 newsroom/dae
 newsroom/notifications
@@ -2644,6 +2760,7 @@ no-en/treasury
 no-en/your
 nonprofits/account
 not_recommended_reviews/preview
+notifications
 ns/hydra/console/?
 nus-trk
 nwshp$
@@ -2696,8 +2813,12 @@ nz/teams
 nz/transactions
 nz/treasury
 nz/your
+oauth/line
+oauth/login
 oauth/v2
 oauth2/v2
+oauth_error
+offsite
 og
 onebox/media/sports/logos
 onebox/media/sports/photos/soccer/wc2018
@@ -2728,8 +2849,11 @@ pages?oid=-
 pagespeed/insights/ui/logo
 pantheon
 pantheon-platform
+parental-consent/confirmation
+parental-passcode
 participate/conferences.xml
 partners
+password/reset
 password_expired
 patent-search/frontend
 patents/download
@@ -2759,8 +2883,39 @@ photos/gbachelie
 photos/i_love_u_get_away_from_me
 photos/tags/*/page
 photos/youpy
+pin-builder
+pin-communities
+pin-creation-tool
+pin-editor
+pin/*/activities
+pin/*/activity
+pin/*/analytics
+pin/*/brand-catalog
+pin/*/comments
+pin/*/email
+pin/*/feedback
+pin/*/idea-pin/related-products
+pin/*/likes
+pin/*/promote
+pin/*/reaction
+pin/*/related-products
+pin/*/repin
+pin/*/repins
+pin/*/sent
+pin/*/share
+pin/*/shop-the-look
+pin/*/tag
+pin/*/takes
+pin/*/visual-search
+pin/create
+pin/find
+pin/find?url=
+pin_shop
 ping?
 pingback
+pinterest-predicts
+pinterest-wellbeing
+pinterestwellbeing
 pl*/listing/*/favoriters
 pl*/people/*/circle
 pl*/people/*/favorites
@@ -2811,6 +2966,7 @@ pl/teams
 pl/transactions
 pl/treasury
 pl/your
+place
 plan=
 platform-telemetry
 play/store/web
@@ -2840,6 +2996,7 @@ ppob?
 pqa
 preference
 preferences
+prefs
 present
 presentation
 prettyearth/assets
@@ -2850,6 +3007,8 @@ private
 pro
 pro-resources
 pro/about
+product-catalogs
+product-feed
 product-reviews/B0069IY63Y
 productPurchaseDone
 product_
@@ -2926,6 +3085,7 @@ publications
 publicliterature.PublicLiterature.search.html
 purchases
 puzzles/leaderboards/invite
+pw
 pwapi-proxy/pwproxy
 pwapiv2
 px.gif
@@ -2933,6 +3093,7 @@ q=
 qnasearch?
 qr
 quality_form?
+quiz
 r
 r/*.rss
 r/*/comments/*.rss
@@ -2960,6 +3121,7 @@ random
 rd/search
 rd/search$
 rd/search?
+recently-viewed
 recharge/dashboard/car
 recharge/dashboard/static
 recipes/search
@@ -2977,6 +3139,8 @@ refer
 refer/pal
 referrals
 referrer=
+refresh_stored_accounts
+refresh_token
 register
 registration
 registry/wishlist
@@ -2987,10 +3151,14 @@ report
 report-abuse?report=
 report=
 report_abuse.gne
+reports-and-violations
+request-data
 requestList
 requests
 research/participants/cofund
 reservations/*/notifyme
+resource
+resource/*/get
 results
 retail
 retail/old
@@ -3024,6 +3192,7 @@ s2/sitemaps
 s2/static
 s3
 s?
+safe-redirect
 safebrowsing/diagnostic
 safebrowsing/report_badware
 safebrowsing/report_error
@@ -3121,6 +3290,7 @@ searchhistory
 sections
 secure
 seller_onboarding
+sendHeaderEarly
 send_to_friend
 sendarticle
 services
@@ -3196,6 +3366,7 @@ shop/browse/overlay
 shop/browse/ribbon
 shop/iphone/payments/overlay
 shop_name_search_service
+shop_pins
 shopping-content-api/product_images
 shopping/product
 shopping/ratings/account/metrics
@@ -3204,7 +3375,7 @@ shopping/reviewer
 shopping/seller
 shopping/suppliers/search
 shopping?
-showtimes/location/*http://apis.google.com/robots.txt
+showtimes/location/*http://ted.com/robots.txt
 sidewiki/entry
 signature
 signin/inject
@@ -3489,11 +3660,13 @@ smart/buttons
 social/photosui/images
 social/photosui/sitemap
 social/social-security-directory/cai
+socialmanager
 sounds/player
 sounds/player$
 sounds/search
 sounds/search$
 sounds/search?
+source
 source=
 source_repo=
 spain/empresas/
@@ -3515,6 +3688,7 @@ ss
 ss/customer-reviews/lighthouse
 ss/twister/ajax
 sslsingle
+sso
 stadia/gamers
 start
 start_order
@@ -3569,6 +3743,10 @@ storefront
 storepicker
 stores/page/*visitId=
 stories
+story-pin-builder
+story-pin-invite
+story_feed
+story_pins
 studios
 styles/js-on.css$
 submit
@@ -3589,6 +3767,7 @@ support
 support/content
 support/forum/search?
 surprise
+suspension-appeal
 sv-se.www.cloudflare.com
 svc
 svc/crosswords
@@ -3602,12 +3781,14 @@ svc/vertex
 svc/web
 svc/wordle
 svpRecommendations
+sw-shell.html
 swg
 syndicate
 system1/images
 t/terms
 tab=
 tablet
+taggingtool
 tagmanager/containers/xo
 tagmanager/pptm.js
 talk
@@ -3640,8 +3821,10 @@ test
 testpage-forhomepage
 tetro
 th-th.www.cloudflare.com
+thanksgiving-spotlight
 thanx?
 themacallan/worlds-together
+theme
 themes/rss
 things/search
 things/search$
@@ -3716,6 +3899,7 @@ trustedstores/js
 trustedstores/s
 trustedstores/tm2
 trustedstores/verify
+tv
 tv/thumb
 tvschedule
 twitter
@@ -3780,9 +3964,13 @@ uk/teams
 uk/transactions
 uk/treasury
 uk/your
+unauth-profile
+unlink
 unsub
 until=
 updates
+upload-image-lens-history
+upload-lens-image
 uploads
 urchin_test
 us-es*/listing/*/favoriters
@@ -3856,6 +4044,8 @@ uviewer
 v1
 v1/browser-performance
 v1/stats
+v1/urls/count.json
+v3/pidgets/log
 v4
 verify_age
 vi-vn.www.cloudflare.com
@@ -3884,9 +4074,12 @@ watch_queue_ajax
 webmasters/sitemaps/ping?
 webscr
 websearch
+website/confirm
+website/confirm/download
 webstore/search
 webstore/static/*/wall/js
 weekly/signup
+welcome
 whats-next/planes-trains-and-analytics
 whsmiths
 widget_auth.php
@@ -4348,6 +4541,7 @@ xoonboarding
 xoplatform/logger/api/logger
 xorouter
 your
+your-shop
 youth/404_en.htm
 youtube/fonts
 youtube/img/promos

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt

@@ -237,6 +237,8 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 .env
 .eot*************
 .flv$
+.full-text.print$
+.full.print$
 .gif*************
 .git
 .git$
@@ -608,7 +610,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /*/invite
 /*/issues/new
 /*/issues/search
-/*/leaderboard
 /*/metrics
 /*/more_ideas
 /*/network
@@ -5360,6 +5361,7 @@ Help
 HelpContent
 Heritrix
 Home/404Page
+HpImageArchive.aspx
 HubStyle/static-2.309
 HubStyle/static-2.313
 HubStyle/static-2.315
@@ -5543,6 +5545,7 @@ Meo
 MessageRunDetailID=
 MetaURI
 Microsoft.URL.Control
+Mobile/EM/
 Moreover
 Moreoverbot
 MoviesYouveSeen
@@ -5659,6 +5662,7 @@ ProfilesGate
 Project
 PromotionDetailsDisplayView
 PromotionDisplayView
+Proxy.ashx
 QUICKNEWS
 Qinfo
 Qinfo/cams
@@ -5716,6 +5720,7 @@ SOImapPriceDisplayView
 SametimeWebApp
 Scripts
 Search
+Search
 Search/*
 Search/Mail/Public
 SearchNavigationView
@@ -6180,6 +6185,8 @@ ac/
 ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search
 ac2
 academic-subscriptions
+academic/profile
+academic/search
 academy
 academy-free-account
 academy/*/results
@@ -6211,7 +6218,6 @@ account/mypro
 account/password/reset
 account/registration
 account/subscribe
-account/unsubscribe
 account_purchase_history.mhtml
 account_settings$
 account_settings/cancel_membership$
@@ -6263,6 +6269,7 @@ ach/search
 ach/user
 ach/users
 achievements
+aclick
 aclk
 acoustic
 acq/nonprofits
@@ -6530,6 +6537,7 @@ alexa-top-sites/*
 alexa-web-search/*
 alexa/*
 alexaforbusiness/
+alink
 alkali
 all
 all-supported-params
@@ -6553,6 +6561,7 @@ amp
 amp$
 amp-proxy
 amp-tealium
+amp/
 amp/authorize
 amphtml
 amphtml/blogs/test
@@ -6595,7 +6604,6 @@ android/downloads
 android/market
 android/market_images
 annotated
-annotations/*/report_as_abusive
 anon/AdPreview$
 anonymous
 answers
@@ -6612,7 +6620,6 @@ apac/aws-credits-request-confirmation/
 apac/corporate/careers
 apac/news
 api
-api$
 api*?hl=
 api.html?
 api.php
@@ -6627,6 +6634,7 @@ api/http://kobo.com/robots.txt
 api/internal/analytics/songs
 api/jobPostings/jobs
 api/legendary-data/relay
+api/maps/
 api/nextsteps/*
 api/nicorepo/timeline
 api/programas/69730/audios.rss
@@ -6797,7 +6805,6 @@ artist/127231622/radio?autoplay=true
 artist/export_shows_csv
 artist/view_widget_popup
 artists
-artists/*/top_iq_earners
 arts/library/
 artshowcmt/
 as/forums
@@ -7489,7 +7496,6 @@ backgrounds
 backoffice=
 backstage/help/search-results.html
 badges/profile/create
-bagon
 baidu
 ban-me-i-am-a-robot
 bandito
@@ -7737,6 +7743,7 @@ bezahlvorgaenge.html
 bfe/apps/website
 bfl
 bfmp
+bfp/search
 bg
 bg-*/
 bg-bg/
@@ -7798,6 +7805,7 @@ bin
 bin/
 bin/search.go
 bin/search.py
+bing-site-safety
 bingbot
 bipolar-disorder/bipolar-depression-management
 bipolar-disorder/video/bipolar-fatherhood
@@ -7943,6 +7951,7 @@ blogs/blogpost/preview
 blogs/nationals-journal-beta
 blogs/pad$
 blogs/pad/$
+blogs/search/
 blogs/slow-ride
 blogs/technology.atom
 blogs/test
@@ -7962,6 +7971,7 @@ bm/user
 bm/users
 bmajax
 bmind
+bmj_countries
 bn
 bn-BD/forums
 bn-BD/kb/*/discuss
@@ -7987,6 +7997,8 @@ bn/search
 bn/user
 bn/users
 bo
+boacongress2011
+boacongress2012
 board
 boardInviteDeclined
 board_activity
@@ -9153,6 +9165,7 @@ civis/posts
 civis/search
 civis/threads/*in_iframe=1
 civis/threads/*wp_data=
+ck/
 cl
 cl2/feeds
 cl2/ical
@@ -9359,7 +9372,6 @@ comments
 comments
 comments$
 comments/*/likes
-comments/for_profile_page
 comments/like
 comments/quote
 comments/reply
@@ -9419,6 +9431,7 @@ companies/o
 companies/search
 company
 company/*.rss
+company/?s=
 company/add-ons-agreement
 company/index/thank-you
 company/reseller-add-ons-agreement
@@ -9548,6 +9561,7 @@ content
 content-claiming
 content-generator-embed
 content-marketing
+content/*%7Bopenurl%7D
 content/*/async
 content/*/bind
 content/*/cancel
@@ -9811,10 +9825,12 @@ cpro
 cpropub/univercd
 cpspa
 cr
+cr$
 cr/
 cr/$
 cr/corporate/contact
 cr/news
+cr?
 craft
 crash
 crawler4j
@@ -10969,7 +10985,6 @@ email=
 email_a_friend
 email_actions
 email_collect
-email_notifications/*/unsubscribe
 email_optout
 email_unsubscribe
 email_verification_error
@@ -11181,6 +11196,9 @@ enterprise/seo
 entertain
 entertainment/article/Minerva-s-horoscope
 entertainment/bravo
+entities/search
+entityexplore$
+entityexplore?
 entrez
 entrez/eutils
 entrez/sutils
@@ -11714,6 +11732,7 @@ fbl/terms.html
 fbpostaction.cms?article=
 fcgi-
 fckeditor
+fd/
 fdbk
 feature/sf-test
 feature_nine_browse-bin=
@@ -12289,6 +12308,8 @@ full
 fullscreen
 fullscreen/ReportLinker
 fun
+fun/api/
+fun/g/
 fundraiser/charity
 fundraiser/search
 fur/forums
@@ -12996,9 +13017,9 @@ hidden
 hidden-content
 highereducation/higheddocs
 highlights
+highwire
 hilfe/fundingdetail.html
 hilfe/live/launch-day/launch-day-show-voting.html
-hilfe/live/launch-day/launch-day-videoupload.html
 hilfe/live/officehour/officehour-link.html
 hims-hard-mints
 hinweis/datenleck
@@ -13179,12 +13200,14 @@ hosted
 hosted/images
 hosting/search
 hotel/index.hi.
+hotel/reviews?
 hotel/us/the-airstream-van.*.html
 hotel_attractions
 hotel_rt_onview
 hotelfinder/rpc
 hotels
 hotels/rpc
+hotels/search
 hotelsonmap.*.json
 hotfix/
 houseads
@@ -13198,6 +13221,8 @@ howto
 howtobuy/
 hp/focus-retrocompatibile$
 hp/video
+hpm
+hpmob
 hr
 hr-*/
 hr-hr/
@@ -13298,11 +13323,13 @@ http://mashable.com/robots.txt
 http://mrisoftware.com/robots.txt
 http://nationalgeographic.com/robots.txt
 http://news.cnet.com/robots.txt
+http://news.yahoo.com/robots.txt
 http://npmjs.com/robots.txt
 http://patreon.com/robots.txt
 http://producthunt.com/robots.txt
 http://raiplay.it/robots.txt
 http://redbubble.com/robots.txt
+http://statcounter.com/robots.txt
 http://support.apple.com/robots.txt
 http://techradar.com/robots.txt
 http://tiny.cc/robots.txt
@@ -13680,11 +13707,16 @@ images/$
 images/*
 images/?
 images/Podcasts
+images/async/?
+images/async?
 images/carBlogsimages
 images/chart_gifs
 images/chart_pdfs
 images/chart_png300
 images/chart_png460
+images/hosted/?
+images/hosted/search?
+images/hosted?
 images/interview_pdfs
 images/li.gif
 images/logging_requests.gif
@@ -13693,8 +13725,12 @@ images/ndtvvideo/*
 images/pad/search?rpt=imageview
 images/pad/search?text=*&source=morda
 images/resume
+images/search%3F
+images/search/?
+images/search?
 images/search?rpt=imageview
 images/search?text=*&source=morda
+images/searchbyimage
 images/showroomimages
 images/smart/$
 images/touch/$
@@ -13705,6 +13741,7 @@ images/usedcarimages
 images/userprofileimg
 images/v$
 images/viewconfig$
+images?
 imagine/
 imesync
 img
@@ -13829,6 +13866,7 @@ inapp
 inbox
 inbox$
 inbox$
+inboxcommerce
 inc5000dev
 include
 include/
@@ -14454,8 +14492,6 @@ journal/famsci
 journal/famsci$
 journal/fis
 journal/fis$
-journal/geometry
-journal/geometry$
 journal/hazardous_matters
 journal/hazardous_matters$
 journal/ijip
@@ -14841,7 +14877,6 @@ latest-playlist
 latest-talk
 latin
 latin_en
-launch-day-show-voting.html
 launch2005/
 law-school/search-results/
 lawlib/archive
@@ -15088,6 +15123,7 @@ loans
 loc/volunteers
 loc=
 local
+local
 local-services-ads
 local/ajax
 local/blogsandcolumns/slow-ride-front
@@ -15407,6 +15443,7 @@ maps/*?lang=uk$
 maps/*?lang=uz$
 maps/*?ll=
 maps/about
+maps/adsendpoint
 maps/api/js
 maps/api/place/js
 maps/api/staticmap
@@ -15476,7 +15513,6 @@ mbr/
 mcd
 mcreator/AppBox.html
 me
-mecha
 media
 media-center
 media-manager
@@ -15510,7 +15546,6 @@ media/popup/
 media/thumbnails/
 media/tv/channel/credits
 media/tyfo*
-media/tyfo*http://statcounter.com/robots.txt
 mediabase
 mediaembed
 medialibrary/de/brief
@@ -15808,6 +15843,7 @@ merchant/directions/*
 merchant/imageUploadPreview/*
 merchant/loc/*
 merchant/moreProducts?*
+merchant/reviews?
 merchant/update/*
 merchantapps
 mes
@@ -16809,6 +16845,7 @@ notificarelacionadas
 notifications
 notifications
 notifications$
+notifications/
 notifications/*
 nova
 now/mt3
@@ -16991,6 +17028,8 @@ offers/nonprofits
 offers/nonprofits$
 offers/nonprofits%
 offers/nonprofits?
+offers/proxy/dealsserver/api/log
+offers/proxy/dealsserver/buy
 offers/referral-terms-conditions
 offers/referral-terms-conditions$
 offers/referral-terms-conditions%
@@ -18076,6 +18115,7 @@ pro-resources
 pro-site
 pro/about
 pro2
+proFile/history?
 proMatch
 proProjects
 proVideoWidget
@@ -18094,6 +18134,7 @@ product-reviews/B0069IY63Y
 product/electrode/api/logger
 product/electrode/api/wpa
 product/recommended
+product/reviews?
 productBrand=
 productId=
 productPurchaseDone
@@ -18148,6 +18189,7 @@ profile/*/followers
 profile/*/following
 profile/*/map$
 profile/downloads_limit
+profile/history?
 profile/license
 profile/payment/getstarted
 profile/view
@@ -18573,6 +18615,7 @@ range_chapter=
 range_subcategory=
 ranking-factors
 ranking_graph
+rapid-responses
 ratefinder/home?cc=US&language=en&locId=express
 rating
 ratings$
@@ -18741,6 +18784,8 @@ research
 research-ai
 research-ai$
 research-credits/data-science-bowl
+research/feed
+research/highlight/feed
 research/participants/cofund
 researchtools/ose
 researchtools/ose/$
@@ -18788,6 +18833,7 @@ resources/xml/sitemaps
 responder
 rest
 results
+results
 results-export-email-by-search-data
 results-export-ids
 results-export-search-by-year
@@ -18839,6 +18885,7 @@ reviews/seller
 reviews/top-reviewers
 reviews_page=
 rewards
+rewardsapp/
 rf=*
 rfcdiff*
 ribbon.php
@@ -19618,6 +19665,7 @@ server-status
 server/banners
 server/body-include
 server/bottom-notes
+server/docs/search
 server/footer
 server/fs-gang
 server/generic
@@ -19671,6 +19719,7 @@ set_cookie.pl
 setbucket
 setprefs
 settings
+settings
 settings/*
 settings/deactivated
 settings/linkaccount
@@ -19802,6 +19851,7 @@ sharing/files
 sharing/folders
 sheet/help/search-results.html
 shelf/user_shelves
+shenghuo
 shenghuo/search?
 shifen/
 shihui
@@ -19812,6 +19862,8 @@ shipping-payment
 shippingassistant
 shippinginformation
 shop
+shop$
+shop/
 shop/*&
 shop/bag
 shop/browse/campaigns/mobile_overlay
@@ -19820,6 +19872,7 @@ shop/browse/ribbon
 shop/global
 shop/iphone/payments/overlay
 shop/search.ep
+shop?
 shop_name_search_service
 shop_pins
 shopfront-api
@@ -19869,7 +19922,7 @@ shows/
 showtag
 showtime/.requestinvite.html.swp
 showtime/help/search-results.html
-showtimes/location/*http://j.mp/robots.txt
+showtimes/location/*http://img.youtube.com/robots.txt
 shr*
 shutterstock
 si
@@ -20412,6 +20465,7 @@ social-connect/send
 social/help/search-results.html
 social/photosui/images
 social/photosui/sitemap
+social/search?
 social/social-security-directory/cai
 social_media
 socialapp/
@@ -20493,11 +20547,6 @@ son/user
 son/users
 song/data
 song/search
-songs/*/apple_music_player
-songs/*/comments
-songs/*/metadata
-songs/*/show_amp_header_play_toggle
-songs/for_profile_page
 sony
 sorry
 sort=
@@ -20524,10 +20573,12 @@ sp=
 sp_url=
 spartan
 spatl
+spbasic
 spbot
 spc-source=
 special-promo/2for1
 specialhandling-payments/
+specialities/*/all-bmj/feed
 specializations-noperf
 specializations-perf
 specials
@@ -20599,6 +20650,7 @@ sports/whatson
 sports_fp_ms
 spp/
 spreadsheet
+spresults
 sprint_wml
 sprint_xhtml
 sprints/help/search-results.html
@@ -20682,6 +20734,7 @@ statement
 static
 static-assets/documents/PwnedPasswordsTop100k.txt
 static-assets/images/collection
+static/
 static/button/button1.html
 static/button/button1.js
 static/button/button2.html
@@ -20798,11 +20851,15 @@ storeclick
 storefront
 storepicker
 storerequest
+stores$
+stores/
 stores/page/*visitId=
 stories
 story
 story-pin-builder
 story-pin-invite
+story/amp/police-chief-accused-of-using-taxpayer-money-to-book-hotels-for-sex-with-female-colleagues-13107270
+story/police-chief-accused-of-using-taxpayer-money-to-book-hotels-for-sex-with-female-colleagues-13107270
 story/twittershare
 story_feed
 story_pins
@@ -21351,6 +21408,7 @@ text
 textedition
 tfemail
 th
+th$
 th-*/
 th-th.www.cloudflare.com
 th-th/
@@ -21373,6 +21431,7 @@ th/solutions/case-studies/
 th/solutions/case-studies/innovators/innovators-eu/
 th/user
 th/users
+th?
 th_th/templates/M
 thank-you
 thank-you-enterprise-demo
@@ -21617,6 +21676,7 @@ translate_static
 translate_suggestion
 translate_t
 translation
+translator/?ref=
 transmute/static-1.85
 transmute/static-2.6
 transmute/static-2.8
@@ -21644,10 +21704,24 @@ travel/
 travel/2013/aug/22/been-there-readers-competition?
 travel/8711559/The-Telegraph-Travel-Awards-2011.html
 travel/clk
+travel/css
+travel/flight/flightSearch
+travel/flight/flightSearchAction
+travel/flight/search/?
+travel/flight/search?
 travel/flights/s
+travel/hotel-search
 travel/hotel/e
+travel/hotel/hotelMiniSearchRequest
+travel/hotel/hotelSearch
+travel/hotels/checkout
+travel/hotels/detail
 travel/hotels/hotel-finder
 travel/hotels/hotel-finder/$
+travel/hotels/search/?
+travel/hotels/search?
+travel/scripts
+travel/secure
 travelnevadahighway95tc
 treasury
 treasury/treasdocuments
@@ -22144,6 +22218,7 @@ ur/user
 ur/users
 urbanag
 urchin_test
+url
 urlinator/static-1.54
 urlinator/static-1.55
 urlinator/static-1.57
@@ -22422,7 +22497,6 @@ users/*/listings
 users/*/replies
 users/*/rss
 users/*/snippets
-users/*/top_artist_iq_rank
 users/*?
 users/*?tab=accounts
 users/*?tab=activity
@@ -22641,6 +22715,7 @@ video/embed-player
 video/embedded
 video/games/tags
 video/iframe
+video/moment-joseph-mccann-arrested-in-a-tree-11878890
 video/mucinex-md
 video/pad
 video/play
@@ -22667,11 +22742,21 @@ video_passthrough
 video_passthrough.php
 videorequest
 videos
+videos/?
 videos/assets
+videos/browsing
 videos/e
+videos/explore
+videos/favorites
+videos/feed
+videos/feedcontent
 videos/get_video_by_uid/
 videos/images
 videos/js
+videos/music
+videos/music/
+videos/music/?
+videos/music?
 videos/search.php
 videos/search.php?filter=*&s=Fight+Compilation$
 videos/search.php?filter=*&s=Fight+Compilation*&start=*$
@@ -22716,8 +22801,13 @@ videos/search.php?s=street+fight$
 videos/search.php?s=street+fight*&start=*$
 videos/search.php?s=woman+fight$
 videos/search.php?s=woman+fight*&start=*$
+videos/search/?
+videos/search?
+videos/trending
+videos/trendingcontent
 videos/viewmeta/
 videos/week
+videos?
 videosearch
 view
 viewReview
@@ -22972,6 +23062,10 @@ whythisad
 widget
 widget-landing-page.html
 widget/*
+widget/cr
+widget/entity/search/?
+widget/render
+widget/snapshot
 widget_auth.php
 widget_chunks
 widgets
@@ -23737,6 +23831,7 @@ word/embed
 wordpress/wp-admin
 wordpress/wp-admin/admin-ajax.php
 work
+work
 work/_
 work/acceptinvitation
 work/account