[Github Action] Automated trickest wordlists update.

.bin/wordlist-updaters/status.json

@@ -3,6 +3,6 @@
         "last_update": 1734894188
     },
     "Trickest wordlist update": {
-        "last_update": 1734861807
+        "last_update": 1734948263
     }
 }

Discovery/Web-Content/CMS/trickest-cms-wordlist/opencart-all-levels.txt

@@ -56598,6 +56598,16 @@ upload/extension/oc_payment_example/install.json
 extension/oc_payment_example/install.json
 oc_payment_example/install.json
 install.json
+upload/extension/oc_payment_example/image/mastercard.svg
+extension/oc_payment_example/image/mastercard.svg
+oc_payment_example/image/mastercard.svg
+image/mastercard.svg
+mastercard.svg
+upload/extension/oc_payment_example/image/visa.svg
+extension/oc_payment_example/image/visa.svg
+oc_payment_example/image/visa.svg
+image/visa.svg
+visa.svg
 upload/extension/oc_payment_example/admin/controller/payment/credit_card.php
 extension/oc_payment_example/admin/controller/payment/credit_card.php
 oc_payment_example/admin/controller/payment/credit_card.php

Discovery/Web-Content/CMS/trickest-cms-wordlist/opencart.txt

@@ -7241,6 +7241,8 @@ upload/install/language/en-gb/install/step_3.php
 upload/install/language/en-gb/install/step_1.php
 upload/install/language/en-gb/upgrade/upgrade.php
 upload/extension/oc_payment_example/install.json
+upload/extension/oc_payment_example/image/mastercard.svg
+upload/extension/oc_payment_example/image/visa.svg
 upload/extension/oc_payment_example/admin/controller/payment/credit_card.php
 upload/extension/oc_payment_example/admin/model/payment/credit_card.php
 upload/extension/oc_payment_example/admin/view/template/payment/credit_card.twig

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt

@@ -1468,8 +1468,8 @@ hp/video
 hpc/*/supported-applications.aspx?
 hpm
 hpmob
-http://npr.org/robots.txt
+http://calendar.google.com/robots.txt
-http://wa.me/robots.txt
+http://ncbi.nlm.nih.gov/robots.txt
 hub?category=
 hz/contact-us/ajax/initiate-trusted-contact
 hz/help/contact/*/message/$
@@ -3080,7 +3080,7 @@ wiki/Wikipedia_Diskussion:Checkuser
 wiki/Wikipedia_Diskussion:Spam-Blacklist-Log
 wiki/Wikipedia_talk%3AAbuse_reports
 wiki/Wikipedia_talk%3AAbuse_response
-wiki/Wikipedia_talk%3AAdministrator_electionshttp://goo.gl/robots.txt
+wiki/Wikipedia_talk%3AAdministrator_electionshttp://policies.google.com/robots.txt
 wiki/Wikipedia_talk%3AAdministrator_recall
 wiki/Wikipedia_talk%3AAdministrator_review
 wiki/Wikipedia_talk%3AAdministrators%27_noticeboard

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt

@@ -3258,7 +3258,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /newsletter
 /newssearch
 /nightly-news
-/no-referrer-when-downgradehttp://globesimregistrations.ph/robots.txt
+/no-referrer-when-downgradehttp://fb.watch/robots.txt
 /no-search-results
 /node
 /node/add
@@ -3903,6 +3903,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /{link}
 0
 0*?
+0;((b=google.c)==null?0:b.cwh)&&google.c.wh<=1&&(e=!0);b=void
 ;jsessionid
 ;jsessionid
 ;preview=
@@ -3928,7 +3929,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*__rmid
 ?*__rmidpage
 ?*ajs_event
-?*apikey*http://vk.com/robots.txt
+?*apikey*http://stupidhackathon.github.io/robots.txt
 ?*app
 ?*aspects=
 ?*attachment_id
@@ -4364,7 +4365,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?q=user
 ?q=user/login
 ?q=user/logout
-?q=user/logout/http://rebrand.ly/robots.txt
+?q=user/logout/http://academic.oup.com/robots.txt
 ?q=user/password
 ?q=user/register
 ?q=vote
@@ -4417,7 +4418,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?showAllGroups=
 ?show_error=true
 ?showcomment*
-?shttp://jigsaw.google.com/robots.txt
+?shttp://smithsonianmag.com/robots.txt
 ?sid
 ?sitrackingid=
 ?slogin
@@ -4545,7 +4546,7 @@ API/user
 API/user/abtest
 API/user/combinedinfo
 API/uxr-survey-management
-API/v2$http://hbr.org/robots.txt
+API/v2$http://uk.linkedin.com/robots.txt
 APIAccessSupport
 AWSEC2/2006-06-26
 AWSEC2/2006-10-01
@@ -4608,7 +4609,6 @@ AddListing
 AddToList
 AddToRegistry
 Admin
-Admin/
 Administration/
 AdsManager
 AdsTest
@@ -4943,6 +4943,7 @@ HotelsManageListing
 HotelsManagementCenter
 HotelsProductCart
 HowTheSiteWorks
+HpImageArchive.aspx
 HubStyle/static-2.309
 HubStyle/static-2.313
 HubStyle/static-2.315
@@ -5006,7 +5007,6 @@ INSTALL.sqlite.txt
 INSTALL.txt
 IgnoreMe
 Image/DetailPrint
-Images/
 ImpressionRecord
 ImproveListing
 Inbox
@@ -5063,7 +5063,6 @@ MAPS
 MUST
 MaPS
 MachineTranslation
-Mail/
 MailEmergency
 Manage/
 ManageListing
@@ -5435,12 +5434,12 @@ ProductCart
 ProductComparisonCmd
 ProductDetailsTabView
 ProductError
-Production/
 Profile
 Project
 PromoTools
 PromotionDetailsDisplayView
 PromotionDisplayView
+Proxy.ashx
 ProxyToolsAccountSettingsTool
 ProxyToolsBillingAndPaymentsTool
 ProxyToolsBillingDetailTool
@@ -5567,7 +5566,6 @@ RestaurantSearch
 RestaurantSolutions
 RestaurantXSellAjax
 Resubscribe
-Review/
 Review/LeaveReview/
 Review/List/
 ReviewCollect
@@ -5602,6 +5600,7 @@ Saves
 ScreensaverShade
 Scripts
 Search
+Search
 Search/Mail/Public
 SearchNavigationView
 SelectLocale?fromPage=
@@ -6189,6 +6188,8 @@ ac_universities
 ac_university_departments
 aca/authorinformation/
 academic-subscriptions
+academic/profile
+academic/search
 academy
 academy-free-account
 academy/*/results
@@ -6213,6 +6214,7 @@ account-login
 account-prefs-change
 account-settings
 account-setup
+account/
 account/*
 account/accomplishments
 account/ackgift
@@ -6274,6 +6276,7 @@ ach/kb/*/discuss
 ach/search
 ach/user
 ach/users
+aclick
 aclk
 acoustic
 acq
@@ -6584,6 +6587,7 @@ alexa-top-sites/*
 alexa-web-search/*
 alexa/*
 alexaforbusiness/
+alink
 alive/
 alkali
 all-supported-params
@@ -6609,6 +6613,7 @@ american-ninja-warrior-ninja-vs-ninja
 american-sniper
 aml
 amp
+amp/
 amp/deals/analytics?*=
 amp/local/*/www.groupon.com
 amp_next_page_ga.cms*
@@ -6700,12 +6705,13 @@ api/editionurlgenerator
 api/extensions/1.0/extensions
 api/federation
 api/html
-api/http://kobo.com/robots.txt
+api/http://tumblr.com/robots.txt
 api/infinite-list.html
 api/internal/*/location
 api/internal/_bb
 api/jobPostings/jobs
 api/legendary-data/relay
+api/maps/
 api/nextsteps
 api/nts/v1/login
 api/partner_select/*/price_query
@@ -7378,7 +7384,6 @@ backstage/api/
 bad-boys
 bad-boys-ii
 badges/profile/create
-baidu
 bal/impression
 bal/impression/RecordImpression.htm?
 ban-me-i-am-a-robot
@@ -7631,6 +7636,7 @@ bewerten
 bfe/apps/website
 bfl
 bfmp
+bfp/search
 bg
 bg-bg/
 bg/
@@ -7663,6 +7669,7 @@ bin/*
 bin/sapdx
 bin/search.go
 bin/search.py
+bing-site-safety
 bingbot
 bipolar-disorder/bipolar-depression-management
 bipolar-disorder/video/bipolar-fatherhood
@@ -7758,7 +7765,7 @@ blog/*?text=
 blog/*sitemap.xml$
 blog/*utm_source=
 blog/.
-blog/.html.*http://samsung.com/robots.txt
+blog/.html.*http://statista.com/robots.txt
 blog/?p=
 blog/?s=
 blog/add
@@ -7821,6 +7828,7 @@ blogs/blogpost/preview
 blogs/nationals-journal-beta
 blogs/pad$
 blogs/pad/$
+blogs/search/
 blogs/slow-ride
 blogs/technology.atom
 blogs/test
@@ -7883,7 +7891,7 @@ bookmark_mana2
 bookmarklet
 bookmarks
 bookmarks/widget
-bookmarkshttp://ecrs.com/robots.txt
+bookmarkshttp://google.it/robots.txt
 books
 books/NBK
 books/about
@@ -8097,6 +8105,7 @@ buzzfeed/api
 bvroute=Review
 bvrrp
 bvtab
+bxin.cisco.com
 byptwd
 bzzagent
 c
@@ -8763,7 +8772,7 @@ cdn-cgi
 cdn-cgi/
 cdn-cgi/bm/cv
 cdn-cgi/challenge-platform
-cdn-cgi/http://medium.com/robots.txt
+cdn-cgi/http://kids.youtube.com/robots.txt
 cdn-cgi/images/trace
 cdn-cgi/rum
 cdn-cgi/scripts
@@ -8794,7 +8803,7 @@ cgi-bin/koha/
 cgi-bin/loginpage
 cgi-bin/search.fpl
 cgi-bin/wx
-cgi-binhttp://ifttt.com/robots.txt
+cgi-binhttp://healthtips1dr.blogspot.com/robots.txt
 cgi-local
 cgi-mad/
 cgi-pct/
@@ -8907,7 +8916,7 @@ checkout-external
 checkout/
 checkout/cart
 checkout/package
-checkouthttp://moderncalculators.com/robots.txt
+checkouthttp://bilgematics.com/robots.txt
 checkoutnow
 checkouts
 checkpoint
@@ -8970,6 +8979,7 @@ civis/posts
 civis/search
 civis/threads/*in_iframe=1
 civis/threads/*wp_data=
+ck/
 cl
 cl/Download
 cl2/feeds
@@ -9131,7 +9141,7 @@ co/users
 co/ventureone
 code
 code/*
-code/http://w3techs.com/robots.txt
+code/http://marriott.com/robots.txt
 codeCoverage
 codesamples/
 coin*mid=
@@ -9224,7 +9234,7 @@ community/*/comments
 community/*/feed
 community/*/live
 community/*/post
-community/*/profile/*http://commission.europa.eu/robots.txt
+community/*/profile/*http://beatstars.com/robots.txt
 community/*/topics
 community/*/trackback
 community/*?
@@ -9351,7 +9361,7 @@ confirmation
 conflagexp
 connect
 connect/@
-connect/http://blog.twitter.com/robots.txt
+connect/http://irishtimes.com/robots.txt
 connect/social-intranet.html
 connections
 connections/recent
@@ -9630,11 +9640,12 @@ cp/api/logger
 cp/api/wpa
 cpds2
 cpresources
-cpro
 cpropub/univercd
 cpspa
+cr$
 cr/
 cr/Download
+cr?
 craft
 crash
 crc/
@@ -10432,7 +10443,7 @@ docs
 docs$
 docs-api-legacy
 docs-api-legacy$
-docs-api-legacy/http://deezer.com/robots.txt
+docs-api-legacy/http://nam.edu/robots.txt
 docs-footer
 docs/*
 docs/3.x/fr
@@ -10995,6 +11006,9 @@ enterprise/blog/search?
 enterprise/orgs-terms
 enterprise/seo
 entertainment/article/Minerva-s-horoscope
+entities/search
+entityexplore$
+entityexplore?
 entrez
 entrez/eutils
 entrez/sutils
@@ -11545,6 +11559,7 @@ fbml/ajax/dialog
 fbpostaction.cms?article=
 fcgi-
 fckeditor
+fd/
 fdbk
 feature/sf-test
 feature_nine_browse-bin=
@@ -11733,7 +11748,7 @@ files/live/sites/isoorg/files/_noindex
 files/resources
 fileview
 filing/free-file-fillable-forms/free-file-fillable-forms-support
-filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://iso.org/robots.txt
+filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://justice.gov/robots.txt
 filter/
 filter/tips
 filter/tips
@@ -11755,7 +11770,7 @@ financial-services/ofss
 find
 find$
 find-more
-find/*http://freepik.com/robots.txt
+find/*http://namecheap.com/robots.txt
 finetune
 fios
 firefox/?
@@ -11844,7 +11859,7 @@ forum/members
 forum/online
 forum/posts
 forum/search
-forum/ucp.phphttp://embassybel.ru/robots.txt
+forum/ucp.phphttp://roxanegay.com/robots.txt
 forum/viewtopic.php
 forum/whats-new
 forumTeaser
@@ -12072,6 +12087,8 @@ ftpinstructions
 full
 fullscreen
 fullscreen/ReportLinker
+fun/api/
+fun/g/
 funding?
 fundraiser/charity
 fur/forums
@@ -12094,7 +12111,7 @@ fy-NL/search
 fy-NL/user
 fy-NL/users
 g
-g/4arE9Chttp://flickr.com/robots.txt
+g/4arE9Chttp://animoto.com/robots.txt
 g/api
 ga-IE/android/collections
 ga-IE/android/search
@@ -12303,7 +12320,7 @@ government-education/nonprofits/leadership-forum-poc
 government-education/nonprofits/leadership-forum-poc/confirmation
 government-education/public-sector-cloud-start/public-sector-cloud-start/
 government-education/public-sector-cloud-start/public-sector-cloud-start1/
-government/uploads/system/uploads/attachment_data/file/881746/pc1-print.pdf*http://asyura2.com/robots.txt
+government/uploads/system/uploads/attachment_data/file/881746/pc1-print.pdf*http://scratch.mit.edu/robots.txt
 gp
 gp/aag
 gp/aag/main?*seller=ABVFEJU8LS620
@@ -12803,7 +12820,6 @@ homeimprovement/EmbeddedProfile.htm
 homeloans/*zga_z_guid=
 homepage
 homepage-video-test
-homepage/
 homepage/2013
 homepage/HomepageVideoMetrics.htm
 homes
@@ -12838,10 +12854,12 @@ horoscopes
 hosted/images
 hostgator
 hosting/search
+hotel/reviews?
 hotelfinder/rpc
 hotels
 hotels/newAdaptiveRoomTypePhotoViewer
 hotels/rpc
+hotels/search
 hotfix/
 houseads
 houzz-historyzz
@@ -12852,6 +12870,8 @@ howto
 howto/GoodNeighborPolicy.htm
 howtobuy/
 hp/video
+hpm
+hpmob
 hr
 hr-hr/
 hr/android/collections
@@ -12900,39 +12920,40 @@ html5
 http
 http-checker
 http2_data
-http://2.bp.blogspot.com/robots.txt
+http://4.bp.blogspot.com/robots.txt
-http://about.fb.com/robots.txt
+http://aljazeera.com/robots.txt
-http://akparti.org.tr/robots.txt
+http://apnews.com/robots.txt
-http://amazon.de/robots.txt
+http://arminbwagner.com/robots.txt
-http://amazon.in/robots.txt
+http://atlasobscura.com/robots.txt
-http://ameblo.jp/robots.txt
+http://cnabrasil.org.br/robots.txt
-http://apple.com/robots.txt
+http://dailymotion.com/robots.txt
-http://bbc.com/robots.txt
+http://developer.apple.com/robots.txt
-http://bbcsfx.acropolis.org.uk/robots.txt
+http://developer.chrome.com/robots.txt
-http://creativecommons.org/robots.txt
+http://developer.mozilla.org/robots.txt
+http://developers.google.com/robots.txt
+http://doi.org/robots.txt
+http://duckduckgo.com/robots.txt
 http://economist.com/robots.txt
-http://europa.eu/robots.txt
+http://engadget.com/robots.txt
-http://expedia.com/robots.txt
+http://eventbrite.co.uk/robots.txt
-http://gist.github.com/robots.txt
+http://history.com/robots.txt
-http://govostis.gr/robots.txt
+http://investopedia.com/robots.txt
-http://healthline.com/robots.txt
+http://istockphoto.com/robots.txt
-http://help.twitter.com/robots.txt
+http://journals.sagepub.com/robots.txt
-http://irishtimes.com/robots.txt
+http://nature.com/robots.txt
-http://iubenda.com/robots.txt
+http://ncbi.nlm.nih.gov/robots.txt
-http://msdn.microsoft.com/robots.txt
+http://parliament.uk/robots.txt
-http://nationalgeographic.com/robots.txt
+http://pbs.org/robots.txt
-http://patents.google.com/robots.txt
+http://research.youtube/robots.txt
-http://producthunt.com/robots.txt
+http://scmp.com/robots.txt
-http://python.org/robots.txt
+http://starbucks.com/robots.txt
-http://reconstructingjudaism.org/robots.txt
+http://support.cloudflare.com/robots.txt
-http://smithsonianmag.com/robots.txt
+http://technorati.com/robots.txt
-http://static.wixstatic.com/robots.txt
+http://ted.com/robots.txt
-http://techradar.com/robots.txt
+http://telegram.org/robots.txt
-http://theverge.com/robots.txt
+http://usnews.com/robots.txt
-http://threads.net/robots.txt
+http://vr.youtube.com/robots.txt
-http://tiny.cc/robots.txt
+http://w3schools.com/robots.txt
-http://tracks.arte.tv/robots.txt
-http://weebly.com/robots.txt
 https:/www.alibaba.com
 hu
 hu-hu/
@@ -13286,10 +13307,20 @@ images/$
 images/*
 images/*.png
 images/?
+images/async/?
+images/async?
+images/hosted/?
+images/hosted/search?
+images/hosted?
 images/pad/search?rpt=imageview
 images/pad/search?text=*&source=morda
+images/search%3F
+images/search%5C
+images/search/?
+images/search?
 images/search?rpt=imageview
 images/search?text=*&source=morda
+images/searchbyimage
 images/smart/$
 images/touch/$
 images/touch/?
@@ -13297,6 +13328,9 @@ images/touch/search?rpt=imageview
 images/touch/search?text=*&source=morda
 images/v$
 images/viewconfig$
+images?
+images?*mmbaseline*
+images?*nr*
 imagine/
 imesync
 imf-fetch-message
@@ -13427,6 +13461,7 @@ inapp
 inbox
 inbox$
 inbox$
+inboxcommerce
 inc5000dev
 include/
 includes
@@ -13513,7 +13548,7 @@ index.php/user/login/
 index.php/user/logout
 index.php/user/logout
 index.php/user/logout/
-index.php/user/logout/http://diigo.com/robots.txt
+index.php/user/logout/http://haveibeenpwned.com/robots.txt
 index.php/user/password
 index.php/user/password
 index.php/user/password/
@@ -13574,7 +13609,7 @@ integrate/
 integrations
 intelligencer/amp
 intent
-interactive*http://fda.gov/robots.txt
+interactive*http://woocommerce.com/robots.txt
 interactive/reading-preference
 interactives
 intercept
@@ -14106,7 +14141,6 @@ journal/wem
 journal/wem$
 journals
 journals/*/about-this-journal
-journals/testfieldhumanities/
 jp
 jp$
 jp*/blog-article-remove-faq-utms-*.js
@@ -14421,7 +14455,7 @@ law-abiding-citizen
 layout
 layouts
 lb
-leadingtogether/http://validator.w3.org/robots.txt
+leadingtogether/http://sintradaytrips.com/robots.txt
 leaf
 learn
 learn-noperf
@@ -14535,7 +14569,6 @@ limited-release
 link
 link-intersect
 link-intersect/$
-link?
 link_to
 linked
 linkedin
@@ -14583,6 +14616,7 @@ loans
 loc/volunteers
 loc=
 local
+local
 local-services-ads
 local/*/all-deals
 local/*/goods
@@ -14879,6 +14913,7 @@ maps/*?lang=uk$
 maps/*?lang=uz$
 maps/*?ll=
 maps/about
+maps/adsendpoint
 maps/api/js
 maps/api/place/js
 maps/api/staticmap
@@ -15075,6 +15110,7 @@ merchanism
 merchant-page
 merchant-page/js/locale.js
 merchant/*/contracts
+merchant/reviews?
 merchantapps
 mes
 message
@@ -15290,7 +15326,7 @@ mre/*.jpg$
 mre/*.js$
 mre/*.pdf$
 mre/*.png$
-mre/*.zip$http://gov.uk/robots.txt
+mre/*.zip$http://modernbluesharmonica.com/robots.txt
 mre/*?
 mre/*@@configuracoes_view
 mre/*@@download/file$
@@ -15352,7 +15388,7 @@ multiproduct
 multisite_files/fastcompany/imagecache
 multivu
 munin
-munin*http://lefigaro.fr/robots.txt
+munin*http://folsominteriordesigners.com/robots.txt
 muse/identity/v2/identity.js
 museumdays/accounts
 music
@@ -16110,6 +16146,7 @@ notfound.html
 notifications
 notifications
 notifications$
+notifications/
 notifications/*
 nova
 novel/comments.php?id=
@@ -16287,6 +16324,8 @@ offers/nonprofits$
 offers/nonprofits%
 offers/nonprofits?
 offers/preview/
+offers/proxy/dealsserver/api/log
+offers/proxy/dealsserver/buy
 offers/referral-terms-conditions
 offers/referral-terms-conditions$
 offers/referral-terms-conditions%
@@ -16302,7 +16341,7 @@ ohcm
 ojp
 okxbet.com
 oma
-on-demand/session/*http://nymag.com/robots.txt
+on-demand/session/*http://podcasts.google.com/robots.txt
 onboard
 onboarding
 one-app-preview
@@ -16439,7 +16478,7 @@ p/*/v/*?
 p/*/v?
 p/*/victory$
 p/*/victory?
-p/*http://socialimpact.youtube.com/robots.txt
+p/*http://ifcncodeofprinciples.poynter.org/robots.txt
 p/*surveyopen
 p/*thm=
 p/60
@@ -17334,7 +17373,7 @@ premium$
 premium-reports
 premium/
 premium/$
-premium/http://s.w.org/robots.txt
+premium/http://coindesk.com/robots.txt
 premiumsupport/iam/
 premiumsupport/knowledge-center/feedback-no/
 premiumsupport/knowledge-center/feedback-yes/
@@ -17411,7 +17450,7 @@ privacy/request-data
 privacy/zoho-group.html
 private
 private-promo
-private/http://sgtrains.com/robots.txt
+private/http://datatracker.ietf.org/robots.txt
 private/wtrack
 pro
 pro-learn
@@ -17421,6 +17460,7 @@ pro-site
 pro-solutions
 pro/about
 pro2
+proFile/history?
 proMatch
 proProjects
 proVideoWidget
@@ -17434,12 +17474,13 @@ product-catalogs
 product-detail
 product-detail/alisite-mobile.alibaba.com
 product-feed
-product-ratings/http://wix.com/robots.txt
+product-ratings/http://jesus-comes.com/robots.txt
 product-reviews/B0069IY63Y
 product/*
 product/electrode/api/logger
 product/electrode/api/wpa
 product/recommended
+product/reviews?
 productBrand=
 productId=
 productPurchaseDone
@@ -17448,7 +17489,6 @@ productdetail/*
 producthunt
 productid=
 production-monitoring
-production/
 products
 products/*--*--*--
 products/*/accent-chests-and-cabinets/review--4-stars-and-up
@@ -17560,6 +17600,7 @@ profile/*/photos
 profile/account
 profile/following
 profile/general
+profile/history?
 profile/mylists
 profile/premium
 profile/view
@@ -18497,6 +18538,7 @@ restricted
 restricted/
 restricted_temporary/
 results
+results
 results-export-email-by-search-data
 results-export-ids
 results-export-search-by-year
@@ -18513,7 +18555,6 @@ review-by-author
 review-drafts
 review-order
 review-sample
-review/
 review/*/transparency
 review/common/du
 review/dynamic/sims-box
@@ -18545,6 +18586,7 @@ reviews/write/?s=
 reviews/write?s=
 reviews_page=
 rewards
+rewardsapp/
 rf=*
 rip/
 rlv/
@@ -18759,7 +18801,6 @@ s3
 s=
 s=p
 s?
-s?
 s?*rh=n%3A1380045031
 s_id=
 sa
@@ -19272,6 +19313,7 @@ set_author_id
 setbucket
 setprefs
 settings
+settings
 settings/*
 settings/deactivated
 settings/linkaccount
@@ -19407,8 +19449,8 @@ sharing
 sharing/files
 sharing/folders
 shelf/user_shelves
+shenghuo
 shenghuo/search?
-shifen/
 shihui
 shihui?
 ship
@@ -19417,11 +19459,15 @@ shipping
 shipping-payment
 shippingassistant
 shop
+shop$
+shop/
 shop/*&
 shop/browse/overlay
+shop/collections
 shop/global
 shop/iphone/payments/overlay
 shop/view/product
+shop?
 shop_name_search_service
 shop_pins
 shopcart/*
@@ -19466,7 +19512,7 @@ showcase/Click.htm?
 showroom
 shows/
 showtime/.requestinvite.html.swp
-showtimes/location/*http://img.youtube.com/robots.txt
+showtimes/location/*http://cnbc.com/robots.txt
 shr*
 shrek
 si
@@ -19941,6 +19987,7 @@ social
 social-connect/send
 social/photosui/images
 social/photosui/sitemap
+social/search?
 social/social-security-directory/cai
 social_media
 socialapp/
@@ -19999,7 +20046,7 @@ sort=
 sortBy=
 sos/clerical/
 sos/online_corps/
-sos/registration/http://inc.com/robots.txt
+sos/registration/http://public.tableau.com/robots.txt
 soundcloud-tpa
 soundoff
 sounds/player
@@ -20020,6 +20067,7 @@ sp/news3
 sp=
 sp_url=
 spartan
+spbasic
 spc-source=
 special-promo/2for1
 specialhandling-payments/
@@ -20067,6 +20115,7 @@ sports/nfl/includes
 sports/whatson
 sports_fp_ms
 spreadsheet
+spresults
 sprint_wml
 sprint_xhtml
 sq/android/collections
@@ -20140,6 +20189,7 @@ stat
 statement
 statements
 static
+static/
 static/*.css$
 static/*.css?
 static/*.gif
@@ -20268,7 +20318,9 @@ storeclick
 storefront
 storepicker
 storerequest
+stores$
 stores-manage
+stores/
 stores/page/*visitId=
 stories
 story
@@ -20558,7 +20610,7 @@ tag
 tag-manager
 tag-manager-360
 tag/
-tag/*+http://dinotim.com/robots.txt
+tag/*+http://yumpu.com/robots.txt
 tag/*/shoutbox
 tag/*/shoutbox?
 tag/credible
@@ -20720,7 +20772,7 @@ test.html
 test_community
 testfl
 testgpr
-testhttp://monroewvhistory.org/robots.txt
+testhttp://edition.cnn.com/robots.txt
 testing
 testpage-forhomepage
 tests/*
@@ -20729,6 +20781,7 @@ text/*?page=
 text/yandex-news.region.xml
 tfemail
 th
+th$
 th*/blog-article-remove-faq-utms-*.js
 th*/cdn-cgi/challenge-platform
 th*/step
@@ -20794,6 +20847,7 @@ th/tools/logo-maker/onboarding
 th/user
 th/users
 th/ventureone
+th?
 th_th
 th_th/templates/M
 thank-you
@@ -21150,6 +21204,7 @@ translate_p
 translate_static
 translate_suggestion
 translate_t
+translator/?ref=
 transmute/static-1.85
 transmute/static-2.6
 transmute/static-2.8
@@ -21176,13 +21231,30 @@ travel-frontend/images
 travel-vacations
 travel/
 travel/2013/aug/22/been-there-readers-competition?
+travel/api
 travel/clk
+travel/css
 travel/entity
+travel/flight-search
+travel/flight/flightSearch
+travel/flight/flightSearchAction
+travel/flight/search/?
+travel/flight/search?
+travel/flights
 travel/flights/s
+travel/hotel-search
+travel/hotel/hotelMiniSearchRequest
+travel/hotel/hotelSearch
+travel/hotels/checkout
+travel/hotels/detail
 travel/hotels/entity
+travel/hotels/search/?
+travel/hotels/search?
 travel/hotels/stories
 travel/passport
+travel/scripts
 travel/search
+travel/secure
 travelnevadahighway95tc
 treadstone
 treasury
@@ -21494,7 +21566,6 @@ uk/your
 uk/zazzle
 uki/search
 uli/trac
-ulink?
 ulk/i
 ulk/sch
 ulk/usr
@@ -21583,6 +21654,7 @@ ur/search
 ur/user
 ur/users
 urchin_test
+url
 url-shortner
 urlinator/static-1.54
 urlinator/static-1.55
@@ -22010,7 +22082,7 @@ uy
 uy/es/ServicesLandingPage/AdvisoryLanding/RPA/Robotic_Process%20Automation_insert.pdf
 uz/en/
 uz/forums
-uz/http://bilgematics.com/robots.txt
+uz/http://propublica.org/robots.txt
 uz/kb/*/discuss
 uz/ru/
 uz/search
@@ -22188,11 +22260,29 @@ videodash.cms*
 videodetails
 videorequest
 videos
+videos/?
+videos/browsing
 videos/expander
+videos/explore
+videos/favorites
+videos/feed
+videos/feedcontent
+videos/music
+videos/music/
+videos/music/?
+videos/music?
+videos/riverview/relatedvideo
+videos/riverview/relatedvideo/?
+videos/riverview/relatedvideo?
 videos/search
+videos/search/?
+videos/search?
 videos/show_trailer
 videos/slider
 videos/transcripts
+videos/trending
+videos/trendingcontent
+videos?
 videosearch
 view
 view-3d-home
@@ -22460,7 +22550,11 @@ whythisad
 widget
 widget-landing-page.html
 widget-uploads
+widget/cr
+widget/entity/search/?
 widget/player
+widget/render
+widget/snapshot
 widget_auth.php
 widget_chunks
 widgets
@@ -22916,9 +23010,9 @@ wiki/Wikipedia_discusión:Solicitudes_de_verificación_de_usuarios
 wiki/Wikipedia_discusión:Supresores/Candidaturas
 wiki/Wikipedia_talk%3AAbuse_reports
 wiki/Wikipedia_talk%3AAbuse_response
-wiki/Wikipedia_talk%3AAdministrator_electionshttp://en.wikipedia.org/robots.txt
+wiki/Wikipedia_talk%3AAdministrator_electionshttp://itunes.apple.com/robots.txt
-wiki/Wikipedia_talk%3AAdministrator_electionshttp://engadget.com/robots.txt
+wiki/Wikipedia_talk%3AAdministrator_electionshttp://producthunt.com/robots.txt
-wiki/Wikipedia_talk%3AAdministrator_electionshttp://win.gs/robots.txt
+wiki/Wikipedia_talk%3AAdministrator_electionshttp://thehill.com/robots.txt
 wiki/Wikipedia_talk%3AAdministrator_recall
 wiki/Wikipedia_talk%3AAdministrator_review
 wiki/Wikipedia_talk%3AAdministrators%27_noticeboard
@@ -23115,6 +23209,7 @@ word/embed
 wordpress/wp-admin
 wordpress/wp-admin/admin-ajax.php
 work
+work
 work-at-ucl/redeployment
 work/_
 work/acceptinvitation
@@ -23210,8 +23305,7 @@ wp-admin
 wp-admin/
 wp-admin/admin-ajax.php
 wp-admin/admin-ajax.php
-wp-admin/admin-ajax.phphttp://rtbf.be/robots.txt
+wp-admin/admin-ajax.phphttp://bible.com/robots.txt
-wp-admin/admin-ajax.phphttp://starbucks.com/robots.txt
 wp-admin/load-scripts.php
 wp-admin/load-styles.php
 wp-adv
@@ -23401,7 +23495,7 @@ your
 your-account
 your-shop
 youreurope/businessACC
-youreurope/businessDEV/http://europarl.europa.eu/robots.txt
+youreurope/businessDEV/http://fortune.com/robots.txt
 youreurope/citizensACC
 youreurope/citizensDEV
 yours
@@ -23550,7 +23644,7 @@ zh-tw.www.cloudflare.com
 zh-tw/
 zh-tw/feedback
 zh-tw/lp
-zh-tw/searchhttp://blog.giallozafferano.it/robots.txt
+zh-tw/searchhttp://bizjournals.com/robots.txt
 zh-tw/searchresults
 zh/
 zh/*?*shpxid=