[Github Action] Automated trickest wordlists update.

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt

@@ -2769,8 +2769,8 @@ hotelfinder/rpc
 hotels/rpc
 hp/video
 hpc/*/supported-applications.aspx?
-http://linkedin.com/robots.txt
-http://theguardian.com/robots.txt
+http://i2.wp.com/robots.txt
+http://imdb.com/robots.txt
 hub?category=
 hz/contact-us/ajax/initiate-trusted-contact
 hz/help/contact/*/message/$
@@ -4211,7 +4211,7 @@ shopping/seller
 shopping/suppliers/search
 shopping?
 shopstream
-showtimes/location/*http://bloomberg.com/robots.txt
+showtimes/location/*http://patreon.com/robots.txt
 sidewiki/entry
 signature
 signin/inject

Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt

@@ -261,6 +261,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 .mp4$
 .mrss$
 .mspx
+.nocache.html
 .otf*************
 .otmi$
 .pdf
@@ -307,7 +308,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 .xml?uuid=
 /$action.getCategoryBrowserLink($val)
 /%20library/
-/*+*+*+
 /*-Off-or-More
 /*-anal$
 /*-anal-
@@ -2306,12 +2306,18 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /?orpq
 /?q=
 /?s=
+/?sc_lang
 /?xml=
 /?z=1
 /@async
 /A3
 /Aes/$
+/BucketContent
 /C
+/CVS
+/Careers/Form
+/Careers/Profiles
+/Careers/Registration
 /Client/
 /Community?
 /CountyAssessorPage.htm?
@@ -2327,6 +2333,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /Ntx
 /P
 /PWD
+/RedesignBucket
 /Results.aspx
 /Results?
 /SelectLocale?fromPage=
@@ -2338,6 +2345,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /_
 /_/rsrc
 /__wishlist__
+/_acnmedia/
 /_activities
 /_activity
 /_category
@@ -2348,6 +2356,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /_employer_name
 /_followers
 /_following
+/_globalreferences/
 /_jcr_content
 /_jcr_content/par
 /_m2
@@ -2357,6 +2366,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /_shop
 /_shopping_list
 /_type
+/a-com-no-follow-no-index/
 /a3
 /about
 /about
@@ -2392,8 +2402,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /aisearch
 /ajax
 /ajax/*
-/ajax/jsLogging.html
-/albert-heijn-premium
 /album/*/*?i=
 /alert
 /all_comments
@@ -2470,8 +2478,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /author
 /autos/marketplace/product/*
 /avg-review
-/b/*/*
-/b/*/*+
 /badge
 /badges
 /base/logout
@@ -2496,7 +2502,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /blogs/*%2b
 /blogs/*+
 /blogvisualizer/
-/bltg
 /boobs-photo.html
 /book
 /bookmark-*.html
@@ -2524,10 +2529,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /business/search
 /buyback
 /buyer/widget
-/c/*
-/c/*/*+3132/$
-/c/*/*+3133/$
-/c/*/*+8299/$
 /c/p
 /c2
 /cache
@@ -2565,6 +2566,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /click?type=*$
 /client$
 /client/
+/clients/
 /clima/
 /cm
 /cnet_global_app_redirect.xml
@@ -2619,15 +2621,16 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /contact-form
 /contact/menu*
 /contact/partner*
-/contactformulier
 /contactus/mwaas*
 /content-feedback/accessibility-feedback
+/content/acom/
 /content/search
 /content/usergenerated/asi/mongo/content/sites/activitystreams/activities/community
 /contributor
 /cookie$
 /cookie.js
 /cookie/dmaid
+/core/
 /country
 /cover
 /covid-19-across-world/*
@@ -2709,7 +2712,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /dynamics/
 /dzcfg
 /ebook
-/ebook/preview.html
 /ece
 /ecommerce/cart
 /ecommerce/checkout
@@ -2751,6 +2753,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /entrylist-*.html
 /entrylist.html
 /error
+/error/
 /es/partidos
 /etag$
 /evalcenter/
@@ -2835,6 +2838,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /hack
 /hardware/*/cart
 /havadurumu/
+/header/global-header/
 /health/search/filter
 /healthvault/
 /height_*--
@@ -2969,7 +2973,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /libs
 /life-style-landing/*
 /lightreg/index.html
-/lijstjes
 /liked-posts
 /likes
 /limited-release
@@ -2996,6 +2999,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /login*/search?
 /login/verify-email
 /login?
+/loginpage
 /logout
 /longhorn/
 /lookup?
@@ -3098,7 +3102,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /newssearch
 /no-internet-heading-assigned
 /no-internet-heading-assisted
-/no-referrer-when-downgradehttp://iana.org/robots.txt
+/no-referrer-when-downgradehttp://google.ru/robots.txt
 /no-search-results
 /node
 /node/add
@@ -3181,6 +3185,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /plus
 /pmredirectshow/*
 /pmtoinavigation/*
+/po
 /pocasi/
 /podcast/*/*?i=
 /podcast/ITEM_NAME
@@ -3209,12 +3214,9 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /price--
 /price-comparison
 /prices
-/prijsoverzicht
 /print
 /print$
 /probr-1~/project-type--*/$
-/productaanbevelingen
-/producten-vergelijken
 /products
 /products/*?*filter=
 /professionals/*/*project-type-*-probr1-
@@ -3232,7 +3234,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /query
 /questions
 /r
-/ra
 /ramavtal
 /random
 /rapist
@@ -3271,13 +3272,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /rg*/mediaviewer/rm*/tr
 /rino-_att
 /rl
-/rnwy/account
-/rnwy/ajax
-/rnwy/artikelen
-/rnwy/product/reviews
-/rnwy/product_compare
-/rnwy/slots/ajax
-/rnwy/studybooks
 /roadshow/finder
 /rodate.js
 /rss
@@ -3294,12 +3288,9 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /scheduled-pins
 /schemes
 /schoollist
-/schrijf-een-review
 /science/ereader
 /science/fabrication
 /screenshots
-/sdl
-/sdl/shop-de-look
 /search
 /search*%7C
 /search*&node=*&sort=
@@ -3389,15 +3380,13 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /searchDirectory
 /searchresults/
 /searchsuggest
-/select*?
-/select-cadeau-sparen/?
+/secure/
 /selectlocale-dmc?fromPage=
 /sell-now
 /seller--
 /seminar/
 /sendpv$
 /sensor
-/serie
 /services
 /services/identity/login
 /services/learn/classes._filter.alphaSort.P*.
@@ -3435,6 +3424,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /singlepage/pipe
 /singlepage/uncached_pipe
 /site/
+/sitecore/
 /sitemap
 /sitemap_gpdc.xml
 /skip-video
@@ -3536,7 +3526,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /tv-season/*/*?i=
 /twerk
 /twerk.html
-/u
 /u1_sch
 /unanswered
 /undefined
@@ -3569,7 +3558,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /util$
 /utils
 /v2
-/vaak-samen-gekocht
 /vader/
 /vejr/
 /vendorActivityManager
@@ -3603,7 +3591,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /visited/*/group
 /visited/group
 /vsarticleshow/*
-/w
 /w--
 /watch
 /watch-fragments
@@ -3633,6 +3620,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /windowsvista/
 /windvane$
 /winfx/
+/workshop
 /workshop/
 /wp-admin
 /wp-content
@@ -3682,7 +3670,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*__rmid
 ?*__rmidpage
 ?*ajs_event
-?*apikey*http://rebrand.ly/robots.txt
+?*apikey*http://lifestylesimplify.com/robots.txt
 ?*app
 ?*appidt=*
 ?*aspects=
@@ -3790,121 +3778,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*stars=
 ?*subcategories=
 ?*subscription
-?*term=一
-?*term=三
-?*term=上
-?*term=下
-?*term=不
-?*term=与
-?*term=个
-?*term=中
-?*term=为
-?*term=主
-?*term=么
-?*term=之
-?*term=也
-?*term=了
-?*term=事
-?*term=二
-?*term=于
-?*term=人
-?*term=从
-?*term=他
-?*term=以
-?*term=们
-?*term=仿
-?*term=会
-?*term=作
-?*term=你
-?*term=假
-?*term=入
-?*term=公
-?*term=其
-?*term=军
-?*term=出
-?*term=分
-?*term=刚
-?*term=到
-?*term=制
-?*term=前
-?*term=办
-?*term=十
-?*term=印
-?*term=去
-?*term=又
-?*term=发
-?*term=只
-?*term=可
-?*term=同
-?*term=后
-?*term=和
-?*term=国
-?*term=在
-?*term=地
-?*term=多
-?*term=大
-?*term=天
-?*term=好
-?*term=如
-?*term=子
-?*term=学
-?*term=定
-?*term=家
-?*term=对
-?*term=将
-?*term=小
-?*term=就
-?*term=年
-?*term=当
-?*term=得
-?*term=德
-?*term=心
-?*term=成
-?*term=我
-?*term=所
-?*term=护
-?*term=方
-?*term=无
-?*term=日
-?*term=时
-?*term=是
-?*term=有
-?*term=来
-?*term=此
-?*term=没
-?*term=法
-?*term=澳
-?*term=焦
-?*term=然
-?*term=生
-?*term=用
-?*term=的
-?*term=看
-?*term=着
-?*term=种
-?*term=经
-?*term=网
-?*term=者
-?*term=而
-?*term=能
-?*term=自
-?*term=行
-?*term=西
-?*term=要
-?*term=见
-?*term=订
-?*term=说
-?*term=起
-?*term=过
-?*term=还
-?*term=这
-?*term=进
-?*term=道
-?*term=那
-?*term=都
-?*term=里
-?*term=雅
-?*term=面
 ?*tpcid=
 ?*trustscore=
 ?*ua
@@ -3914,7 +3787,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*utm_medium
 ?*utm_source
 ?*utm_source=
-?*utm_source=http://15721.courses.cs.cmu.edu/robots.txt
+?*utm_source=http://pixel.quantserve.com/robots.txt
 ?*verified=
 ?*xid
 ?252525253Bpreview=
@@ -4135,7 +4008,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?p=
 ?p=
 ?page*&=sort
-?page=*&promo
 ?page=[0-9]
 ?page=[0-9][0-9]
 ?page=[0-9][0-9][0-9]
@@ -5680,7 +5552,6 @@ I18n/static-7.314
 I18n/static-7.315
 I18n/static-7.329
 IBCommerce
-ID"),!1,d)}}catch(x){google.ml(x,!0,{"jsl.dh":!0})}};(function(){var
 IEB/ToolBox/SB
 ILC-refresh
 IM
@@ -7146,7 +7017,6 @@ ads
 ads$
 ads-beacon.js
 ads.txt
-ads.txt
 ads/gpa/vidgen
 ads/hotels/partners
 ads/learn
@@ -7406,7 +7276,7 @@ api/editionurlgenerator
 api/eventlogging
 api/federation
 api/html
-api/http://photos.app.goo.gl/robots.txt
+api/http://israelnationalnews.com/robots.txt
 api/internal/*/location
 api/internal/_bb
 api/internal/analytics/songs
@@ -7416,7 +7286,6 @@ api/nextsteps
 api/nicorepo/timeline
 api/partner_select/*/price_query
 api/programas/69730/audios.rss
-api/reactions
 api/recaptcha/enabled
 api/rest_v1/?doc
 api/search/community_resources?
@@ -7483,7 +7352,7 @@ appchart
 appdynamics-wem.cisco.com
 appealed-pin
 apple-app-site-association
-apple-app-site-associationhttp://purdue.edu/robots.txt
+apple-app-site-associationhttp://freewebs.com/robots.txt
 apple-news
 appliance
 applicant-eligibility
@@ -7721,7 +7590,6 @@ at/your
 atari/images/public
 atc
 atct_album_view$
-atg/commerce/order/purchase
 athletic*/?source=
 athletic*/boxscore
 athletic*access_token
@@ -8480,7 +8348,6 @@ be/enterprise/blog/search$
 be/enterprise/blog/search?
 be/ferramentas/criador-de-logo/onboarding
 be/forums
-be/fr/p/-
 be/growth-tools-assets
 be/guest-pass
 be/help/sitemap
@@ -8701,7 +8568,7 @@ blog/*?text=
 blog/*sitemap.xml$
 blog/*utm_source=
 blog/.
-blog/.html.*http://netflix.com/robots.txt
+blog/.html.*http://www1.nyc.gov/robots.txt
 blog/?p=
 blog/?s=
 blog/Neuer_Beitrag.html
@@ -8808,7 +8675,7 @@ bookmark_mana2
 bookmarklet
 bookmarks
 bookmarks/widget
-bookmarkshttp://google.co.in/robots.txt
+bookmarkshttp://spreaker.com/robots.txt
 books
 books/NBK
 books/about
@@ -9025,6 +8892,7 @@ buy_buttons
 buyline-personal-finance
 buyline-shopping
 buyside-e2e-monitor
+buyside/search-results?*term=
 buzz/*.xml$
 buzz/justin
 buzz/post
@@ -9728,7 +9596,7 @@ cdn-cgi
 cdn-cgi/
 cdn-cgi/bm/cv
 cdn-cgi/challenge-platform
-cdn-cgi/http://iso.org/robots.txt
+cdn-cgi/http://xkcd.com/robots.txt
 cdn-cgi/rum
 cdn-cgi/zaraz/s.js
 cdn-cgi/zaraz/t
@@ -9750,11 +9618,12 @@ cgi-bin
 cgi-bin/
 cgi-bin/consumeraffairs/search
 cgi-bin/dobi/licenseesearch
+cgi-bin/estseek.cgi
 cgi-bin/homelandsecurity
 cgi-bin/loginpage
 cgi-bin/state
 cgi-bin/wx
-cgi-binhttp://youtube.de/robots.txt
+cgi-binhttp://zendesk.com/robots.txt
 cgi-local
 cgi-perl
 cgi-registry
@@ -10096,7 +9965,7 @@ co/users
 co/ventureone
 code
 code/*
-code/http://m.youtube.com/robots.txt
+code/http://mozilla.org/robots.txt
 codeCoverage
 codesamples/
 coin*mid=
@@ -10176,7 +10045,6 @@ committees/*/studiesdownload.html
 common
 common/
 common/austin-summit
-common/error
 common/mail/
 common/services
 common/survey.aspx
@@ -10190,6 +10058,8 @@ community-guidelines
 community/*/category/*
 community/*/comments
 community/*/feed
+community/*/foaf
+community/*/rss
 community/*/trackback
 community/*?
 community/account
@@ -10307,7 +10177,7 @@ confirmation.html
 conflagexp
 connect
 connect/@
-connect/http://de.wikipedia.org/robots.txt
+connect/http://dribbble.com/robots.txt
 connect/social-intranet.html
 connections
 connections/recent
@@ -10424,7 +10294,7 @@ content/dam/ircc/documents/pdf/francais/trousses/form/imm5476
 content/dam/ircc/documents/pdf/francais/trousses/form/imm5644
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0002
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0004
-content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://ko-fi.com/robots.txt
+content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://w3c.github.io/robots.txt
 content/dam/ircc/documents/pdf/francais/trousses/guides/guide-0142-avion-afghanistan-voie-acces-rp.pdf
 content/dam/oecd/
 content/dam/redcross/lp
@@ -11761,7 +11631,7 @@ email-preferences/
 email-unsubscribe
 email-validation
 email/$
-email/*http://avexnet.or.jp/robots.txt
+email/*http://smugmug.com/robots.txt
 email/EnrollmentUnsubscribe.htm
 email/SubscriptionTypeUnsubscribe.htm
 email/Unsubscribe.htm
@@ -12720,7 +12590,7 @@ files/techabuse-welsh.pdf
 files/techabuse.pdf
 fileview
 filing/free-file-fillable-forms/free-file-fillable-forms-support
-filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://arte.tv/robots.txt
+filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://globalresearch.ca/robots.txt
 filter
 filter/tips
 filter/tips
@@ -13124,7 +12994,7 @@ fy-NL/search
 fy-NL/user
 fy-NL/users
 g
-g/4arE9Chttp://caniuse.com/robots.txt
+g/4arE9Chttp://ademe.fr/robots.txt
 g/api
 g00
 g01
@@ -13356,7 +13226,7 @@ gh/news
 gh/useracquisition
 gh_jid=
 gif
-gifshttp://business.twitter.com/robots.txt
+gifshttp://animoto.com/robots.txt
 gift-redemption
 gift-terms
 gift_card=
@@ -13383,7 +13253,7 @@ glass/images/commerce
 global
 global$
 global-consumer-survey/tool
-global-infrastructure/services/get-async/http://lifestylesimplify.com/robots.txt
+global-infrastructure/services/get-async/http://mentalshinri.net/robots.txt
 global-master
 global-nav-esi
 global/RU/
@@ -13716,7 +13586,7 @@ ha/search
 ha/user
 ha/users
 hackathons/reinvent-2013/
-hacker/*/followhttp://thefader.com/robots.txt
+hacker/*/followhttp://thefa.com/robots.txt
 hacker/*/unfollow
 hair-transplant-services
 handler
@@ -14148,41 +14018,39 @@ html5
 http
 http-checker
 http2_data
-http:
-http://1.gravatar.com/robots.txt
+http://about.google/robots.txt
 http://addthis.com/robots.txt
-http://airtable.com/robots.txt
-http://audiomack.com/robots.txt
-http://australia.com/robots.txt
-http://bbb.org/robots.txt
-http://blog.hubspot.com/robots.txt
-http://chromium.org/robots.txt
-http://csmonitor.com/robots.txt
-http://daringfireball.net/robots.txt
-http://diigo.com/robots.txt
+http://bluehost.com/robots.txt
+http://booking.com/robots.txt
+http://code.google.com/robots.txt
+http://dev.twitter.com/robots.txt
+http://developer.apple.com/robots.txt
+http://ec.europa.eu/robots.txt
 http://fda.gov/robots.txt
-http://firstrunfeatures.com/robots.txt
-http://flic.kr/robots.txt
-http://fun888asia.com/robots.txt
-http://gstatic.com/robots.txt
-http://hubspot.com/robots.txt
+http://foxnews.com/robots.txt
+http://google.ch/robots.txt
+http://hootsuite.com/robots.txt
 http://ikea.com/robots.txt
-http://justgiving.com/robots.txt
-http://kck.st/robots.txt
-http://lh5.googleusercontent.com/robots.txt
-http://lnk.to/robots.txt
-http://mixer.com/robots.txt
-http://mobile.twitter.com/robots.txt
-http://nj.gov/robots.txt
-http://onedrive.live.com/robots.txt
+http://instapaper.com/robots.txt
+http://msdn.microsoft.com/robots.txt
+http://nbcnews.com/robots.txt
+http://npr.org/robots.txt
+http://nydailynews.com/robots.txt
+http://online.wsj.com/robots.txt
 http://opera.com/robots.txt
-http://phys.org/robots.txt
-http://roadtrippers.com/robots.txt
-http://s.w.org/robots.txt
-http://salesforce.com/robots.txt
+http://penguinrandomhouse.com/robots.txt
+http://pkp.sfu.ca/robots.txt
 http://salon.com/robots.txt
-http://tools.google.com/robots.txt
-http://variety.com/robots.txt
+http://scientificamerican.com/robots.txt
+http://sonymusic.co.jp/robots.txt
+http://ssl.google-analytics.com/robots.txt
+http://theguardian.com/robots.txt
+http://today.com/robots.txt
+http://video.google.com/robots.txt
+http://vpro.nl/robots.txt
+http://w3techs.com/robots.txt
+http://yandex.ru/robots.txt
+http://yumpu.com/robots.txt
 hu
 hu-*/
 hu-hu/
@@ -14870,7 +14738,7 @@ intcmp=
 integrate/
 integromedb
 intelligencer/amp
-interactive*http://click.linksynergy.com/robots.txt
+interactive*http://istockphoto.com/robots.txt
 interactive/reading-preference
 interactives
 interface/pda
@@ -15087,7 +14955,7 @@ it/search*price_bucket=
 it/search*search_type=
 it/search*ship_to=
 it/search/autocomplete
-it/searchhttp://wix.com/robots.txt
+it/searchhttp://2.bp.blogspot.com/robots.txt
 it/share
 it/shop_name_search_service
 it/solutions/case-studies/
@@ -17500,7 +17368,6 @@ nl/firefox/collections/mozilla
 nl/firefox/collections/mozilla/$
 nl/firefox/search
 nl/forums
-nl/fr
 nl/growth-tools-assets
 nl/guest-pass
 nl/help/sitemap
@@ -19014,6 +18881,8 @@ premiumsupport/iam/
 premiumsupport/knowledge-center/feedback-no/
 premiumsupport/knowledge-center/feedback-yes/
 preorders
+prep/gnumaint
+prep/wrappers-and-scripts
 present
 presentation
 preserv/extranet
@@ -19656,7 +19525,6 @@ redirect=
 redirect=mobile
 redirect?
 redirect?url=
-redirectSearch?
 redirector
 redirector
 rediscover
@@ -20095,6 +19963,7 @@ sap/saphana/pilot/
 sap/saphana/pilot/offer/
 sap/saphana/pilot/offer/confirmation
 sars-cov-2
+savannah-checkouts
 save
 save-document
 save-query
@@ -20195,6 +20064,7 @@ scorecard
 scorecard/
 screener/insider
 screensaver
+screenshots
 script
 scripts
 scripts/accountextlogin.php
@@ -20530,7 +20400,7 @@ section/3
 section/36
 section/6
 section/8
-section/9http://shareasale.com/robots.txt
+section/9http://amazon.in/robots.txt
 sectionized
 sections
 sections$
@@ -20642,7 +20512,25 @@ seokicks
 series
 server
 server-status
+server/banners
+server/body-include
+server/bottom-notes
 server/docs/search
+server/footer
+server/fs-gang
+server/generic
+server/gnun
+server/head-include
+server/header
+server/html5-head-include
+server/html5-header
+server/include-file-list
+server/outdated
+server/select-language.html
+server/source
+server/staging
+server/top-addendum
+server/trans-map.html
 servererror.aspx
 service
 service-search/dist/images/loading.gif
@@ -20882,7 +20770,7 @@ showdependency
 shows/
 showtag
 showtime/.requestinvite.html.swp
-showtimes/location/*http://npr.org/robots.txt
+showtimes/location/*http://ad.doubleclick.net/robots.txt
 shr*
 shutterstock
 si
@@ -21455,6 +21343,10 @@ soft/punto/mac/uninstall
 soft/punto/win/uninstall
 soft/win/?
 software
+software/gnun/linc
+software/gnun/proofread
+software/gnun/reports
+software/gnun/test
 software/winboard/whats_new
 sogou
 solr
@@ -22113,7 +22005,7 @@ tag
 tag-manager
 tag-manager-360
 tag/
-tag/*+http://cdbaby.com/robots.txt
+tag/*+http://bizjournals.com/robots.txt
 tag/*/a
 tag/*/a/
 tag/*/b
@@ -22349,9 +22241,9 @@ test5
 test6
 testcases
 testfl
-testhttp://use.typekit.net/robots.txt
+testhttp://pca.st/robots.txt
 testing
-testing/http://amzn.com/robots.txt
+testing/http://ubuntu.com/robots.txt
 testpage-forhomepage
 tests/*
 tetro
@@ -22708,7 +22600,7 @@ transmute/static-2.6
 transmute/static-2.8
 transmute/static-2.9
 transparency
-transparency/http://producthunt.com/robots.txt
+transparency/http://j.mp/robots.txt
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14885
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14895
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14896
@@ -23252,6 +23144,7 @@ urr
 urw
 urw/*/product-reviews
 us
+us-en/careers/admin/careersxmlresult
 us-es*/listing/*/favoriters
 us-es*/people/*/circle
 us-es*/people/*/favorites
@@ -23526,8 +23419,10 @@ userimages
 userinfo
 users
 users/*/comments
+users/*/foaf
 users/*/listings
 users/*/replies
+users/*/rss
 users/*/snippets
 users/*/top_artist_iq_rank
 users/*?
@@ -24908,9 +24803,9 @@ wp-admin
 wp-admin/
 wp-admin/admin-ajax.php
 wp-admin/admin-ajax.php
-wp-admin/admin-ajax.phphttp://dictionary.com/robots.txt
-wp-admin/admin-ajax.phphttp://hayhouse.com/robots.txt
-wp-admin/admin-ajax.phphttp://store.cdbaby.com/robots.txt
+wp-admin/admin-ajax.phphttp://amazon.com.br/robots.txt
+wp-admin/admin-ajax.phphttp://edu.gcfglobal.org/robots.txt
+wp-admin/admin-ajax.phphttp://mirror.co.uk/robots.txt
 wp-admin/load-scripts.php
 wp-admin/load-styles.php
 wp-adv
@@ -24981,7 +24876,7 @@ wp-content/uploads/
 wp-content/uploads/wc-logs
 wp-content/uploads/woocommerce_transient_files
 wp-content/uploads/woocommerce_uploads
-wp-content/uploads/wp-import-export-lite/http://brownpapertickets.com/robots.txt
+wp-content/uploads/wp-import-export-lite/http://allrecipes.com/robots.txt
 wp-dyn
 wp-includes
 wp-includes*.css
@@ -25114,7 +25009,7 @@ your-shop
 yourAccount
 youraccount
 youreurope/businessACC
-youreurope/businessDEV/http://photos.google.com/robots.txt
+youreurope/businessDEV/http://businesswire.com/robots.txt
 youreurope/citizensACC
 youreurope/citizensDEV
 yours