From a21bd50622adb1d6d7a13246c705a661ed4c85b9 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Mon, 11 Nov 2024 10:04:18 +0000
Subject: [PATCH] [Github Action] Automated trickest wordlists update.

---
 .bin/wordlist-updaters/status.json            |   2 +-
 .../espocrm-all-levels.txt                    | 105 ---
 .../CMS/trickest-cms-wordlist/espocrm.txt     |  21 -
 .../flarum-all-levels.txt                     |  18 +-
 .../CMS/trickest-cms-wordlist/flarum.txt      |   3 +-
 .../opensourcepos-all-levels.txt              |   6 +-
 .../trickest-cms-wordlist/opensourcepos.txt   |   2 +-
 .../woocommerce-all-levels.txt                |  61 ++
 .../CMS/trickest-cms-wordlist/woocommerce.txt |   9 +
 .../top-100.txt                               |   4 +-
 .../top-1000.txt                              | 376 ++------
 .../top-10000.txt                             | 809 ++++++++++--------
 12 files changed, 592 insertions(+), 824 deletions(-)

diff --git a/.bin/wordlist-updaters/status.json b/.bin/wordlist-updaters/status.json
index f6ed6c0ec..3bbc7b7d7 100644
--- a/.bin/wordlist-updaters/status.json
+++ b/.bin/wordlist-updaters/status.json
@@ -3,6 +3,6 @@
         "last_update": 1731265438
     },
     "Trickest wordlist update": {
-        "last_update": 1731233000
+        "last_update": 1731319448
     }
 }
\ No newline at end of file
diff --git a/Discovery/Web-Content/CMS/trickest-cms-wordlist/espocrm-all-levels.txt b/Discovery/Web-Content/CMS/trickest-cms-wordlist/espocrm-all-levels.txt
index 5a128e945..2d33a55b1 100644
--- a/Discovery/Web-Content/CMS/trickest-cms-wordlist/espocrm-all-levels.txt
+++ b/Discovery/Web-Content/CMS/trickest-cms-wordlist/espocrm-all-levels.txt
@@ -8798,18 +8798,6 @@ FormulaFunctions/ExtGroup/AccountGroup/FindByEmailAddressType.php
 ExtGroup/AccountGroup/FindByEmailAddressType.php
 AccountGroup/FindByEmailAddressType.php
 FindByEmailAddressType.php
-application/Espo/Modules/Crm/Services/TargetList.php
-Espo/Modules/Crm/Services/TargetList.php
-Modules/Crm/Services/TargetList.php
-Crm/Services/TargetList.php
-Services/TargetList.php
-TargetList.php
-application/Espo/Modules/Crm/Services/Opportunity.php
-Espo/Modules/Crm/Services/Opportunity.php
-Modules/Crm/Services/Opportunity.php
-Crm/Services/Opportunity.php
-Services/Opportunity.php
-Opportunity.php
 application/Espo/Modules/Crm/Binding.php
 Espo/Modules/Crm/Binding.php
 Modules/Crm/Binding.php
@@ -19165,30 +19153,14 @@ application/Espo/Services/Import.php
 Espo/Services/Import.php
 Services/Import.php
 Import.php
-application/Espo/Services/Pdf.php
-Espo/Services/Pdf.php
-Services/Pdf.php
-Pdf.php
-application/Espo/Services/EmailTemplate.php
-Espo/Services/EmailTemplate.php
-Services/EmailTemplate.php
-EmailTemplate.php
 application/Espo/Services/ExternalAccount.php
 Espo/Services/ExternalAccount.php
 Services/ExternalAccount.php
 ExternalAccount.php
-application/Espo/Services/InboundEmail.php
-Espo/Services/InboundEmail.php
-Services/InboundEmail.php
-InboundEmail.php
 application/Espo/Services/RecordTree.php
 Espo/Services/RecordTree.php
 Services/RecordTree.php
 RecordTree.php
-application/Espo/Services/EmailAccount.php
-Espo/Services/EmailAccount.php
-Services/EmailAccount.php
-EmailAccount.php
 application/Espo/Services/Integration.php
 Espo/Services/Integration.php
 Services/Integration.php
@@ -20099,22 +20071,12 @@ Core/Acl/AccessChecker/ScopeChecker.php
 Acl/AccessChecker/ScopeChecker.php
 AccessChecker/ScopeChecker.php
 ScopeChecker.php
-application/Espo/Core/Acl/Acl.php
-Espo/Core/Acl/Acl.php
-Core/Acl/Acl.php
-Acl/Acl.php
-Acl.php
 application/Espo/Core/Acl/Cache/Clearer.php
 Espo/Core/Acl/Cache/Clearer.php
 Core/Acl/Cache/Clearer.php
 Acl/Cache/Clearer.php
 Cache/Clearer.php
 Clearer.php
-application/Espo/Core/Acl/Base.php
-Espo/Core/Acl/Base.php
-Core/Acl/Base.php
-Acl/Base.php
-Base.php
 application/Espo/Core/Acl/Table.php
 Espo/Core/Acl/Table.php
 Core/Acl/Table.php
@@ -21300,16 +21262,6 @@ Espo/Core/MassAction/MassAction.php
 Core/MassAction/MassAction.php
 MassAction/MassAction.php
 MassAction.php
-application/Espo/Core/AclPortal/Acl.php
-Espo/Core/AclPortal/Acl.php
-Core/AclPortal/Acl.php
-AclPortal/Acl.php
-Acl.php
-application/Espo/Core/AclPortal/Base.php
-Espo/Core/AclPortal/Base.php
-Core/AclPortal/Base.php
-AclPortal/Base.php
-Base.php
 application/Espo/Core/Field/PhoneNumber.php
 Espo/Core/Field/PhoneNumber.php
 Core/Field/PhoneNumber.php
@@ -21339,12 +21291,6 @@ Core/Field/Currency/CurrencyFactory.php
 Field/Currency/CurrencyFactory.php
 Currency/CurrencyFactory.php
 CurrencyFactory.php
-application/Espo/Core/Field/Currency/CurrencyConverter.php
-Espo/Core/Field/Currency/CurrencyConverter.php
-Core/Field/Currency/CurrencyConverter.php
-Field/Currency/CurrencyConverter.php
-Currency/CurrencyConverter.php
-CurrencyConverter.php
 application/Espo/Core/Field/EmailAddress.php
 Espo/Core/Field/EmailAddress.php
 Core/Field/EmailAddress.php
@@ -21958,11 +21904,6 @@ application/Espo/Core/AclManager.php
 Espo/Core/AclManager.php
 Core/AclManager.php
 AclManager.php
-application/Espo/Core/Services/Base.php
-Espo/Core/Services/Base.php
-Core/Services/Base.php
-Services/Base.php
-Base.php
 application/Espo/Core/Action/Api/PostProcess.php
 Espo/Core/Action/Api/PostProcess.php
 Core/Action/Api/PostProcess.php
@@ -24849,12 +24790,6 @@ Core/ExternalAccount/Clients/Google.php
 ExternalAccount/Clients/Google.php
 Clients/Google.php
 Google.php
-application/Espo/Core/ORM/Repositories/RDB.php
-Espo/Core/ORM/Repositories/RDB.php
-Core/ORM/Repositories/RDB.php
-ORM/Repositories/RDB.php
-Repositories/RDB.php
-RDB.php
 application/Espo/Core/ORM/MetadataDataProvider.php
 Espo/Core/ORM/MetadataDataProvider.php
 Core/ORM/MetadataDataProvider.php
@@ -24883,11 +24818,6 @@ Espo/Core/ORM/EntityManagerFactory.php
 Core/ORM/EntityManagerFactory.php
 ORM/EntityManagerFactory.php
 EntityManagerFactory.php
-application/Espo/Core/ORM/Repository.php
-Espo/Core/ORM/Repository.php
-Core/ORM/Repository.php
-ORM/Repository.php
-Repository.php
 application/Espo/Core/ORM/EntityManager.php
 Espo/Core/ORM/EntityManager.php
 Core/ORM/EntityManager.php
@@ -25590,12 +25520,6 @@ Core/Select/AccessControl/DefaultPortalFilterResolver.php
 Select/AccessControl/DefaultPortalFilterResolver.php
 AccessControl/DefaultPortalFilterResolver.php
 DefaultPortalFilterResolver.php
-application/Espo/Core/Select/AccessControl/BypassFilterResolver.php
-Espo/Core/Select/AccessControl/BypassFilterResolver.php
-Core/Select/AccessControl/BypassFilterResolver.php
-Select/AccessControl/BypassFilterResolver.php
-AccessControl/BypassFilterResolver.php
-BypassFilterResolver.php
 application/Espo/Core/Select/Where/Item.php
 Espo/Core/Select/Where/Item.php
 Core/Select/Where/Item.php
@@ -26075,16 +25999,6 @@ Core/Hook/Hook/AfterSave.php
 Hook/Hook/AfterSave.php
 Hook/AfterSave.php
 AfterSave.php
-application/Espo/Core/Notificators/DefaultNotificator.php
-Espo/Core/Notificators/DefaultNotificator.php
-Core/Notificators/DefaultNotificator.php
-Notificators/DefaultNotificator.php
-DefaultNotificator.php
-application/Espo/Core/Notificators/Base.php
-Espo/Core/Notificators/Base.php
-Core/Notificators/Base.php
-Notificators/Base.php
-Base.php
 application/Espo/Core/WebSocket/ServerStarter.php
 Espo/Core/WebSocket/ServerStarter.php
 Core/WebSocket/ServerStarter.php
@@ -27058,11 +26972,6 @@ Espo/Core/Entities/CategoryTreeItem.php
 Core/Entities/CategoryTreeItem.php
 Entities/CategoryTreeItem.php
 CategoryTreeItem.php
-application/Espo/Core/Cleanup/Base.php
-Espo/Core/Cleanup/Base.php
-Core/Cleanup/Base.php
-Cleanup/Base.php
-Base.php
 application/Espo/Core/Cleanup/Cleanup.php
 Espo/Core/Cleanup/Cleanup.php
 Core/Cleanup/Cleanup.php
@@ -27326,16 +27235,6 @@ Espo/Core/EntryPoints/Base.php
 Core/EntryPoints/Base.php
 EntryPoints/Base.php
 Base.php
-application/Espo/Core/Interfaces/Injectable.php
-Espo/Core/Interfaces/Injectable.php
-Core/Interfaces/Injectable.php
-Interfaces/Injectable.php
-Injectable.php
-application/Espo/Core/Hooks/Base.php
-Espo/Core/Hooks/Base.php
-Core/Hooks/Base.php
-Hooks/Base.php
-Base.php
 application/Espo/Core/Formula/ArgumentList.php
 Espo/Core/Formula/ArgumentList.php
 Core/Formula/ArgumentList.php
@@ -28663,10 +28562,6 @@ application/Espo/Core/Application.php
 Espo/Core/Application.php
 Core/Application.php
 Application.php
-application/Espo/Core/Injectable.php
-Espo/Core/Injectable.php
-Core/Injectable.php
-Injectable.php
 application/Espo/Core/Controllers/Record.php
 Espo/Core/Controllers/Record.php
 Core/Controllers/Record.php
diff --git a/Discovery/Web-Content/CMS/trickest-cms-wordlist/espocrm.txt b/Discovery/Web-Content/CMS/trickest-cms-wordlist/espocrm.txt
index 31e6c7a0a..88ccef5ee 100644
--- a/Discovery/Web-Content/CMS/trickest-cms-wordlist/espocrm.txt
+++ b/Discovery/Web-Content/CMS/trickest-cms-wordlist/espocrm.txt
@@ -1466,8 +1466,6 @@ application/Espo/Modules/Crm/Classes/FieldProcessing/TargetList/OptedOutCountLoa
 application/Espo/Modules/Crm/Classes/FieldProcessing/Campaign/StatsLoader.php
 application/Espo/Modules/Crm/Classes/FormulaFunctions/ExtGroup/CalendarGroup/UserIsBusyType.php
 application/Espo/Modules/Crm/Classes/FormulaFunctions/ExtGroup/AccountGroup/FindByEmailAddressType.php
-application/Espo/Modules/Crm/Services/TargetList.php
-application/Espo/Modules/Crm/Services/Opportunity.php
 application/Espo/Modules/Crm/Binding.php
 application/Espo/Modules/Crm/Resources/module.json
 application/Espo/Modules/Crm/Resources/data/freeEmailProviderDomains.json
@@ -2775,12 +2773,8 @@ application/Espo/Services/User.php
 application/Espo/Services/Email.php
 application/Espo/Services/Record.php
 application/Espo/Services/Import.php
-application/Espo/Services/Pdf.php
-application/Espo/Services/EmailTemplate.php
 application/Espo/Services/ExternalAccount.php
-application/Espo/Services/InboundEmail.php
 application/Espo/Services/RecordTree.php
-application/Espo/Services/EmailAccount.php
 application/Espo/Services/Integration.php
 application/Espo/ORM/Value/ValueAccessor.php
 application/Espo/ORM/Value/GeneralAttributeExtractor.php
@@ -2953,9 +2947,7 @@ application/Espo/Core/Acl/AccessChecker/AccessCheckerFactory.php
 application/Espo/Core/Acl/AccessChecker/ScopeCheckerData.php
 application/Espo/Core/Acl/AccessChecker/AccessCheckers/Foreign.php
 application/Espo/Core/Acl/AccessChecker/ScopeChecker.php
-application/Espo/Core/Acl/Acl.php
 application/Espo/Core/Acl/Cache/Clearer.php
-application/Espo/Core/Acl/Base.php
 application/Espo/Core/Acl/Table.php
 application/Espo/Core/Acl/AccessEntityEditChecker.php
 application/Espo/Core/Acl/AccessEntityDeleteChecker.php
@@ -3173,14 +3165,11 @@ application/Espo/Core/MassAction/Data.php
 application/Espo/Core/MassAction/ServiceResult.php
 application/Espo/Core/MassAction/QueryBuilder.php
 application/Espo/Core/MassAction/MassAction.php
-application/Espo/Core/AclPortal/Acl.php
-application/Espo/Core/AclPortal/Base.php
 application/Espo/Core/Field/PhoneNumber.php
 application/Espo/Core/Field/LinkMultiple/LinkMultipleAttributeExtractor.php
 application/Espo/Core/Field/LinkMultiple/LinkMultipleFactory.php
 application/Espo/Core/Field/Currency/CurrencyAttributeExtractor.php
 application/Espo/Core/Field/Currency/CurrencyFactory.php
-application/Espo/Core/Field/Currency/CurrencyConverter.php
 application/Espo/Core/Field/EmailAddress.php
 application/Espo/Core/Field/Address/AddressFormatterFactory.php
 application/Espo/Core/Field/Address/AddressAttributeExtractor.php
@@ -3284,7 +3273,6 @@ application/Espo/Core/PhoneNumber/Util.php
 application/Espo/Core/PhoneNumber/Sanitizer.php
 application/Espo/Core/Container.php
 application/Espo/Core/AclManager.php
-application/Espo/Core/Services/Base.php
 application/Espo/Core/Action/Api/PostProcess.php
 application/Espo/Core/Action/Service.php
 application/Espo/Core/Action/Params.php
@@ -3726,13 +3714,11 @@ application/Espo/Core/ExternalAccount/OAuth2/Client.php
 application/Espo/Core/ExternalAccount/Clients/OAuth2Abstract.php
 application/Espo/Core/ExternalAccount/Clients/IClient.php
 application/Espo/Core/ExternalAccount/Clients/Google.php
-application/Espo/Core/ORM/Repositories/RDB.php
 application/Espo/Core/ORM/MetadataDataProvider.php
 application/Espo/Core/ORM/Repository/HookMediator.php
 application/Espo/Core/ORM/Repository/Option/SaveOption.php
 application/Espo/Core/ORM/Entity.php
 application/Espo/Core/ORM/EntityManagerFactory.php
-application/Espo/Core/ORM/Repository.php
 application/Espo/Core/ORM/EntityManager.php
 application/Espo/Core/ORM/Defs/AttributeParam.php
 application/Espo/Core/ORM/EntityManagerProxy.php
@@ -3862,7 +3848,6 @@ application/Espo/Core/Select/AccessControl/Filters/PortalAll.php
 application/Espo/Core/Select/AccessControl/FilterFactory.php
 application/Espo/Core/Select/AccessControl/DefaultFilterResolver.php
 application/Espo/Core/Select/AccessControl/DefaultPortalFilterResolver.php
-application/Espo/Core/Select/AccessControl/BypassFilterResolver.php
 application/Espo/Core/Select/Where/Item.php
 application/Espo/Core/Select/Where/CheckerFactory.php
 application/Espo/Core/Select/Where/ItemConverterFactory.php
@@ -3939,8 +3924,6 @@ application/Espo/Core/Hook/Hook/AfterRemove.php
 application/Espo/Core/Hook/Hook/AfterUnrelate.php
 application/Espo/Core/Hook/Hook/BeforeSave.php
 application/Espo/Core/Hook/Hook/AfterSave.php
-application/Espo/Core/Notificators/DefaultNotificator.php
-application/Espo/Core/Notificators/Base.php
 application/Espo/Core/WebSocket/ServerStarter.php
 application/Espo/Core/WebSocket/SubscriberFactory.php
 application/Espo/Core/WebSocket/ZeroMQSubscriber.php
@@ -4106,7 +4089,6 @@ application/Espo/Core/EntryPoint/Traits/NoAuth.php
 application/Espo/Core/EntryPoint/EntryPointManager.php
 application/Espo/Core/Entities/Person.php
 application/Espo/Core/Entities/CategoryTreeItem.php
-application/Espo/Core/Cleanup/Base.php
 application/Espo/Core/Cleanup/Cleanup.php
 application/Espo/Core/Acl.php
 application/Espo/Core/Upgrades/UpgradeManager.php
@@ -4149,8 +4131,6 @@ application/Espo/Core/EntryPoints/EntryPoint.php
 application/Espo/Core/EntryPoints/NotStrictAuth.php
 application/Espo/Core/EntryPoints/NoAuth.php
 application/Espo/Core/EntryPoints/Base.php
-application/Espo/Core/Interfaces/Injectable.php
-application/Espo/Core/Hooks/Base.php
 application/Espo/Core/Formula/ArgumentList.php
 application/Espo/Core/Formula/Argument.php
 application/Espo/Core/Formula/FunctionFactory.php
@@ -4347,7 +4327,6 @@ application/Espo/Core/Application/Exceptions/RunnerException.php
 application/Espo/Core/Application/RunnerRunner.php
 application/Espo/Core/Application/Runner.php
 application/Espo/Core/Application.php
-application/Espo/Core/Injectable.php
 application/Espo/Core/Controllers/Record.php
 application/Espo/Core/Controllers/RecordTree.php
 application/Espo/Core/Controllers/RecordBase.php
diff --git a/Discovery/Web-Content/CMS/trickest-cms-wordlist/flarum-all-levels.txt b/Discovery/Web-Content/CMS/trickest-cms-wordlist/flarum-all-levels.txt
index 7749ec548..835776dd8 100644
--- a/Discovery/Web-Content/CMS/trickest-cms-wordlist/flarum-all-levels.txt
+++ b/Discovery/Web-Content/CMS/trickest-cms-wordlist/flarum-all-levels.txt
@@ -10048,12 +10048,6 @@ tests/integration/extenders/FrontendContentTest.php
 integration/extenders/FrontendContentTest.php
 extenders/FrontendContentTest.php
 FrontendContentTest.php
-framework/core/tests/integration/extenders/ApiControllerTest.php
-core/tests/integration/extenders/ApiControllerTest.php
-tests/integration/extenders/ApiControllerTest.php
-integration/extenders/ApiControllerTest.php
-extenders/ApiControllerTest.php
-ApiControllerTest.php
 framework/core/tests/integration/extenders/FrontendTitleTest.php
 core/tests/integration/extenders/FrontendTitleTest.php
 tests/integration/extenders/FrontendTitleTest.php
@@ -10162,12 +10156,6 @@ tests/integration/extenders/SessionTest.php
 integration/extenders/SessionTest.php
 extenders/SessionTest.php
 SessionTest.php
-framework/core/tests/integration/extenders/ApiSerializerTest.php
-core/tests/integration/extenders/ApiSerializerTest.php
-tests/integration/extenders/ApiSerializerTest.php
-integration/extenders/ApiSerializerTest.php
-extenders/ApiSerializerTest.php
-ApiSerializerTest.php
 framework/core/tests/integration/extenders/ValidatorTest.php
 core/tests/integration/extenders/ValidatorTest.php
 tests/integration/extenders/ValidatorTest.php
@@ -10180,6 +10168,12 @@ tests/integration/extenders/PostTest.php
 integration/extenders/PostTest.php
 extenders/PostTest.php
 PostTest.php
+framework/core/tests/integration/extenders/ApiResourceTest.php
+core/tests/integration/extenders/ApiResourceTest.php
+tests/integration/extenders/ApiResourceTest.php
+integration/extenders/ApiResourceTest.php
+extenders/ApiResourceTest.php
+ApiResourceTest.php
 framework/core/tests/integration/extenders/ConditionalTest.php
 core/tests/integration/extenders/ConditionalTest.php
 tests/integration/extenders/ConditionalTest.php
diff --git a/Discovery/Web-Content/CMS/trickest-cms-wordlist/flarum.txt b/Discovery/Web-Content/CMS/trickest-cms-wordlist/flarum.txt
index d3ad96996..c97917c6b 100644
--- a/Discovery/Web-Content/CMS/trickest-cms-wordlist/flarum.txt
+++ b/Discovery/Web-Content/CMS/trickest-cms-wordlist/flarum.txt
@@ -1890,7 +1890,6 @@ framework/core/tests/integration/extenders/FrontendPreloadTest.php
 framework/core/tests/integration/extenders/ConsoleTest.php
 framework/core/tests/integration/extenders/PolicyTest.php
 framework/core/tests/integration/extenders/FrontendContentTest.php
-framework/core/tests/integration/extenders/ApiControllerTest.php
 framework/core/tests/integration/extenders/FrontendTitleTest.php
 framework/core/tests/integration/extenders/ModelTest.php
 framework/core/tests/integration/extenders/MailTest.php
@@ -1909,9 +1908,9 @@ framework/core/tests/integration/extenders/FormatterTest.php
 framework/core/tests/integration/extenders/ThrottleApiTest.php
 framework/core/tests/integration/extenders/RoutesTest.php
 framework/core/tests/integration/extenders/SessionTest.php
-framework/core/tests/integration/extenders/ApiSerializerTest.php
 framework/core/tests/integration/extenders/ValidatorTest.php
 framework/core/tests/integration/extenders/PostTest.php
+framework/core/tests/integration/extenders/ApiResourceTest.php
 framework/core/tests/integration/extenders/ConditionalTest.php
 framework/core/tests/integration/extenders/ThemeTest.php
 framework/core/tests/integration/policy/DiscussionPolicyTest.php
diff --git a/Discovery/Web-Content/CMS/trickest-cms-wordlist/opensourcepos-all-levels.txt b/Discovery/Web-Content/CMS/trickest-cms-wordlist/opensourcepos-all-levels.txt
index 23b337a8b..613f974d4 100644
--- a/Discovery/Web-Content/CMS/trickest-cms-wordlist/opensourcepos-all-levels.txt
+++ b/Discovery/Web-Content/CMS/trickest-cms-wordlist/opensourcepos-all-levels.txt
@@ -177,12 +177,12 @@ writable/.htaccess
 .htaccess
 writable/index.html
 index.html
-writable/uploads/import_customers.csv
-uploads/import_customers.csv
-import_customers.csv
 writable/uploads/index.html
 uploads/index.html
 index.html
+writable/uploads/importCustomers.csv
+uploads/importCustomers.csv
+importCustomers.csv
 writable/session/index.html
 session/index.html
 index.html
diff --git a/Discovery/Web-Content/CMS/trickest-cms-wordlist/opensourcepos.txt b/Discovery/Web-Content/CMS/trickest-cms-wordlist/opensourcepos.txt
index b8d1432b1..f7ba72219 100644
--- a/Discovery/Web-Content/CMS/trickest-cms-wordlist/opensourcepos.txt
+++ b/Discovery/Web-Content/CMS/trickest-cms-wordlist/opensourcepos.txt
@@ -73,8 +73,8 @@ docker-compose.nginx.yml
 docker-compose.test.yml
 writable/.htaccess
 writable/index.html
-writable/uploads/import_customers.csv
 writable/uploads/index.html
+writable/uploads/importCustomers.csv
 writable/session/index.html
 writable/cache/.htaccess
 writable/cache/index.html
diff --git a/Discovery/Web-Content/CMS/trickest-cms-wordlist/woocommerce-all-levels.txt b/Discovery/Web-Content/CMS/trickest-cms-wordlist/woocommerce-all-levels.txt
index 28c662eea..c0d94d5cd 100644
--- a/Discovery/Web-Content/CMS/trickest-cms-wordlist/woocommerce-all-levels.txt
+++ b/Discovery/Web-Content/CMS/trickest-cms-wordlist/woocommerce-all-levels.txt
@@ -27485,6 +27485,13 @@ Admin/Features/MarketingRecommendations/Init.php
 Features/MarketingRecommendations/Init.php
 MarketingRecommendations/Init.php
 Init.php
+plugins/woocommerce/src/Admin/Features/MarketingRecommendations/MiscRecommendationsDataSourcePoller.php
+woocommerce/src/Admin/Features/MarketingRecommendations/MiscRecommendationsDataSourcePoller.php
+src/Admin/Features/MarketingRecommendations/MiscRecommendationsDataSourcePoller.php
+Admin/Features/MarketingRecommendations/MiscRecommendationsDataSourcePoller.php
+Features/MarketingRecommendations/MiscRecommendationsDataSourcePoller.php
+MarketingRecommendations/MiscRecommendationsDataSourcePoller.php
+MiscRecommendationsDataSourcePoller.php
 plugins/woocommerce/src/Admin/Features/ProductBlockEditor/ProductFormsController.php
 woocommerce/src/Admin/Features/ProductBlockEditor/ProductFormsController.php
 src/Admin/Features/ProductBlockEditor/ProductFormsController.php
@@ -43155,6 +43162,10 @@ plugins/woocommerce/changelog/e2e-remove-color-palette-iterations
 woocommerce/changelog/e2e-remove-color-palette-iterations
 changelog/e2e-remove-color-palette-iterations
 e2e-remove-color-palette-iterations
+plugins/woocommerce/changelog/52661-feature-51567-order-attribution-install-and-connect
+woocommerce/changelog/52661-feature-51567-order-attribution-install-and-connect
+changelog/52661-feature-51567-order-attribution-install-and-connect
+52661-feature-51567-order-attribution-install-and-connect
 plugins/woocommerce/changelog/51725-dev-tt25-theme-compatibility
 woocommerce/changelog/51725-dev-tt25-theme-compatibility
 changelog/51725-dev-tt25-theme-compatibility
@@ -54537,6 +54548,56 @@ client/marketing/hooks/useRecommendedChannels.ts
 marketing/hooks/useRecommendedChannels.ts
 hooks/useRecommendedChannels.ts
 useRecommendedChannels.ts
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/order-attribution-install-banner.js
+woocommerce/client/admin/client/order-attribution-install-banner/order-attribution-install-banner.js
+client/admin/client/order-attribution-install-banner/order-attribution-install-banner.js
+admin/client/order-attribution-install-banner/order-attribution-install-banner.js
+client/order-attribution-install-banner/order-attribution-install-banner.js
+order-attribution-install-banner/order-attribution-install-banner.js
+order-attribution-install-banner.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/constants.js
+woocommerce/client/admin/client/order-attribution-install-banner/constants.js
+client/admin/client/order-attribution-install-banner/constants.js
+admin/client/order-attribution-install-banner/constants.js
+client/order-attribution-install-banner/constants.js
+order-attribution-install-banner/constants.js
+constants.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/style.scss
+woocommerce/client/admin/client/order-attribution-install-banner/style.scss
+client/admin/client/order-attribution-install-banner/style.scss
+admin/client/order-attribution-install-banner/style.scss
+client/order-attribution-install-banner/style.scss
+order-attribution-install-banner/style.scss
+style.scss
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/index.js
+woocommerce/client/admin/client/order-attribution-install-banner/index.js
+client/admin/client/order-attribution-install-banner/index.js
+admin/client/order-attribution-install-banner/index.js
+client/order-attribution-install-banner/index.js
+order-attribution-install-banner/index.js
+index.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/order-attribution-install-banner-image.js
+woocommerce/client/admin/client/order-attribution-install-banner/order-attribution-install-banner-image.js
+client/admin/client/order-attribution-install-banner/order-attribution-install-banner-image.js
+admin/client/order-attribution-install-banner/order-attribution-install-banner-image.js
+client/order-attribution-install-banner/order-attribution-install-banner-image.js
+order-attribution-install-banner/order-attribution-install-banner-image.js
+order-attribution-install-banner-image.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/use-order-attribution-install-banner.js
+woocommerce/client/admin/client/order-attribution-install-banner/use-order-attribution-install-banner.js
+client/admin/client/order-attribution-install-banner/use-order-attribution-install-banner.js
+admin/client/order-attribution-install-banner/use-order-attribution-install-banner.js
+client/order-attribution-install-banner/use-order-attribution-install-banner.js
+order-attribution-install-banner/use-order-attribution-install-banner.js
+use-order-attribution-install-banner.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/order-editor/slot.js
+woocommerce/client/admin/client/order-attribution-install-banner/order-editor/slot.js
+client/admin/client/order-attribution-install-banner/order-editor/slot.js
+admin/client/order-attribution-install-banner/order-editor/slot.js
+client/order-attribution-install-banner/order-editor/slot.js
+order-attribution-install-banner/order-editor/slot.js
+order-editor/slot.js
+slot.js
 plugins/woocommerce/client/admin/client/products/product-variation-page.tsx
 woocommerce/client/admin/client/products/product-variation-page.tsx
 client/admin/client/products/product-variation-page.tsx
diff --git a/Discovery/Web-Content/CMS/trickest-cms-wordlist/woocommerce.txt b/Discovery/Web-Content/CMS/trickest-cms-wordlist/woocommerce.txt
index b4d1ce060..10b2d4ceb 100644
--- a/Discovery/Web-Content/CMS/trickest-cms-wordlist/woocommerce.txt
+++ b/Discovery/Web-Content/CMS/trickest-cms-wordlist/woocommerce.txt
@@ -4003,6 +4003,7 @@ plugins/woocommerce/src/Admin/Features/ShippingPartnerSuggestions/DefaultShippin
 plugins/woocommerce/src/Admin/Features/MarketingRecommendations/MarketingRecommendationsDataSourcePoller.php
 plugins/woocommerce/src/Admin/Features/MarketingRecommendations/DefaultMarketingRecommendations.php
 plugins/woocommerce/src/Admin/Features/MarketingRecommendations/Init.php
+plugins/woocommerce/src/Admin/Features/MarketingRecommendations/MiscRecommendationsDataSourcePoller.php
 plugins/woocommerce/src/Admin/Features/ProductBlockEditor/ProductFormsController.php
 plugins/woocommerce/src/Admin/Features/ProductBlockEditor/ProductTemplates/GroupInterface.php
 plugins/woocommerce/src/Admin/Features/ProductBlockEditor/ProductTemplates/SectionInterface.php
@@ -6405,6 +6406,7 @@ plugins/woocommerce/changelog/fix-obw-continue-button-overlap
 plugins/woocommerce/changelog/dev-consolidate-syncpack-config
 plugins/woocommerce/changelog/fix-e2e-test-core-profiler
 plugins/woocommerce/changelog/e2e-remove-color-palette-iterations
+plugins/woocommerce/changelog/52661-feature-51567-order-attribution-install-and-connect
 plugins/woocommerce/changelog/51725-dev-tt25-theme-compatibility
 plugins/woocommerce/changelog/fix-mobile-app-modal-ui
 plugins/woocommerce/changelog/52070-fix-use-method-title-or-gateway-title-in-checkout-notice
@@ -7969,6 +7971,13 @@ plugins/woocommerce/client/admin/client/marketing/hooks/test/useCampaigns.test.t
 plugins/woocommerce/client/admin/client/marketing/hooks/useIntroductionBanner.ts
 plugins/woocommerce/client/admin/client/marketing/hooks/useCampaignTypes.ts
 plugins/woocommerce/client/admin/client/marketing/hooks/useRecommendedChannels.ts
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/order-attribution-install-banner.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/constants.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/style.scss
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/index.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/order-attribution-install-banner-image.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/use-order-attribution-install-banner.js
+plugins/woocommerce/client/admin/client/order-attribution-install-banner/order-editor/slot.js
 plugins/woocommerce/client/admin/client/products/product-variation-page.tsx
 plugins/woocommerce/client/admin/client/products/product-page.scss
 plugins/woocommerce/client/admin/client/products/tour/components/guide/index.tsx
diff --git a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt
index 9f7cec711..f43bbfadc 100644
--- a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt
+++ b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt
@@ -2774,8 +2774,8 @@ hotelfinder/rpc
 hotels/rpc
 hp/video
 hpc/*/supported-applications.aspx?
-http://creativecommons.org/robots.txt
-http://slideshare.net/robots.txt
+http://ec.europa.eu/robots.txt
+http://news.google.com/robots.txt
 hub?category=
 hz/contact-us/ajax/initiate-trusted-contact
 hz/help/contact/*/message/$
diff --git a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt
index a04c0918f..87df254d0 100644
--- a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt
+++ b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-1000.txt
@@ -2553,7 +2553,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /capital/consumer/invoice/merchants
 /cars/marketplace/product/*
 /cart
-/cart
 /cas-redirect
 /cast-list
 /castr
@@ -2571,7 +2570,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /chats/
 /checkin
 /checkout
-/checkout
 /cinema-entertainment/*
 /citations
 /citedby
@@ -3014,7 +3012,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /login
 /login$
 /login*/search?
-/login/
 /login/verify-email
 /login?
 /loginpage
@@ -3080,7 +3077,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /muslim-sexy-video
 /muslim-sexy-videos.html
 /mvopinions/*
-/my-account
 /my_account/register
 /mychannel
 /naboo
@@ -3121,7 +3117,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /newssearch
 /no-internet-heading-assigned
 /no-internet-heading-assisted
-/no-referrer-when-downgradehttp://steamcommunity.com/robots.txt
+/no-referrer-when-downgradehttp://google.ru/robots.txt
 /no-search-results
 /node
 /node/add
@@ -3272,7 +3268,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /recommendations/products
 /references
 /referrer$
-/register/
 /registration-confirmation/
 /related
 /related
@@ -3692,7 +3687,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 0;if((a=google.c)==null?0:a.cwh)a=void
 ;jsessionid
 ;preview=
-<script
 =$
 ?
 ?
@@ -3713,7 +3707,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*__rmid
 ?*__rmidpage
 ?*ajs_event
-?*apikey*http://w3techs.com/robots.txt
+?*apikey*http://bmj.com/robots.txt
 ?*app
 ?*appidt=*
 ?*aspects=
@@ -3830,7 +3824,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 ?*utm_medium
 ?*utm_source
 ?*utm_source=
-?*utm_source=http://pixel.quantserve.com/robots.txt
+?*utm_source=http://mozilla.com/robots.txt
 ?*verified=
 ?*xid
 ?252525253Bpreview=
@@ -4318,10 +4312,8 @@ AcceptDeferredTermsAndConditions
 AccommodationCrossSells
 AccommodationTips
 Account
-AccountAccess
 AccountDisavow?
 AccountMerge
-AccountStatus
 Achievements
 ActLikeAnOwner
 ActionRecord
@@ -4403,7 +4395,6 @@ App_Data/
 App_Themes/
 App_config/
 AppleMapsAction
-Arabic
 Archive/New_Records_Testing
 Archive/PrintMaterial/ACS_Journal/1/files
 Archive/PrintMaterial/ACS_Journal/10/files
@@ -5262,7 +5253,6 @@ BeachSweeps
 Bender
 Benoit/Public/DVDDB
 BetterBySea
-BillingActivity
 BillingAndPayments
 BillingDetail
 BillingInvoiceFile
@@ -5289,14 +5279,12 @@ CampaignConfiguration
 CampaignId
 CampaignManagement
 CancelBooking
-CancelPlan
 CaptchaValidationLog
 CarsComparePartners
 Cart
 CauseTeaserPopup
 Celebrate100Million
 CertificateOfExcellenceRequestSupport
-ChangePlan
 CheapFlightsCommerce
 CheapFlightsController
 CheapFlightsHome
@@ -5311,7 +5299,6 @@ CheckoutEditItemsDisplayView
 CheckoutOrderBillingView
 CheckoutOrderShippingView
 CheckoutSignInView
-Chinese
 Citation/Download
 Civ
 Claim
@@ -5384,7 +5371,6 @@ Disallow
 Disallow:
 DiscountGroup
 DisplayMap
-DoNotTest
 Doc
 Domains/DomainControlPanel
 Download
@@ -5395,14 +5381,12 @@ ESPDisplayOptionsViewCmd
 ESPModal
 EULegal
 EarlyRequesterJS/static-1.38
-EditProfiles
 ElasticLoadBalancing/2009-05-15
 ElasticLoadBalancing/2010-07-01
 EmailCartView
 EmailHotel
 EmailMap
 EmailMemberRegistration
-EmailPreferences
 EmailSpellCheckJson
 Emailer
 EmbedSlideshow
@@ -5444,7 +5428,6 @@ FiatsCmd
 FilterState
 FindRestaurants
 FinishAppInstall
-FireTV
 Flamingo_SearchEngine
 Flights?
 FlightsCommerce
@@ -5515,7 +5498,6 @@ HRMConnect
 HTTrack
 HYx10rg3
 HandleHotelSuperPortal
-HdToggle
 HeadJS/static-2.115
 HeadJS/static-2.83
 HeadJS/static-2.84
@@ -5540,7 +5522,6 @@ HotelsManageListing
 HotelsManagementCenter
 HotelsProductCart
 HowTheSiteWorks
-HpImageArchive.aspx
 HubStyle/static-2.309
 HubStyle/static-2.313
 HubStyle/static-2.315
@@ -5641,7 +5622,6 @@ JustForYouGuide
 Jyxobot
 Kimengi
 Kimengi/nineconnections.com
-Korean
 LFE/Live.controls.sharedpages/error.aspx
 LICENSE.txt
 LNAjax
@@ -5649,7 +5629,6 @@ LP
 LanderLogger
 LandingPage
 Landingpages.html
-LanguagePreferences
 Leaderboard
 LexxeBot/1.0
 Library
@@ -5671,9 +5650,7 @@ LocationTab
 LocationWidget
 LogMediaClick
 Login.html
-LoginHelp
 LogonForm
-Logout
 MAINTAINERS.txt
 MAPS
 MJ12bot
@@ -5685,12 +5662,10 @@ MachineTranslation
 Mail/
 MailEmergency
 Manage/
-ManageDevices
 ManageListing
 ManageOrder
 ManageOwnersTopReasonsApi
 ManagePhotos
-ManageProfiles
 ManageReturns
 ManageShopListingApi
 ManageShopping
@@ -5702,7 +5677,6 @@ MapS
 Maps
 MarketPerception
 Marketing/EcapSurvey.aspx
-Maxis
 Maxthon
 McKinsey
 MediaBadgeResponse
@@ -5780,7 +5754,6 @@ MemberSubscribe
 MemberSubscriptionsController
 MemberTagsAjax
 MemberTravelMap
-Meo
 MessageRunDetailID=
 Messages
 MessagingOptions
@@ -5926,7 +5899,6 @@ ModuleAjax
 ModuleBootstrap
 Moreover
 Moreoverbot
-MoviesYouveSeen
 MptUrl
 MsnForumFeedServlet
 MultiOwners
@@ -5962,7 +5934,6 @@ NetSeer
 NewMessage
 NewOwners
 NewTopic
-NewWatchInstantlyRSS
 News
 News/Flash.aspx/
 News/Index
@@ -6034,7 +6005,6 @@ PacMan
 PageBuilderControls
 PageSpeed=noscript
 PageviewRecord
-Panasonic
 PanoramaXml
 Partners
 PasswordReset
@@ -6068,7 +6038,6 @@ PoiRecommendationEditor
 PoiRecommendationList
 PoiReviewDetail
 PoiTipDashboard
-PopupDetails
 PortalIdParser/static-1.27
 PortalIdParser/static-1.28
 PortalIdParser/static-1.29
@@ -6108,12 +6077,10 @@ Products/Reports.aspx
 Products/ShoppingCart.aspx
 Products/subscriptionsSpecial.aspx
 Profile
-ProfilesGate
 Project
 PromoTools
 PromotionDetailsDisplayView
 PromotionDisplayView
-Proxy.ashx
 ProxyToolsAccountSettingsTool
 ProxyToolsBillingAndPaymentsTool
 ProxyToolsBillingDetailTool
@@ -6247,7 +6214,6 @@ Saves
 ScreensaverShade
 Scripts
 Search
-Search
 Search/*
 Search/Mail/Public
 SearchNavigationView
@@ -6310,7 +6276,6 @@ Snap
 SocialIapReporting
 Societies
 SolutionsDisplay
-Sony
 SortAttractions
 SortEateries
 SortHotels
@@ -6550,7 +6515,6 @@ WebReaper
 WebStripper
 WebZIP
 WhatsNearby
-WiViewingActivity
 WidgetBrowserExtension
 WidgetEmbed
 WidgetListAsync.naver
@@ -6568,8 +6532,6 @@ Xenu
 Xhr
 Yandex
 Yasni
-YourAccount
-YourAccountPayment
 Zao
 Zealbot
 ZendeskRedirector
@@ -6862,8 +6824,6 @@ ac/
 ac/globalnav/2.0/*/images/ac-globalnav/globalnav/search
 ac2
 academic-subscriptions
-academic/profile
-academic/search
 academy
 academy-free-account
 academy/*/results
@@ -6927,7 +6887,6 @@ account_settings/set_password?
 account_settings/set_your_password$
 account_settings/set_your_password?
 account_settings?
-accountaccess
 accounthelp
 accounts
 accounts/ClientAuth
@@ -6940,7 +6899,6 @@ accounts/login
 accounts/new
 accounts/o8
 accounts/o8/id
-accountstatus
 acct
 ach/forums
 ach/kb/*/discuss
@@ -6948,7 +6906,6 @@ ach/search
 ach/user
 ach/users
 achievements
-aclick
 aclk
 acoustic
 acq/nonprofits
@@ -7050,7 +7007,6 @@ addtocart
 adell-mutfak
 adfox
 adframe
-adfs/
 admanager
 admantx
 admin
@@ -7159,7 +7115,6 @@ ai
 ai_log
 aihub/tfhub
 aim
-airtel
 ajah
 ajax
 ajax/*
@@ -7233,7 +7188,6 @@ alexa-top-sites/*
 alexa-web-search/*
 alexa/*
 alexaforbusiness/
-alink
 alkali
 all
 all-supported-params
@@ -7260,7 +7214,6 @@ amp
 amp$
 amp-proxy
 amp-tealium
-amp/
 amp_next_page_ga.cms*
 amphtml
 amphtml/blogs/test
@@ -7292,7 +7245,6 @@ analytics/portal
 analytics/seomagic
 analytics/traffic/overview
 analytics/uploads
-anan
 and
 android
 android/downloads
@@ -7331,13 +7283,12 @@ api/editionurlgenerator
 api/eventlogging
 api/federation
 api/html
-api/http://israelnationalnews.com/robots.txt
+api/http://figma.com/robots.txt
 api/internal/*/location
 api/internal/_bb
 api/internal/analytics/songs
 api/jobPostings/jobs
 api/legendary-data/relay
-api/maps/
 api/nextsteps
 api/nicorepo/timeline
 api/partner_select/*/price_query
@@ -7412,7 +7363,7 @@ appchart
 appdynamics-wem.cisco.com
 appealed-pin
 apple-app-site-association
-apple-app-site-associationhttp://freewebs.com/robots.txt
+apple-app-site-associationhttp://mediafire.com/robots.txt
 apple-news
 appliance
 applicant-eligibility
@@ -7828,7 +7779,6 @@ audio_ad?
 audio_file
 audio_news
 audio_player
-aui/inbound
 aura
 aura/promo
 auth
@@ -7838,7 +7788,6 @@ auth/*
 auth/createchallenge
 auth/sso/proxy-login
 auth/verifychallenge
-authenticate
 authentication
 authentication$
 authkey=
@@ -7858,7 +7807,6 @@ autocomplete/projects
 autocomplete/users
 autocompletes
 autodiscover/
-autologin
 autoplay=
 autor/francisco_jose_pedrosa_rodriguez/a/*/
 autor/francisco_jose_pedrosa_rodriguez/a/ 
@@ -8474,7 +8422,6 @@ bezahlvorgaenge.html
 bfe/apps/website
 bfl
 bfmp
-bfp/search
 bg
 bg-*/
 bg-bg/
@@ -8505,7 +8452,6 @@ bin
 bin/
 bin/search.go
 bin/search.py
-bing-site-safety
 bingbot
 bipolar-disorder/bipolar-depression-management
 bipolar-disorder/video/bipolar-fatherhood
@@ -8635,7 +8581,6 @@ blogs/blogpost/preview
 blogs/nationals-journal-beta
 blogs/pad$
 blogs/pad/$
-blogs/search/
 blogs/slow-ride
 blogs/technology.atom
 blogs/test
@@ -8703,7 +8648,7 @@ bookmark_mana2
 bookmarklet
 bookmarks
 bookmarks/widget
-bookmarkshttp://spiegel.de/robots.txt
+bookmarkshttp://openstreetmap.org/robots.txt
 books
 books/NBK
 books/about
@@ -8726,7 +8671,6 @@ books?*zoom=5
 bookshelf
 booksrightsholders
 boutique/sitemaps
-bouyguestelecom
 bp
 bp/c
 bp/guest_mfg_brand
@@ -8801,11 +8745,9 @@ brands
 breakingnews/toihome.html
 breast-cancer/bc-treatment-21/provider-select
 briefEditorPublicationId=
-britishairways
 browse
 browse-2
 browse.aspx
-browse/genre
 browse/homes/all
 browse/more_page
 browse/reviews/recent
@@ -8817,7 +8759,6 @@ browse_content
 browser/tutorials
 browsers
 bruce
-brutus
 brw
 bs/android/collections
 bs/android/search
@@ -9620,7 +9561,7 @@ cdn-cgi
 cdn-cgi/
 cdn-cgi/bm/cv
 cdn-cgi/challenge-platform
-cdn-cgi/http://focus.de/robots.txt
+cdn-cgi/http://eff.org/robots.txt
 cdn-cgi/rum
 cdn-cgi/zaraz/s.js
 cdn-cgi/zaraz/t
@@ -9648,7 +9589,7 @@ cgi-bin/loginpage
 cgi-bin/search.fpl
 cgi-bin/state
 cgi-bin/wx
-cgi-binhttp://kck.st/robots.txt
+cgi-binhttp://youtube.de/robots.txt
 cgi-local
 cgi-perl
 cgi-registry
@@ -9714,7 +9655,6 @@ change-email
 change-password
 change-shipping
 changebrowser
-changeplan
 channels
 chappai
 chapter
@@ -9786,7 +9726,6 @@ checkpoint
 chemo-combo
 chiaki1007akie/videolist-*.html
 chiaki1007akie/videolist.html
-chinese
 chisla.html
 chistory.php
 chocolatechip
@@ -9844,7 +9783,6 @@ civis/posts
 civis/search
 civis/threads/*in_iframe=1
 civis/threads/*wp_data=
-ck/
 cl
 cl2/feeds
 cl2/ical
@@ -9870,7 +9808,6 @@ classnames/static-2.3
 classroom
 clck
 clean
-clearcookies
 cli
 click
 clickId=
@@ -9992,7 +9929,7 @@ co/users
 co/ventureone
 code
 code/*
-code/http://m.youtube.com/robots.txt
+code/http://bbc.com/robots.txt
 codeCoverage
 codesamples/
 coin*mid=
@@ -10038,7 +9975,6 @@ comeuntochrist/*/internal-use-only/
 comeuntochrist/*/search-results/
 comeuntochrist/lp/
 comeuntochrist/teach/
-comhem
 coming-soon*language=
 comm
 comm-part-test
@@ -10116,11 +10052,9 @@ community_comment
 community_post
 community_s
 compagnie
-companies
 companies/c
 companies/o
 companies/search
-company
 company/*.rss
 company/?s=
 company/add-ons-agreement
@@ -10206,7 +10140,7 @@ confirmation.html
 conflagexp
 connect
 connect/@
-connect/http://eventbrite.co.uk/robots.txt
+connect/http://dribbble.com/robots.txt
 connect/social-intranet.html
 connections
 connections/recent
@@ -10323,7 +10257,7 @@ content/dam/ircc/documents/pdf/francais/trousses/form/imm5476
 content/dam/ircc/documents/pdf/francais/trousses/form/imm5644
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0002
 content/dam/ircc/documents/pdf/francais/trousses/form/irm0004
-content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://feedproxy.google.com/robots.txt
+content/dam/ircc/documents/pdf/francais/trousses/form/irm0005/http://genius.com/robots.txt
 content/dam/ircc/documents/pdf/francais/trousses/guides/guide-0142-avion-afghanistan-voie-acces-rp.pdf
 content/dam/oecd/
 content/dam/redcross/lp
@@ -10479,7 +10413,6 @@ coupons.
 coupons/visit
 courier
 courses/
-courts
 cover
 covid19
 covid19/mobility
@@ -10499,9 +10432,7 @@ cpresources
 cpro
 cpropub/univercd
 cpspa
-cr$
 cr/
-cr?
 craft
 crash
 crawler4j
@@ -10580,7 +10511,6 @@ cse?
 cservice
 cservice/ya_index.gsp
 csi_204
-csl
 csmlists/addbookmark
 csmlists/topic
 csp
@@ -11010,7 +10940,6 @@ device
 device-ajax
 device-prefs
 deviceList
-deviceManagement
 devmailmobile/
 devmx/
 devopsconsole/images
@@ -11500,7 +11429,6 @@ editor.jsp
 editor2.jsp
 editorial-images
 editorial/*
-editpayment
 edu/cs4hs
 edu/ping
 edu/tasks
@@ -11551,7 +11479,6 @@ element/commun/afficher
 eli-search
 eliminar-busqueda
 eliminar-favoritos
-elisa
 elite?
 elp
 eltiempo/buscador
@@ -11565,7 +11492,7 @@ email-preferences/
 email-unsubscribe
 email-validation
 email/$
-email/*http://avexnet.or.jp/robots.txt
+email/*http://store.google.com/robots.txt
 email/EnrollmentUnsubscribe.htm
 email/SubscriptionTypeUnsubscribe.htm
 email/Unsubscribe.htm
@@ -11587,7 +11514,6 @@ emailpreferences.aspx
 emails
 emailservice
 emailsignup
-emailunsubscribe
 embed
 embed/
 embed/*
@@ -11785,18 +11711,13 @@ enterprise/blog/search$
 enterprise/blog/search?
 enterprise/orgs-terms
 enterprise/seo
-entertain
 entertainment/article/Minerva-s-horoscope
 entertainment/bravo
-entities/search
-entityexplore$
-entityexplore?
 entrez
 entrez/eutils
 entrez/sutils
 entrez/utils
 entrez/viewer.fcgi
-entrytrap
 enus/
 enviro/static-3.70
 enviro/static-3.72
@@ -12158,7 +12079,6 @@ eu/promos
 eu/search
 eu/user
 eu/users
-eula
 eur-lex
 eures/main.jsp
 europeaid/companion
@@ -12317,7 +12237,6 @@ fbml/ajax/dialog
 fbpostaction.cms?article=
 fcgi-
 fckeditor
-fd/
 fdbk
 feature/sf-test
 feature_nine_browse-bin=
@@ -12499,7 +12418,7 @@ files/techabuse-welsh.pdf
 files/techabuse.pdf
 fileview
 filing/free-file-fillable-forms/free-file-fillable-forms-support
-filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://sxsw.com/robots.txt
+filing/free-file-fillable-forms/free-file-fillable-forms-support-2http://newsweek.com/robots.txt
 filter
 filter/tips
 filter/tips
@@ -12528,7 +12447,6 @@ firefox
 firefox/?
 firefox/downloads
 firefox/new_tab_learn_more
-firetv
 firststeps.
 fisheries/fleet
 fix-it
@@ -12861,7 +12779,6 @@ free-video/*/square
 free-video/*/vertical
 free/cloud-computing-free-tier/
 free/databases-free-tier/
-freemonth
 freertos/archive
 french-studies/
 fresh
@@ -12882,8 +12799,6 @@ full
 fullscreen
 fullscreen/ReportLinker
 fun
-fun/api/
-fun/g/
 fundraiser/charity
 fundraiser/search
 fur/forums
@@ -12905,7 +12820,7 @@ fy-NL/search
 fy-NL/user
 fy-NL/users
 g
-g/4arE9Chttp://nngroup.com/robots.txt
+g/4arE9Chttp://healthaffairs.org/robots.txt
 g/api
 g00
 g01
@@ -13097,7 +13012,6 @@ geo/tools
 geocoder
 geoip.php
 geolocation
-geooverride
 get
 get$
 get-author-review
@@ -13463,7 +13377,7 @@ ha/search
 ha/user
 ha/users
 hackathons/reinvent-2013/
-hacker/*/followhttp://ikea.com/robots.txt
+hacker/*/followhttp://s3-eu-west-1.amazonaws.com/robots.txt
 hacker/*/unfollow
 hair-transplant-services
 handler
@@ -13576,10 +13490,9 @@ hidden-content
 highereducation/higheddocs
 highlights
 highwire
+hilfe/assistent.html
 hilfe/fundingdetail.html
-hilfe/live/launch-day/launch-day-show-voting.html
 hilfe/live/launch-day/launch-day-videoupload.html
-hilfe/live/officehour/officehour-link.html
 hims-hard-mints
 hinweis/datenleck
 hire/browse
@@ -13797,14 +13710,12 @@ hosted
 hosted/images
 hosting/search
 hotel/index.hi.
-hotel/reviews?
 hotel/us/the-airstream-van.*.html
 hotel_attractions
 hotel_rt_onview
 hotelfinder/rpc
 hotels
 hotels/rpc
-hotels/search
 hotelsonmap.*.json
 hotfix/
 houseads
@@ -13819,8 +13730,6 @@ howto/GoodNeighborPolicy.htm
 howtobuy/
 hp/focus-retrocompatibile$
 hp/video
-hpm
-hpmob
 hr
 hr-*/
 hr-hr/
@@ -13870,40 +13779,40 @@ http
 http-checker
 http2_data
 http:
-http://amazon.com.br/robots.txt
-http://amzn.to/robots.txt
-http://audible.com/robots.txt
-http://bizjournals.com/robots.txt
-http://blog.google/robots.txt
-http://bloomberg.com/robots.txt
-http://bluehost.com/robots.txt
+http://1.bp.blogspot.com/robots.txt
+http://1.gravatar.com/robots.txt
 http://boredpanda.com/robots.txt
 http://change.org/robots.txt
-http://durhamcollege.ca/robots.txt
-http://eji.org/robots.txt
-http://europol.europa.eu/robots.txt
-http://extension.purdue.edu/robots.txt
+http://cloudflare.com/robots.txt
+http://code.google.com/robots.txt
+http://csmonitor.com/robots.txt
+http://deepmind.com/robots.txt
+http://dx.doi.org/robots.txt
+http://fastcompany.com/robots.txt
 http://fda.gov/robots.txt
-http://feeds.feedburner.com/robots.txt
-http://help.twitter.com/robots.txt
-http://iso.org/robots.txt
-http://justgiving.com/robots.txt
-http://katyperry.com/robots.txt
-http://l.facebook.com/robots.txt
-http://onlinelibrary.wiley.com/robots.txt
-http://opensource.org/robots.txt
-http://penguinrandomhouse.com/robots.txt
-http://php.net/robots.txt
-http://phys.org/robots.txt
-http://playstation.com/robots.txt
-http://s3-eu-west-1.amazonaws.com/robots.txt
-http://salon.com/robots.txt
-http://samsung.com/robots.txt
+http://hackaday.io/robots.txt
+http://hootsuite.com/robots.txt
+http://imgur.com/robots.txt
+http://liveinternet.ru/robots.txt
+http://microformats.org/robots.txt
+http://msha.ke/robots.txt
+http://nature.com/robots.txt
+http://networkadvertising.org/robots.txt
+http://onedrive.live.com/robots.txt
+http://opera.com/robots.txt
+http://oprah.com/robots.txt
+http://orcd.co/robots.txt
+http://podcasts.google.com/robots.txt
+http://polygraph.cool/robots.txt
+http://sfgate.com/robots.txt
+http://slack.com/robots.txt
+http://support.office.com/robots.txt
 http://svt.se/robots.txt
-http://ted.com/robots.txt
-http://today.com/robots.txt
-http://washingtonpost.com/robots.txt
-http://web.facebook.com/robots.txt
+http://sxsw.com/robots.txt
+http://uk.linkedin.com/robots.txt
+http://usnews.com/robots.txt
+http://weibo.com/robots.txt
+http://zendesk.com/robots.txt
 hu
 hu-*/
 hu-hu/
@@ -14241,7 +14150,6 @@ image-*tool=
 image-server
 image/detailprint
 imageClipperUpload
-imagelibrary
 images
 images-apphost
 images-data
@@ -14251,16 +14159,11 @@ images/$
 images/*
 images/?
 images/Podcasts
-images/async/?
-images/async?
 images/carBlogsimages
 images/chart_gifs
 images/chart_pdfs
 images/chart_png300
 images/chart_png460
-images/hosted/?
-images/hosted/search?
-images/hosted?
 images/interview_pdfs
 images/li.gif
 images/logging_requests.gif
@@ -14269,13 +14172,8 @@ images/ndtvvideo/*
 images/pad/search?rpt=imageview
 images/pad/search?text=*&source=morda
 images/resume
-images/search%3F
-images/search%5C
-images/search/?
-images/search?
 images/search?rpt=imageview
 images/search?text=*&source=morda
-images/searchbyimage
 images/showroomimages
 images/smart/$
 images/touch/$
@@ -14286,9 +14184,6 @@ images/usedcarimages
 images/userprofileimg
 images/v$
 images/viewconfig$
-images?
-images?*mmbaseline*
-images?*nr*
 imagine/
 imesync
 imf-fetch-message
@@ -14415,7 +14310,6 @@ inapp
 inbox
 inbox$
 inbox$
-inboxcommerce
 inc5000dev
 include
 include/
@@ -14520,33 +14414,8 @@ info
 info-pages/advertising-guidelines
 info-pages/custom-services-agreement-23-10
 info-pages/terms-of-use/custom-services-agreement-23-10
-info/agb/anb-2018.html
 info/agb/api.html
 info/agb/starter-2018.html
-info/agb/teilnahmebedingungen.html
-info/jobs/Development-Operations-Engineer.html
-info/jobs/Kooperations-Manager.html
-info/jobs/Social-Media-Online-Kommunikation.html
-info/jobs/UX-UI-Designer.html
-info/jobs/ba-medieninformatik.html
-info/jobs/buchhaltung-payment.html
-info/jobs/buchhaltung-personal.html
-info/jobs/compliance.html
-info/jobs/crowd-agentin.html
-info/jobs/frontend-entwickler.html
-info/jobs/kommunikationsdesign.html
-info/jobs/kommunikationsstrategin.html
-info/jobs/multimedia-journalistin.html
-info/jobs/product-owner.html
-info/jobs/projektberater.html
-info/jobs/scrum-master.html
-info/jobs/showmaster.html
-info/jobs/site-reliability-engineer.html
-info/jobs/social-media.html
-info/jobs/support.html
-info/jobs/texterin-uxwriterin.html
-info/jobs/werkstudentin-demokratie-botschafterin.html
-info/jobs/werkstudentin-social-media.html
 info/tech-feedback
 info/verstoss-melden.html
 informa
@@ -14577,7 +14446,7 @@ integrate/
 integrations
 integromedb
 intelligencer/amp
-interactive*http://istockphoto.com/robots.txt
+interactive*http://click.linksynergy.com/robots.txt
 interactive/reading-preference
 interactives
 interface/pda
@@ -14794,7 +14663,7 @@ it/search*price_bucket=
 it/search*search_type=
 it/search*ship_to=
 it/search/autocomplete
-it/searchhttp://adssettings.google.com/robots.txt
+it/searchhttp://3.bp.blogspot.com/robots.txt
 it/share
 it/shop_name_search_service
 it/solutions/case-studies/
@@ -15315,8 +15184,6 @@ ko/user
 ko/users
 ko_kr/templates/M
 kolkata
-korean
-kpn
 kr*/blog-article-remove-faq-utms-*.js
 kr*/cdn-cgi/challenge-platform
 kr*/step
@@ -15415,7 +15282,7 @@ layout/set/txt_pq
 layout/set/xml
 layouts
 lb
-leadingtogether/http://cse.google.com/robots.txt
+leadingtogether/http://sfchronicle.com/robots.txt
 leaf
 learn
 learn-noperf
@@ -15449,8 +15316,6 @@ learning/settings
 learning/settings$
 learning/settings?
 learnmore
-learnmorelayer
-learnmorelayertv
 lecture
 lees/gids/exclusief
 lees/gids/exclusief$
@@ -15641,7 +15506,6 @@ loans
 loc/volunteers
 loc=
 local
-local
 local-services-ads
 local/ajax
 local/blogsandcolumns/slow-ride-front
@@ -15718,10 +15582,7 @@ login_or_join/redirected$
 login_or_join/redirected?
 login_or_join?
 login_with_redirect
-loginhelp
 loginhelp/
-loginhelp/lookup
-loginhelpsucess
 loginidentification/
 logo
 logo-maker/brief
@@ -15930,7 +15791,6 @@ maps/*?lang=uk$
 maps/*?lang=uz$
 maps/*?ll=
 maps/about
-maps/adsendpoint
 maps/api/js
 maps/api/place/js
 maps/api/staticmap
@@ -15993,13 +15853,11 @@ mats/?diagid=100001&eulaaccept=yes
 mats/default.aspx?diagid=100001&eulaaccept=yes
 max=
 max_price=
-maxis
 maxprice=
 mb/
 mbf
 mbox
 mbr/
-mcd
 me
 mecha
 media
@@ -16375,7 +16233,6 @@ memorandum
 mental-health-genetic-testing
 menu
 menu/*/feedback_form
-meo
 meps/*/eptvAjax
 meps/*/pdf*
 meps/*/xml*
@@ -16387,7 +16244,6 @@ merchant/directions
 merchant/imageUploadPreview
 merchant/loc
 merchant/moreProducts?
-merchant/reviews?
 merchant/update
 merchantapps
 mes
@@ -16527,7 +16383,6 @@ modern-formats-eng/ucp/static/*/client
 modern-formats-eng/ucp/static/*/csr
 modern-formats-eng/ucp/static/*/dev
 modern-formats-eng/ucp/static/framework_logging
-modernizr
 modules
 modules/*.css$
 modules/*.css?
@@ -17378,6 +17233,7 @@ nofollow=1
 nojs=
 non-searchable/
 nonConfig/api/wpa
+nonce="H1gh19ZbFgTlXeWy9Il0Aw">if(google.cis
 nonprofits/account
 nonstyleishida/videolist-*.html
 nonstyleishida/videolist.html
@@ -17388,16 +17244,12 @@ noscript
 not-allowed
 not-supported.html
 not_recommended_reviews
-notamember
-notfound
 notfound.html
-notices
 noticias/20080416/destituyen-director-jefe-proteccion-asco-tras-fuga-radiactiva/28170.shtml
 notificarelacionadas
 notifications
 notifications
 notifications$
-notifications/
 notifications/*
 nova
 now/mt3
@@ -17407,7 +17259,6 @@ npk
 npln
 npost/
 nqrwns
-nrdapp
 ns/
 ns/apology
 ns/hydra/console/?
@@ -17587,8 +17438,6 @@ offers/nonprofits
 offers/nonprofits$
 offers/nonprofits%
 offers/nonprofits?
-offers/proxy/dealsserver/api/log
-offers/proxy/dealsserver/buy
 offers/referral-terms-conditions
 offers/referral-terms-conditions$
 offers/referral-terms-conditions%
@@ -17641,7 +17490,6 @@ or/search
 or/user
 or/users
 oral-and-maxillofacial-surgeon
-orangefrance
 order
 order-detail
 order-results
@@ -17680,7 +17528,6 @@ over-de-vpro/bedankt.*$
 over-de-vpro/bedankt.html$
 overlay
 overview
-overviewblockseeother
 owner_settings
 p
 p.php
@@ -17808,7 +17655,6 @@ page/*/?egotecpage
 page/*/?hc_location=
 page/*/?iq_id=
 page/*/?pcd=
-page/*/?s
 page/*/?s=
 page/*/?search=
 page/account-confirmation-thanks
@@ -18052,7 +17898,6 @@ paid-posts
 palinsesto/guidatv/*/*.html$
 palinsesto/guidatv/lista/*/*.html$
 palinsesto/retrocompatibile/*/*.html
-panasonic
 panel
 panelweb
 panorama/welt/playboy_aid_119773.html
@@ -18238,7 +18083,6 @@ phbr1-
 phbr2-
 phbr~r_
 phone/compare/?
-phonenumber
 photo
 photo-editor
 photo-test-editor-x
@@ -18281,7 +18125,6 @@ pictures/related
 pictures/search
 pift/retest
 pigeon
-pin
 pin-builder
 pin-communities
 pin-creation-tool
@@ -18480,7 +18323,6 @@ player/metadata/video
 playerColor=
 playlist.php
 playlists/*/user
-playstation
 plenary/*/vod.html
 plonejsi18n$
 pls
@@ -18775,11 +18617,8 @@ polling
 polopolydevelopment
 pools/c
 pools/campaign
-popup/codewhatisthis
 popup?
 popup_security.jsp
-popupdetails
-popupprivacypolicy
 popups/mediaplayer
 popups/popup_gallery
 portal
@@ -18911,10 +18750,9 @@ privacy/consent
 privacy/request-data
 privacy/zoho-group.html
 privacy_policy
-privacypolicychanges
 private
 private/http://asana.com/robots.txt
-private/http://wunderground.com/robots.txt
+private/http://online.wsj.com/robots.txt
 private/wtrack
 prix-essence
 pro
@@ -18924,7 +18762,6 @@ pro-resources
 pro-site
 pro/about
 pro2
-proFile/history?
 proMatch
 proProjects
 proVideoWidget
@@ -18942,7 +18779,6 @@ product-reviews/B0069IY63Y
 product/electrode/api/logger
 product/electrode/api/wpa
 product/recommended
-product/reviews?
 productBrand=
 productId=
 productPurchaseDone
@@ -18994,7 +18830,6 @@ profile/*/following
 profile/*/map$
 profile/*/photos
 profile/downloads_limit
-profile/history?
 profile/license
 profile/payment/getstarted
 profile/view
@@ -19076,7 +18911,6 @@ protected
 protocol
 protocolexchange/labgroups
 proximic
-proximus
 proxy
 proxy
 prp
@@ -19346,7 +19180,6 @@ q/user/*/subscribers
 q/user/*/verifies
 q=
 qa
-qantas
 qemail
 qi
 qnasearch?
@@ -19550,7 +19383,6 @@ relations.php
 releasenotes/item/
 relevance_feedback
 relnotes
-rememberme
 remote-login.php
 remote.control
 remotecontrol/bestpractices/jl-prototype
@@ -19642,7 +19474,6 @@ resources/xml/sitemaps
 responder
 rest
 results
-results
 results-export-email-by-search-data
 results-export-ids
 results-export-search-by-year
@@ -19699,7 +19530,6 @@ reviews/write/?s=
 reviews/write?s=
 reviews_page=
 rewards
-rewardsapp/
 rf=*
 ribbon.php
 rip/
@@ -19910,7 +19740,6 @@ salesImages/static-1.382
 salesImages/static-1.383
 salesImages/static-1.389
 samemodel.php?imageid=*&pg=*&sortcriteria=6
-samsung
 sap/saphana/pilot/
 sap/saphana/pilot/offer/
 sap/saphana/pilot/offer/confirmation
@@ -20041,9 +19870,6 @@ sdfe/pdf/download/read/am4ir/noindex/pii
 sdfe/pdf/download/read/index/pii
 sdfe/pdf/download/read/noindex/pii
 sdfe/reader/pii
-sdg-newsletter.html
-sdg-newsletter/sdg-newsletter-abbestellen.html
-sdg-newsletter/sdg-newsletter-bestaetigen.html
 sdk
 sdk/js
 sdkterms
@@ -20354,7 +20180,7 @@ section/3
 section/36
 section/6
 section/8
-section/9http://line.me/robots.txt
+section/9http://livestream.com/robots.txt
 sectionized
 sections
 sections$
@@ -20524,7 +20350,6 @@ set_cookie.pl
 setbucket
 setprefs
 settings
-settings
 settings/*
 settings/deactivated
 settings/linkaccount
@@ -20662,7 +20487,6 @@ sharing
 sharing/files
 sharing/folders
 shelf/user_shelves
-shenghuo
 shenghuo/search?
 shifen/
 shihui
@@ -20673,15 +20497,11 @@ shipping-payment
 shippingassistant
 shippinginformation
 shop
-shop$
-shop/
 shop/*&
 shop/browse/overlay
-shop/collections
 shop/global
 shop/iphone/payments/overlay
 shop/search.ep
-shop?
 shop_name_search_service
 shop_pins
 shopfront-api
@@ -20736,7 +20556,7 @@ showdependency
 shows/
 showtag
 showtime/.requestinvite.html.swp
-showtimes/location/*http://slideshare.net/robots.txt
+showtimes/location/*http://arxiv.org/robots.txt
 shr*
 shutterstock
 si
@@ -20760,7 +20580,6 @@ sign-in
 sign-in-6for6Sale
 sign-out
 sign-out
-sign-up
 sign-up-to-be-contacted-for-research
 sign-up/?
 sign_in
@@ -20799,7 +20618,6 @@ signup/payment%
 signup/payment?
 signup?
 signup_modal
-signurl
 simple.html$
 simplepost
 simpleshare
@@ -21224,7 +21042,6 @@ social
 social-connect/send
 social/photosui/images
 social/photosui/sitemap
-social/search?
 social/social-security-directory/cai
 social_media
 socialapp/
@@ -21278,7 +21095,6 @@ songs/*/comments
 songs/*/metadata
 songs/*/show_amp_header_play_toggle
 songs/for_profile_page
-sony
 sorry
 sort=
 sortBy=
@@ -21304,7 +21120,6 @@ sp=
 sp_url=
 spade-widget
 spartan
-spbasic
 spbot
 spc-source=
 special-promo/2for1
@@ -21383,7 +21198,6 @@ sports/whatson
 sports_fp_ms
 spp/
 spreadsheet
-spresults
 sprint_wml
 sprint_xhtml
 sq/android/collections
@@ -21469,7 +21283,6 @@ statements
 static
 static-assets/documents/PwnedPasswordsTop100k.txt
 static-assets/images/collection
-static/
 static/*.css$
 static/*.css?
 static/*.gif
@@ -21601,8 +21414,6 @@ storeclick
 storefront
 storepicker
 storerequest
-stores$
-stores/
 stores/page/*visitId=
 stories
 story
@@ -21689,7 +21500,6 @@ subscribe_success
 subscription
 subscription/*?*onboarded=
 subscription/*?*source=
-subscriptioncancel
 subscriptions
 subscriptions/manage/
 subscriptions/securejson/
@@ -21918,7 +21728,7 @@ tag
 tag-manager
 tag-manager-360
 tag/
-tag/*+http://drupal.org/robots.txt
+tag/*+http://igg.me/robots.txt
 tag/*/a
 tag/*/a/
 tag/*/b
@@ -22000,13 +21810,11 @@ talk/new_topic
 talk/user
 talk/user/*/article
 talks2
-talktalk
 tally-loader-react/static-1.65
 target_baby
 target_group
 targetdirect_group
 tasks
-tastesurvey
 tax-center
 tax.php
 tax_countries.php
@@ -22040,7 +21848,6 @@ tbproxy
 tc/accept
 tdaroundme
 tdautourdemoi
-tdc
 tdid=*
 tdir/p/kids-back-to-school
 tdv2_fp
@@ -22074,7 +21881,6 @@ technology/archive/
 technology/tagged/*page
 telegraph
 telehealth-for-women
-telenor
 television/entretenimiento
 television/teletexto/incibe
 television/teletexto/noticias
@@ -22085,7 +21891,6 @@ television/tve-internacional/asia/?
 television/tve-internacional/europa/?
 television/tve-internacional/parrilla/?
 television/tve-internacional/programacion-tv/?
-telfort
 telsearch
 temp
 temp/
@@ -22136,7 +21941,6 @@ terms/looker/legal/sccs
 terms/looker/legal/suppliers/partner-assignment-faq
 terms/looker/legal/suppliers/supplier-code-of-conduct
 terms_of_use
-termsofusechanges
 test
 test-2
 test-2$
@@ -22153,9 +21957,9 @@ test5
 test6
 testcases
 testfl
-testhttp://deepmind.com/robots.txt
+testhttp://europa.eu/robots.txt
 testing
-testing/http://accenture.com/robots.txt
+testing/http://justanotherdeveloper.net/robots.txt
 testpage-forhomepage
 tests/*
 tetro
@@ -22163,7 +21967,6 @@ text
 text_edition
 tfemail
 th
-th$
 th*/blog-article-remove-faq-utms-*.js
 th*/cdn-cgi/challenge-platform
 th*/step
@@ -22226,7 +22029,6 @@ th/tools/logo-maker/onboarding
 th/user
 th/users
 th/ventureone
-th?
 th_th/templates/M
 thank-you
 thank-you-enterprise-demo
@@ -22280,7 +22082,6 @@ ti/p
 tickets
 tickets-external?
 tiempo/*p=
-tim
 time/change/*?
 time/change/@
 time/map/*_escaped_fragment_
@@ -22517,13 +22318,12 @@ translate_static
 translate_suggestion
 translate_t
 translation
-translator/?ref=
 transmute/static-1.85
 transmute/static-2.6
 transmute/static-2.8
 transmute/static-2.9
 transparency
-transparency/http://redcross.org/robots.txt
+transparency/http://classroom.google.com/robots.txt
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14885
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14895
 transparency/regexpert/index.cfm?do=groupDetail.groupDetailDoc&id=14896
@@ -22545,32 +22345,15 @@ travel-vacations
 travel/
 travel/2013/aug/22/been-there-readers-competition?
 travel/8711559/The-Telegraph-Travel-Awards-2011.html
-travel/api
 travel/clk
-travel/css
 travel/entity
-travel/flight-search
-travel/flight/flightSearch
-travel/flight/flightSearchAction
-travel/flight/search/?
-travel/flight/search?
-travel/flights
 travel/flights/s
-travel/hotel-search
 travel/hotel/e
-travel/hotel/hotelMiniSearchRequest
-travel/hotel/hotelSearch
-travel/hotels/checkout
-travel/hotels/detail
 travel/hotels/entity
 travel/hotels/hotel-finder
 travel/hotels/hotel-finder/$
-travel/hotels/search/?
-travel/hotels/search?
 travel/hotels/stories
-travel/scripts
 travel/search
-travel/secure
 travelnevadahighway95tc
 treasury
 treasury/treasdocuments
@@ -22626,7 +22409,6 @@ trylater
 tsa
 tto
 tucritica
-tudum
 tune*retpath=
 turbo
 turbo*?*ajax=1
@@ -22687,7 +22469,6 @@ tve/b/zonadisney
 tve/programo
 tve/teletexto
 tvschedule
-tvsignup
 tw
 tw*/blog-article-remove-faq-utms-*.js
 tw*/cdn-cgi/challenge-platform
@@ -23000,7 +22781,6 @@ unternehmen/o
 unternehmen/suche
 unterstuetzer.html
 until=
-upcomingevents
 update-your-browser
 update.php
 update_
@@ -23041,7 +22821,6 @@ ur/search
 ur/user
 ur/users
 urchin_test
-url
 url-shortner
 urlinator/static-1.54
 urlinator/static-1.55
@@ -23458,7 +23237,6 @@ verify
 verify_age
 verify_your_email$
 verify_your_email?
-verifyidentity
 version
 versions
 vet/addl/covid19partners
@@ -23572,31 +23350,13 @@ videodash.cms*
 videodetails
 videorequest
 videos
-videos/?
 videos/assets
-videos/browsing
 videos/e
-videos/explore
-videos/favorites
-videos/feed
-videos/feedcontent
 videos/get_video_by_uid/
 videos/images
 videos/js
-videos/music
-videos/music/
-videos/music/?
-videos/music?
-videos/riverview/relatedvideo
-videos/riverview/relatedvideo/?
-videos/riverview/relatedvideo?
 videos/search.php
-videos/search/?
-videos/search?
-videos/trending
-videos/trendingcontent
 videos/viewmeta/
-videos?
 videosearch
 view
 view-3d-home
@@ -23609,7 +23369,6 @@ view_item.pl
 view_message.pl
 viewconfig$
 viewer
-viewingactivity
 viewlocale=zh_CN
 vieworder
 viewrecentepisode
@@ -23618,7 +23377,6 @@ viitem
 vilos
 vinterdev/
 vip
-virginaustralia
 virtual
 virtualization/
 virtualsupportassistant
@@ -23667,8 +23425,6 @@ vn/tools/business-name-generator/searchutf8
 vn/tools/logo-maker/onboarding
 vn/ventureone
 vod
-vodafone
-vodafonedemobilelaunch
 voices/
 volunteer/
 vote
@@ -23849,14 +23605,9 @@ whichmba/forum
 whichmba/webinars?
 whichmba/webinars?page=
 whsmiths
-whysecure
 whythisad
 widget
 widget-landing-page.html
-widget/cr
-widget/entity/search/?
-widget/render
-widget/snapshot
 widget_auth.php
 widget_chunks
 widgets
@@ -24645,7 +24396,6 @@ word/embed
 wordpress/wp-admin
 wordpress/wp-admin/admin-ajax.php
 work
-work
 work/_
 work/acceptinvitation
 work/account
@@ -24743,9 +24493,7 @@ wp-admin
 wp-admin/
 wp-admin/admin-ajax.php
 wp-admin/admin-ajax.php
-wp-admin/admin-ajax.phphttp://mixer.com/robots.txt
-wp-admin/admin-ajax.phphttp://roh.org.uk/robots.txt
-wp-admin/admin-ajax.phphttp://umc.org/robots.txt
+wp-admin/admin-ajax.phphttp://androidauthority.com/robots.txt
 wp-admin/load-scripts.php
 wp-admin/load-styles.php
 wp-adv
@@ -24878,14 +24626,12 @@ x
 x/oauth
 x?
 xb/en/
-xboxone
 xbp/search
 xc/
 xclick-auction
 xcs0MhRsfxHG.js
 xd/en/
 xdoor
-xfinity
 xfx7
 xh/forums
 xh/kb/*/discuss
@@ -24916,7 +24662,6 @@ xmlsearch
 xoonboarding
 xoplatform/logger/api/logger
 xorouter
-xs4all
 xsl/
 y1=
 yaca
@@ -24950,10 +24695,8 @@ your
 your-account
 your-orders
 your-shop
-yourAccount
-youraccount
 youreurope/businessACC
-youreurope/businessDEV/http://use.typekit.net/robots.txt
+youreurope/businessDEV/http://stripe.com/robots.txt
 youreurope/citizensACC
 youreurope/citizensDEV
 yours
@@ -25160,7 +24903,6 @@ zh/ventureone
 zh_CN
 zh_cn
 zh_tw/templates/M
-ziggo
 zite-importer
 zoeken$
 zoeken.html$
diff --git a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt
index 0d6b71b29..444680fb4 100644
--- a/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt
+++ b/Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-10000.txt
@@ -1123,7 +1123,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 .json
 .json
 .json$
-.json$
 .jsp
 .jsp
 .jsp$
@@ -3459,7 +3458,6 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /2018-
 /2023-
 /22
-/22732481
 /23
 /25
 /27
@@ -3545,6 +3543,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /?cid=
 /?clientId=
 /?collection=
+/?comment_page=
 /?consterm
 /?costPlans=
 /?county
@@ -4584,6 +4583,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /checkin
 /checklist$
 /checkout
+/checkout
 /checkout/
 /checkout/*
 /child
@@ -6013,6 +6013,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /login*/search?
 /login-redirect
 /login.cms*
+/login/
 /login/*
 /login/verify-email
 /login?
@@ -6202,6 +6203,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /muslim-sexy-videos.html
 /mvopinions/*
 /my
+/my-account
 /my-account/profile
 /my-account/register/newsletters-setup
 /my-account/register/profile-setup
@@ -6776,6 +6778,7 @@ $rpc/google.internal.maps.mapsjs.v1.MapsJsInternalService
 /reg/regmsg.xhtml
 /register
 /register$
+/register/
 /register/*
 /register/?
 /registration
@@ -9593,7 +9596,6 @@ AG/
 AI
 AI/
 AITripBuilder
-AJAXhttp://allocine.fr/robots.txt
 AK5537
 AK6336com
 AL/
@@ -9685,12 +9687,10 @@ Account/*
 Account/Register
 Account/Register.aspx
 Account/register.aspx
-AccountAccess
 AccountActivityView
 AccountBalanceView
 AccountDisavow?
 AccountMerge
-AccountStatus
 Account_Billing
 AccurintForCollectionsLP
 Achievements
@@ -9733,6 +9733,7 @@ Ajax.htm$
 Ajax/
 AjaxLogonForm
 AjaxOrderItemDisplayView
+AjaxPages
 AjaxSearchNavigationView
 Alamysearchhistory/
 Album.aspx/*
@@ -9804,7 +9805,6 @@ AppleMapsAction
 Applications
 Applications/*
 Apps/
-Arabic
 Architext
 Archive/New_Records_Testing
 Archive/PrintMaterial/ACS_Journal/1/files
@@ -10731,7 +10731,6 @@ Bender
 Benoit/Public/DVDDB
 BetterBySea
 Biker
-BillingActivity
 BillingAndPayments
 BillingDetail
 BillingInvoiceFile
@@ -10813,7 +10812,6 @@ CampaignConfiguration
 CampaignId
 CampaignManagement
 CancelBooking
-CancelPlan
 CaptchaValidationLog
 Cargo-Webdesign-Directory
 Carrello
@@ -10834,7 +10832,6 @@ Celebrate100Million
 CertificateOfExcellenceRequestSupport
 ChangeEmailAddressView?
 ChangePassword
-ChangePlan
 ChangeSecurityInfoView
 ChatGPT
 ChatGPT-User
@@ -10855,7 +10852,6 @@ CheckoutOrderBillingView
 CheckoutOrderShippingView
 CheckoutPayInStoreView
 CheckoutSignInView
-Chinese
 Christian/Books/WISHLIST
 Christian/Books/backorder_email
 Christian/Books/bookbag
@@ -11043,7 +11039,6 @@ DisplayCmd
 DisplayMap
 DisplayView
 Divisions
-DoNotTest
 Doc
 Doc-rm
 Doc-rss
@@ -11095,7 +11090,6 @@ ET/
 EULegal
 EarlyRequesterJS/static-1.38
 EdadsV2
-EditProfiles
 Editorial/
 Education/SchoolHelp.htm
 Education/SchoolStudents/Olympiad/paper2011.asp
@@ -11107,7 +11101,6 @@ EmailCartView
 EmailHotel
 EmailMap
 EmailMemberRegistration
-EmailPreferences
 EmailSpellCheckJson
 Emailer
 Emailv_v2
@@ -11385,7 +11378,6 @@ FinancialServices
 Find*
 FindRestaurants
 FinishAppInstall
-FireTV
 FiveSteps/
 FiveStepsToHealth/
 Flamingo_SearchEngine
@@ -11522,7 +11514,7 @@ GuideRequested
 GuideRequestor
 H
 HACAffiliate
-HBO/
+HBO/
 HK/
 HMDThankYou
 HN/
@@ -11541,7 +11533,6 @@ HX62
 HYx10rg3
 HandleHotelSuperPortal
 Handlers
-HdToggle
 HeadJS/static-2.115
 HeadJS/static-2.83
 HeadJS/static-2.84
@@ -11579,7 +11570,6 @@ HotelsManagementCenter
 HotelsProductCart
 HowTheSiteWorks
 How_To_Get_Sky
-HpImageArchive.aspx
 HubStyle/static-2.309
 HubStyle/static-2.313
 HubStyle/static-2.315
@@ -11776,7 +11766,6 @@ KeyFeatures/
 Kimengi
 Kimengi/nineconnections.com
 Knowledge-Search
-Korean
 Krypto
 LA/
 LB/
@@ -11805,7 +11794,6 @@ Labs/
 LanderLogger
 LandingPage
 Landingpages.html
-LanguagePreferences
 Languages/Pages/Volunteer-Stipend.aspx
 Languages/SiteCollectionDocuments/FAQ-County-Translations.pdf
 Large-Files/
@@ -11862,12 +11850,10 @@ Login-OAuthReentryFacebook*
 Login.html
 Login/
 Login?
-LoginHelp
 Logoff
 LogonForm
 LogonForm
 LogonFormView
-Logout
 Logs
 MA/
 MAINTAINERS.txt
@@ -11940,13 +11926,11 @@ MailHelpR1
 MakePaymentView
 Manage/
 ManageAccountAccess
-ManageDevices
 ManageListing
 ManageOrder
 ManageOwnersTopReasonsApi
 ManagePaymentAccountView
 ManagePhotos
-ManageProfiles
 ManageReturns
 ManageShopListingApi
 ManageShopping
@@ -11967,7 +11951,6 @@ Marketing/EcapSurvey.aspx
 Marketplace/
 MasterPages
 MathJax
-Maxis
 Maxthon
 McKinsey
 MedCenter
@@ -12055,7 +12038,6 @@ MemberTravelMap
 Membership/Memberzone/
 Membership/e-Membership/app-help.asp
 Membersites
-Meo
 Merchant2
 MessageRunDetailID=
 Messages
@@ -12217,7 +12199,6 @@ Moreover
 Moreoverbot
 Moved
 Movies
-MoviesYouveSeen
 MptUrl
 Ms.
 Msgs/
@@ -12240,6 +12221,7 @@ MyBLB.cfm
 MyBookingsDetail
 MyBookingsVoucher
 MyC-SPAN
+MyChristies/
 MyInbox
 MyPaintColorListDetailView
 MyPortfolio
@@ -12290,7 +12272,6 @@ NetSeer
 NewMessage
 NewOwners
 NewTopic
-NewWatchInstantlyRSS
 News
 News/Flash.aspx/
 News/Index
@@ -12458,7 +12439,6 @@ Pages/Search.aspx
 Pages/myetocsunsubscribe.aspx
 Pages/subprocessors.aspx
 PageviewRecord
-Panasonic
 Panier
 PanoramaXml
 Participate
@@ -12512,7 +12492,6 @@ PoiReviewDetail
 PoiTipDashboard
 Politics/qGhqEUxkwY1XCf44StpdPM/Asaram-Bapu-From-building-an-empire-of-Rs-10000-crore-to-b.html
 Politics/qGhqEUxkwY1XCf44StpdPM/Asaram-Bapu-From-building-an-empire-of-Rs-10000-crore-to-b.html?facet=amp&utm_source=googleamp&utm_medium=referral&utm_campaign=googleamp
-PopupDetails
 Portal/
 PortalIdParser/static-1.27
 PortalIdParser/static-1.28
@@ -12621,7 +12600,6 @@ Products/tv/androidtv
 Products/tv/seeingisbelieving
 Profile
 Profile/ViewPdf
-ProfilesGate
 Programmes/Search-Courses
 Programs
 Project
@@ -12636,7 +12614,6 @@ PropertyLayer/
 Provider/Image/
 Provider/PhysicianRatings/
 Providers/
-Proxy.ashx
 ProxyToolsAccountSettingsTool
 ProxyToolsBillingAndPaymentsTool
 ProxyToolsBillingDetailTool
@@ -12806,7 +12783,6 @@ ReceiveAutoRedirect
 Recent
 Recommend
 Recovery_Packages
-Redir.aspx
 Redirect
 Redirect=*
 Redirects
@@ -12837,7 +12813,6 @@ Reports/
 ReputationManager
 Request
 Research/Search-Research
-ResearchLibrary/
 ReservationAjax
 ResetPassword
 Resources
@@ -13065,7 +13040,6 @@ Solr
 Soltango/de-DE/
 Soltango/en-GB/
 SolutionsDisplay
-Sony
 SortAttractions
 SortEateries
 SortHotels
@@ -13507,7 +13481,6 @@ WebResource.axd
 WebResource.axd?*
 WebService
 WebService/
-WebServices/
 WebStop/
 WebStripper
 WebUserControls
@@ -13520,7 +13493,6 @@ WelcomeView
 WhatsNearby
 Wholesale-*-c-*-0-1-*.html
 Wholesale-Attribute-*.html
-WiViewingActivity
 WidgetBrowserExtension
 WidgetEmbed
 WidgetListAsync.naver
@@ -13559,8 +13531,6 @@ YU/
 Yandex
 Yasni
 YfcuVaVBkfnzl2j6UbEcK0Oiqkv88jx6w
-YourAccount
-YourAccountPayment
 Youtube
 ZA/
 ZCestino
@@ -14216,6 +14186,7 @@ about-aws/whats-new/2009/01/05/Amazon-CTO-Chief-of-Year/
 about-aws/whats-new/2009/01/09/Amazon-CTO-Chief-of-Year-2/
 about-aws/whats-new/2009/07/24/aws-survey/
 about-ghostery
+about-iqair
 about-the-council/about-the-website/third-party-accessibility-statements
 about-the-council/about-the-website/third-party-accessibility-statements/*http://kernel.org/robots.txt
 about-the-council/contact-us-new
@@ -14411,8 +14382,6 @@ academia/books/search.html
 academic
 academic-subscriptions
 academic/
-academic/profile
-academic/search
 academicaffairs/commencement/book
 academics/archive
 academics/college/special-programs/public-service-old
@@ -14563,7 +14532,6 @@ account_settings/set_password?
 account_settings/set_your_password$
 account_settings/set_your_password?
 account_settings?
-accountaccess
 accounthelp
 accountinfo.php
 accounts
@@ -14595,7 +14563,6 @@ accounts/new
 accounts/o8
 accounts/o8/id
 accounts/unlock
-accountstatus
 acct
 acctmgmt/*
 acd
@@ -14617,7 +14584,6 @@ acis/whatsnew.html
 acjohnston
 acl
 aclayton
-aclick
 aclk
 aclp-vle-decom
 acmendez
@@ -14806,7 +14772,7 @@ actualite/v2/tj22h/guide-horaire.asp
 actuator
 ad
 ad-blocker-performance-study-in-response-to-manifest-v3-finds-that-ghosterys-ad-blocker-beats-the-competition
-ad-manager/
+ad-manager/
 ad-retargeting-101-online-shopping-in-the-covid-19-era
 ad-scans
 ad-test
@@ -14914,6 +14880,7 @@ adelrio
 adev
 adfox
 adframe
+adfs/
 adg
 adg321
 adgranillo
@@ -14944,7 +14911,7 @@ admin-user
 admin.artnet.*
 admin.php
 admin/
-admin/
+admin/
 admin/*
 admin/aso
 admin/blogmove
@@ -14964,7 +14931,6 @@ adminapps1/
 adminapps2/
 adminapps3/
 adminapps4/
-adminhttp://aa.com/robots.txt
 adminhttp://dji.com/robots.txt
 administration
 administration/
@@ -15271,7 +15237,7 @@ agile064
 agile065
 agile066
 agile067
-agility/
+agility/
 aglassie
 aglynn
 agold
@@ -15316,10 +15282,11 @@ aimarketing
 aipro
 aiproduct
 aiprojecthks
+air-purifiers
+air-quality-monitors
 aircraft-icons/sprite
 airport/
 airport_transfer_deeplink
-airtel
 aisharma
 aislingibson
 aj_windows/travelagencies
@@ -15331,6 +15298,7 @@ ajax
 ajax*
 ajax*.do$
 ajax.htm$
+ajax.htm$
 ajax.php
 ajax/
 ajax/*
@@ -15464,12 +15432,6 @@ albie
 albright
 album
 album.php
-album/delete_user_review
-album/fetch_overview_review
-album/fetch_review_view
-album/report_inappropriate
-album/review_thumbs
-album/save_user_review
 albumarchive
 albums
 albums-cd-us
@@ -15537,7 +15499,6 @@ aliciademaio
 aliciafroio
 alikhani
 alilly
-alink
 aliraja
 alireza.haghighi
 alisamanning
@@ -15711,7 +15672,6 @@ amp-iframe
 amp-proxy
 amp-social-login
 amp-tealium
-amp/
 amp/*/more.html
 amp/*/page-*/$
 amp/amp/*
@@ -15819,7 +15779,6 @@ analyzer
 analyzer-api
 anamikasaha
 anamoss
-anan
 anapi
 anass-mourjane
 anastaciakay
@@ -15891,7 +15850,7 @@ ane103-17
 anesth/employment/faculty/cv
 anesthesia/alumni-thank-you
 anesthesia/pain-management-center/appointments-and-referrals/referral-thankyou
-anga/
+anga/
 angebote
 angel
 angel-cabrera
@@ -16065,6 +16024,7 @@ api.php?
 api.php?*&action=
 api.php?action=
 api/
+api/$
 api/*
 api/*/doc
 api/*/docs$
@@ -16151,8 +16111,8 @@ api/jmuse?id=*&index=*&type=mp3&v2=
 api/job-application
 api/jobPostings/jobs
 api/latest-articles
+api/legacy/pse
 api/legendary-data/relay
-api/maps/
 api/meerkat/page
 api/messages
 api/metainfo/tv
@@ -16163,7 +16123,7 @@ api/nextsteps
 api/ng-looks/
 api/nicorepo/timeline
 api/nts/v1/login
-api/og/*http://etsy.com/robots.txt
+api/og/*http://crowdfundingusa.com/robots.txt
 api/paid-article
 api/partner_select/*/price_query
 api/play/options
@@ -16171,6 +16131,7 @@ api/posts/*/popular_posts
 api/pricePrediction/V1/flight
 api/profile
 api/programas/69730/audios.rss
+api/pse
 api/reactions
 api/recaptcha/enabled
 api/records/*/files
@@ -16285,6 +16246,7 @@ apiproxy
 apis
 apis/maps/v2/locator/search/query.json
 apis/maps/v2/locator/search/viewport.json
+apis/mycars
 apiv2
 apiv2/*
 apk
@@ -16409,7 +16371,7 @@ appliance
 applicant
 applicant-eligibility
 application
-application-content/
+application-content/
 application-survey/
 application-temp
 applications/farel12
@@ -17095,14 +17057,13 @@ asonawane
 asouri
 asp.net/
 asp/
-asp/
+asp/
 asp/*
 asp/*http://arte.tv/robots.txt
 asp/demo_db_edit.asp
 aspnet_client
 aspnet_client/
-aspnet_client/
-aspx$
+aspnet_client/
 aspx/
 aspx/read.aspxhttp://tnt.netmng.com/robots.txt
 aspxerrorpath*
@@ -17770,7 +17731,6 @@ audreytiew
 audreywoz
 augustin_guyonnet
 augustyh
-aui/inbound
 aura
 aura/promo
 auroregonzalez
@@ -17874,7 +17834,6 @@ autocredit/sber_landing
 autodiscover/
 autokolcsonzes/eredmenyek
 autolist
-autologin
 automobile/recherche.php
 autonoleggio/risultati
 autonvuokraus/tulosta
@@ -18465,6 +18424,7 @@ bagley
 bagon
 baharhaghighat
 bahl
+baidu
 baileymerlin
 baileysincox
 bait
@@ -18531,7 +18491,7 @@ bannerinclude_US
 bannerinclude_fr
 banners
 banners-old.php?
-banners/
+banners/
 bannus
 baobao*?
 baohongchen
@@ -19083,7 +19043,6 @@ bfgs
 bfigueroa
 bfl
 bfmp
-bfp/search
 bfrank
 bg
 bg-*/
@@ -19294,7 +19253,6 @@ bin/sleep.cgi
 bin/ti
 binary-data/DIY
 binary-data/EGALLEY
-bing-site-safety
 bingbot
 bingsiteauth.xml
 bingwang
@@ -19752,7 +19710,6 @@ blogs/practitioners-of-the-month
 blogs/product
 blogs/research/search
 blogs/search
-blogs/search/
 blogs/slow-ride
 blogs/tag
 blogs/technology.atom
@@ -20269,7 +20226,6 @@ boutique/sitemaps
 boutiques
 boutiques/la-rentree-des-sportifs
 boutiques/la-rentree-des-sportifs$
-bouyguestelecom
 bova/
 bowles
 box-week
@@ -20521,7 +20477,6 @@ brinda
 brink-roby
 brinton
 brisasarai
-britishairways
 broadband
 broadway-revere/appointments-and-referrals/appointment-thankyou
 brodyzt
@@ -20558,7 +20513,6 @@ browse/devicesetup
 browse/domain
 browse/download
 browse/feed
-browse/genre
 browse/homes/all
 browse/issue/*/follow
 browse/laboratory
@@ -20591,7 +20545,6 @@ brt
 bruce
 brucker
 bruckerjoseph
-brutus
 brw
 bryan
 bryanhsu
@@ -20735,6 +20688,58 @@ business
 business
 business*?
 business-breakfast
+business-customers/aviation/news-and-media-releases/news-and-media-2020/shell-and-red-rock-sign-agreement-to-market-and-distribute-low-carbon-sustainable-aviation-and-diesel-fuel
+business-customers/aviation/news-and-media-releases/news-and-media-2020/shell-and-red-rock-sign-agreement-to-market-and-distribute-low-carbon-sustainable-aviation-and-diesel-fuel.html
+business-customers/aviation/shell-for-suppliers/card-services
+business-customers/aviation/shell-for-suppliers/card-services.html
+business-customers/bitumen/contact-us
+business-customers/bitumen/contact-us.html
+business-customers/chemicals/our-products/contact-us-about-shells-higher-olefins-and-derivatives
+business-customers/chemicals/our-products/contact-us-about-shells-higher-olefins-and-derivatives.html
+business-customers/chemicals/our-products/shell-performance-olefins/contact-us-about-shells-higher-olefins-and-derivatives
+business-customers/chemicals/our-products/shell-performance-olefins/contact-us-about-shells-higher-olefins-and-derivatives.html
+business-customers/chemicals/our-products/shell-performance-surfactants/contact-us-about-shells-higher-olefins-and-derivatives
+business-customers/chemicals/our-products/shell-performance-surfactants/contact-us-about-shells-higher-olefins-and-derivatives.html
+business-customers/chemicals/our-products/styrene-monomer/what-is-the-impact-of-styrene-based-products
+business-customers/chemicals/our-products/styrene-monomer/what-is-the-impact-of-styrene-based-products.html
+business-customers/lubricants-for-business/products/shell-panolin/five-things-to-know
+business-customers/lubricants-for-business/products/shell-panolin/five-things-to-know.html
+business-customers/lubricants-for-business/products/shell-panolin/lubricants-for-business-and-environment
+business-customers/lubricants-for-business/products/shell-panolin/lubricants-for-business-and-environment.html
+business-customers/lubricants-for-business/products/shell-panolin/protect-ecosystems-and-equipment
+business-customers/lubricants-for-business/products/shell-panolin/protect-ecosystems-and-equipment.html
+business-customers/lubricants-for-business/products/shell-panolin/reducing-carbon-intensity
+business-customers/lubricants-for-business/products/shell-panolin/reducing-carbon-intensity.html
+business-customers/lubricants-for-business/products/shell-panolin/supporting-sustainability-ambitions
+business-customers/lubricants-for-business/products/shell-panolin/supporting-sustainability-ambitions.html
+business-customers/lubricants-for-business/reach-for-shell-lubricants-supply-chain
+business-customers/lubricants-for-business/reach-for-shell-lubricants-supply-chain.html
+business-customers/lubricants-for-business/sector-expertise/mining/discover-mining-sector-overcome-challenges
+business-customers/lubricants-for-business/sector-expertise/mining/discover-mining-sector-overcome-challenges.html
+business-customers/lubricants-for-business/sector-expertise/power-industry/people-behind-power
+business-customers/lubricants-for-business/sector-expertise/power-industry/people-behind-power.html
+business-customers/lubricants-for-business/shell-expertise/lubrication-services-for-your-industry/shell-remotesense/engine
+business-customers/lubricants-for-business/shell-expertise/lubrication-services-for-your-industry/shell-remotesense/engine.html
+business-customers/marine/marine-lng
+business-customers/marine/marine-lng.html
+business-customers/shell-fleet-solutions/commercial-road-transport
+business-customers/shell-fleet-solutions/commercial-road-transport.html
+business-customers/shell-fleet-solutions/digital-and-payment-solutions/shell-fleet-app-for-small-businesses/feature
+business-customers/shell-fleet-solutions/digital-and-payment-solutions/shell-fleet-app-for-small-businesses/feature.html
+business-customers/shell-market-hub/shell-services/lube-optimizer
+business-customers/shell-market-hub/shell-services/lube-optimizer.html
+business-customers/shell-market-hub/shell-services/machinemax
+business-customers/shell-market-hub/shell-services/machinemax.html
+business-customers/shell-market-hub/shell-services/shell-desiccant-breathers
+business-customers/shell-market-hub/shell-services/shell-desiccant-breathers.html
+business-customers/shell-market-hub/shell-services/shell-lube-analyst
+business-customers/shell-market-hub/shell-services/shell-lube-analyst.html
+business-customers/shell-market-hub/shell-services/shell-lube-videocheck
+business-customers/shell-market-hub/shell-services/shell-lube-videocheck.html
+business-customers/shell-market-hub/shell-services/shell-remote-sense
+business-customers/shell-market-hub/shell-services/shell-remote-sense.html
+business-customers/trading-and-supply/counterparty-information-request
+business-customers/trading-and-supply/counterparty-information-request.html
 business-enterprise/cesm3/index_v2.php?key5sk1=
 business-enterprise/cesm3/index_v2.php?s=
 business-insider$
@@ -22041,6 +22046,20 @@ careers/MAINTAINERS.txt
 careers/Phoenix-Hiring-Event/
 careers/Sydney-Hiring-Event/
 careers/UPGRADE.txt
+careers/about-careers-at-shell/employee-stories/developing-the-female-leaders-of-tomorrow
+careers/about-careers-at-shell/employee-stories/developing-the-female-leaders-of-tomorrow.html
+careers/about-careers-at-shell/employee-stories/dreaming-big
+careers/about-careers-at-shell/employee-stories/dreaming-big.html
+careers/about-careers-at-shell/employee-stories/endless-opportunities
+careers/about-careers-at-shell/employee-stories/endless-opportunities.html
+careers/about-careers-at-shell/employee-stories/exploring-geology-on-the-shell-graduate-programme
+careers/about-careers-at-shell/employee-stories/exploring-geology-on-the-shell-graduate-programme.html
+careers/about-careers-at-shell/employee-stories/testing-my-abilities
+careers/about-careers-at-shell/employee-stories/testing-my-abilities.html
+careers/about-careers-at-shell/people-at-shell
+careers/about-careers-at-shell/people-at-shell.html
+careers/about-careers-at-shell/we-are-one-team/diversity-inclusion
+careers/about-careers-at-shell/we-are-one-team/diversity-inclusion.html
 careers/admin
 careers/analytics
 careers/apply
@@ -22062,6 +22081,8 @@ careers/aws-sea-event/
 careers/aws-singapore-event/
 careers/aws-sydney-event/
 careers/chattanooga-mobile-event/
+careers/choose-your-career-profile
+careers/choose-your-career-profile.html
 careers/comment/reply
 careers/cron.php
 careers/database-hiring/
@@ -22086,7 +22107,11 @@ careers/job
 careers/jobs
 careers/jobs.php
 careers/jobs.php?id=
+careers/join-the-shell-talent-community
+careers/join-the-shell-talent-community.html
 careers/landing-pages/
+careers/life-at-shell
+careers/life-at-shell.html
 careers/logon
 careers/mentor-knowledge-bank/
 careers/mexico-edge-hiring-2014/
@@ -22130,6 +22155,8 @@ careers/scripts
 careers/search
 careers/search?
 careers/session-img
+careers/students-and-graduates
+careers/students-and-graduates.html
 careers/students-graduates/opportunities
 careers/texas-event/
 careers/thank_you
@@ -24813,7 +24840,7 @@ cgi-bin/loginpage
 cgi-bin/longs.cgi
 cgi-bin/mb
 cgi-bin/msdemo/
-cgi-bin/notices/http://analytics.twitter.com/robots.txt
+cgi-bin/notices/http://ams.usda.gov/robots.txt
 cgi-bin/onTopic
 cgi-bin/orphanWikiPages
 cgi-bin/outline
@@ -25033,7 +25060,6 @@ change-shipping
 changeLanguage
 change_password=
 changebrowser
-changeplan
 changes
 changes-that-qualify-for-special-enrollment/
 changliu
@@ -25056,6 +25082,7 @@ channels
 channels-extern
 channels-old
 channels/*/rss
+channels?q=
 chaochen
 chaplin
 chaplin-office-hours
@@ -25219,7 +25246,6 @@ chcleveland
 chdinh
 chebi
 check
-check-logged-in
 check-redirect
 check-web-status.html
 check.htm
@@ -25355,7 +25381,6 @@ childrenCount=
 childtaxcredit
 chilton
 chimileski
-chinese
 chinese/about/school-staff.php?id=ODAwMjIy&page_var=personal
 chinese/about/staff-by-role.php?id=ODAwMjIy&page_var=personal
 chineselanguageseries
@@ -25792,7 +25817,7 @@ closures/index.ssf?other
 cloud-launcher/sitemap
 cloud-setup
 cloud/
-cloud/
+cloud/
 cloud/131
 cloud/august2015/servicecs_gs
 cloud/farel10
@@ -25940,7 +25965,7 @@ cn/products-lifecycle
 cn/products/online-datasheet/
 cn/products/ras/trial-center
 cn/random
-cn/request-information/
+cn/request-information/
 cn/search
 cn/site-achievement
 cn/social
@@ -26218,7 +26243,6 @@ comeuntochrist/*/internal-use-only/
 comeuntochrist/*/search-results/
 comeuntochrist/lp/
 comeuntochrist/teach/
-comhem
 comic-indies
 comic-indies/$
 comic-indies/about
@@ -26287,6 +26311,10 @@ commerce/signin.asp
 commerce/subscribeToNewsletter.action
 commercesearch/images
 commercesearch/services
+commercial-air-purifiers
+commercial-air-quality-monitors
+commercial-applications
+commercial-hvac-air-filtration
 commercial-preview/
 commercial/flightxml/v3
 commercialservices/catering_and_bars
@@ -26395,7 +26423,6 @@ community/comments
 community/community-partnerships/fireside-chat/
 community/community-partnerships/fireside-chat/thank-you/
 community/community-relations/isc-lighting-request/thank-you/
-community/compose/*
 community/conversations
 community/email
 community/error-subdivision-name-unavailable
@@ -26443,7 +26470,6 @@ comodo-vs-symantec/?
 comp
 comp/
 compagnie
-companies
 companies/
 companies/*/edit
 companies/*/embed
@@ -26814,6 +26840,7 @@ contact-comodo/sales_process.php
 contact-comodo/sales_process.php?af=
 contact-comodo/thank-you.php
 contact-form-7
+contact-form.htm
 contact-myheritage
 contact-opnemen-met-myheritage
 contact-opnemen-met-ondersteuning
@@ -27372,7 +27399,7 @@ content_title=
 content_votes
 contentassets/*.xml$
 contentfeed/getdata?type=synopsi
-contentjam/
+contentjam/
 contentobject
 contentpar_gridblock
 contentresearchtool/
@@ -27759,7 +27786,6 @@ courses/section
 courses/transcript
 courtneytwittekind
 courts
-courts
 cover
 cover-it-live/
 cover-photo.jspa
@@ -27809,16 +27835,15 @@ cpl-edit
 cpoirier
 cpollack
 cpresources
+cpro
 cpropub/univercd
 cpsimopoulos
 cpspa
 cpx.php
 cqiu
 cr
-cr$
 cr/
 cr/Download
-cr?
 craft
 crafts
 crai/big
@@ -28189,7 +28214,6 @@ cservice/ya_index.gsp
 cshalaby
 csi_204
 csilvaportero
-csl
 csmlists/addbookmark
 csmlists/topic
 csn-search-app/*
@@ -28214,14 +28238,72 @@ css/*.css*$
 css/*hotels*?
 css/*restaurants*?
 css/*things-to-do/*?
+css/Bidding.css
+css/ToolTip.css
+css/UserControls.css
+css/cSaleRegistration.css?010120180943
+css/cSaleRegistration.css?030120181509
+css/cSaleRegistration.css?040120181248
+css/cSaleRegistration.css?060120181545
+css/cSaleRegistration.css?070120181312
+css/cSaleRegistration.css?100120181322
+css/cSaleRegistration.css?120120181954
+css/cSaleRegistration.css?131220170707
+css/cSaleRegistration.css?131220171002
+css/cSaleRegistration.css?161220170733
+css/cSaleRegistration.css?211220170547
+css/cSaleRegistration.css?221120170127
+css/cSaleRegistration.css?221220170940
+css/cSaleRegistration.css?241220171114
+css/cSaleRegistration.css?251220171115
+css/cSaleRegistration.css?301220171323
+css/calresults_SaleLanding.css
+css/crsl.css
+css/crsl_2013.css
+css/entersalepop.css
 css/fancybox.css
 css/fonts
 css/ie.css
 css/images/default
 css/index.php
+css/mychristies-p1.css
+css/popup.container.css
 css/print.css
 css/quiz.css
+css/sale-landing-catgrid.css
+css/sale-landing-events.css
+css/sale-landing-microsite.css
+css/sale-landing-single-page.css
+css/sale-landing-v1.css
+css/sale-landing.css
+css/sale-landing_2013.css
 css/screen.css
+css/signup.css
+css/site.css
+css/site.layouts.left_nav.css
+css/site.layouts.leftnav.css
+css/site.pages.MyChristies_Login.css
+css/site.pages.advanced_search.css
+css/site.pages.content_archive.css
+css/site.pages.ecatalogues.css
+css/site.pages.locations.css
+css/site.pages.locations.ny.css
+css/site.pages.lot_details_popup.css
+css/site.pages.popular_categories.css
+css/site.pages.press-center.css
+css/site.pages.promo_landing_page.css
+css/site.pages.rss.css
+css/site.pages.sale_departments_tabs.css
+css/site.pages.special_events.css
+css/site.pages.specialists.css
+css/site.pages.template.css
+css/site.pages.viewingcalendar.css
+css/site.pages.zoom_image.css
+css/site.print.css
+css/site.print.css?date=09012010
+css/site.sections.careers.css
+css/site.sections.careers.peoplebank.css
+css/site.sections.multimedia.css
 css2/
 css=*
 css?
@@ -28359,7 +28441,6 @@ customization-work-integration-ps
 customization-work-multivendor
 customization-work-oc
 customize
-customize$
 customize_autosaved=
 customize_changeset_uuid=
 customize_search
@@ -29326,7 +29407,7 @@ de/inc
 de/includes_content/missing_handler/view_map.html
 de/index.html
 de/info
-de/informationen-anfordern/
+de/informationen-anfordern/
 de/internal
 de/invites/buy
 de/ipns
@@ -29973,7 +30054,6 @@ device-prefs
 device/activate
 device/tracker/
 deviceList
-deviceManagement
 devinmccabe
 devlabs/
 devladervishmahler
@@ -30274,7 +30354,6 @@ diegoestevez
 diegorivera
 diener
 diets/*
-diff
 diffbot/
 diffs
 dig-perf-test
@@ -31281,7 +31360,7 @@ downloadpubliclightbox.php
 downloads
 downloads.php
 downloads/
-downloads/
+downloads/
 downloads/*
 downloads/archicad/install
 downloads/c1_downloads_auswahl
@@ -31338,10 +31417,12 @@ dritte
 drive
 drive-and-listen/
 drive-angry
-drive-in/
+drive-in/
 drive-with-lyft*ref=
 drive-with-lyft*utm
 drivers
+drivethestarship
+drivethestarship.html
 drj
 drjosephinekim
 drlem
@@ -31723,7 +31804,6 @@ editorial/sports/$
 editors
 editorspicks
 editorv2_shop_aux/download_course_student_certificate
-editpayment
 edits
 edmontonalberta
 edoardoteso
@@ -32034,7 +32114,6 @@ elim
 eliminar-busqueda
 eliminar-favoritos
 elio
-elisa
 elisanew
 elisarubbo
 elisewilkes
@@ -32149,7 +32228,6 @@ emails/verify
 emailservice
 emailsignup
 emailsignup/
-emailunsubscribe
 emailv_v2
 emallove
 embargo
@@ -33758,7 +33836,7 @@ en/recruitments
 en/reeds-aangemeld
 en/related-content?
 en/renew.html
-en/request-information/
+en/request-information/
 en/resources/
 en/responsive-header
 en/restaurants/
@@ -34062,16 +34140,12 @@ enterprise/products/checks
 enterprise/products/checks/index.jsp
 enterprise/seo
 enterpriseform/
-entertain
 entertainment/article/Minerva-s-horoscope
 entertainment/bravo
 entertainment/index.html
 entertainment/test$
 enterzipcode.php
 entities
-entities/search
-entityexplore$
-entityexplore?
 entityreference
 entp
 entr/Die%20Geh%C3%A4lter%20wurden%20gek%C3%BCrzt
@@ -34093,7 +34167,6 @@ entry/comment
 entry/image
 entry/jsonlite
 entrypoint_ndp
-entrytrap
 enus/
 enviar_amigo
 enviarnoticia.aspx
@@ -34136,10 +34209,11 @@ epic
 epilepsy-seizures-rapid-rescue-single-treatment
 episerver
 episerver
-episode-engine/
-episode-podcast/
-episode/
+episode-engine/
+episode-podcast/
+episode/
 episode_times
+episodes?q=
 epmt
 epo
 epo-dev
@@ -34935,7 +35009,7 @@ es/sharecontacts-for-microsoft-outlook.aspx
 es/shop_name_search_service
 es/site-achievement
 es/sobre_nosotros
-es/solicitar-informacion/
+es/solicitar-informacion/
 es/solutions/case-studies/
 es/solutions/case-studies/innovators/innovators-eu/
 es/solutions/case-studies/sustainability-emea/
@@ -35132,6 +35206,7 @@ eskupTSUpdate
 esl/layout/set
 esmecaramello
 esnl
+esntial.htm
 esoft-job
 esotiri
 espace-securise/shield
@@ -35404,7 +35479,6 @@ eucistron
 eugenchua
 eugeniah
 eugenio
-eula
 eula.html
 eum
 eumcollector/beacons/browser/v1
@@ -35886,6 +35960,9 @@ extensions/eventack.html
 extensionstable.htm
 extern_js
 external
+external-catalog-services
+external-redirects
+external-redirects.html
 external-test-ids=
 external/
 external/BusArch
@@ -36203,7 +36280,6 @@ fckeditor
 fcollado
 fcyrraci
 fd-fr/
-fd/
 fdbk
 fdias
 fding
@@ -36671,6 +36747,7 @@ films/*/year
 films/year
 filter
 filter-partner-vertical
+filter-subscription
 filter.do
 filter/
 filter/bestseller
@@ -36797,7 +36874,6 @@ firefox/?
 firefox/downloads
 firefox/new_tab_learn_more
 firehose.pl
-firetv
 firm_webinars
 firme/pierluigi-battista_80
 firmen/
@@ -36917,7 +36993,7 @@ flights/index.hi.
 flights/results
 flightsearch
 flint-deals
-flip-player/
+flip-player/
 flip-preview
 flipboardrss
 flipbook
@@ -38172,7 +38248,7 @@ fr/datos-personales
 fr/days/world-radio/grid
 fr/days/world-radio/list
 fr/days/world-radio/map
-fr/demande-informations/
+fr/demande-informations/
 fr/develop/
 fr/develop/review/
 fr/devolucion-de-productos
@@ -38612,6 +38688,7 @@ fragen/
 fragment
 fragment.*.html
 fragment.*.json
+fragment.htm
 fragment.html
 fragment.json
 fragment/
@@ -38663,7 +38740,6 @@ freehosting.php
 freejoin
 freeland
 freemantest
-freemonth
 freepublications/
 freertos/archive
 fren/volumeters
@@ -38746,8 +38822,6 @@ fullscreen
 fullscreen/ReportLinker
 fullwiley
 fun
-fun/api/
-fun/g/
 functions
 funderportal
 funding-for-researchers/our-funding-schemes?*
@@ -38947,6 +39021,7 @@ gabrielchiong
 gabrielguillen
 gabrielhosu
 gafinogenov
+gag-in-app
 gage
 gageraley
 gagne
@@ -39280,7 +39355,6 @@ geomag/WMM/data
 geomag/cdroms
 geomag/data
 geomag/pmag/datafiles
-geooverride
 georganteli
 georgeblaustein
 georgehana
@@ -40610,6 +40684,7 @@ help-desk-series-resetting-your-password
 help-desk-series-smart-blocking
 help-desk-series-the-mystery-of-the-unknown-category
 help-desk-series-trust-site
+help-me-choose
 help-panel
 help-v1
 help-with-your-research/research-guides/adoptions
@@ -40806,10 +40881,9 @@ highwire
 hijack
 hilaryweingarden
 hilf
+hilfe/assistent.html
 hilfe/fundingdetail.html
-hilfe/live/launch-day/launch-day-show-voting.html
 hilfe/live/launch-day/launch-day-videoupload.html
-hilfe/live/officehour/officehour-link.html
 hillarystein
 hillis
 him
@@ -41204,6 +41278,7 @@ homepage
 homepage-b
 homepage-beta
 homepage-video-test
+homepage/
 homepage/2013
 homepage/HomepageVideoMetrics.htm
 homepage/health/2009/fall/nobel-prize.shtml
@@ -41310,7 +41385,6 @@ hotel/*/?type
 hotel/h
 hotel/index.hi.
 hotel/redirect
-hotel/reviews?
 hotel/s
 hotel/search
 hotel/us/the-airstream-van.*.html
@@ -41361,7 +41435,6 @@ hotels/reservation-payment.htm
 hotels/rpc
 hotels/s
 hotels/search
-hotels/search
 hotels/search-results.htm
 hotels/search/list*
 hotels/search/map*
@@ -41426,8 +41499,6 @@ hp/focus-retrocompatibile$
 hp/video
 hp_channels
 hpan
-hpm
-hpmob
 hpols_catalog/
 hpws/max-connections.php
 hpws/parallel-downloads.php
@@ -44582,7 +44653,7 @@ html?fr=
 html?google_editors_picks=
 html?noamp=
 html_bbs/
-html_email/
+html_email/
 html_externe
 html_file/
 html_templates
@@ -44603,7 +44674,7 @@ http://1731177.fls.doubleclick.net/robots.txt
 http://1password.com/robots.txt
 http://2224480.fls.doubleclick.net/robots.txt
 http://4shared.com/robots.txt
-http://6323411.fls.doubleclick.net/robots.txt
+http://8293531.fls.doubleclick.net/robots.txt
 http://abc7chicago.com/robots.txt
 http://abc7ny.com/robots.txt
 http://abcnews.go.com/robots.txt
@@ -44642,7 +44713,6 @@ http://bitcoin-informant.de/robots.txt
 http://blackholerecordings.com/robots.txt
 http://bleepingcomputer.com/robots.txt
 http://blog.bufferapp.com/robots.txt
-http://boeing.com/robots.txt
 http://businessbuiltonline.blogspot.com/robots.txt
 http://businesstoday.in/robots.txt
 http://cambridge.org/robots.txt
@@ -44689,7 +44759,6 @@ http://form.jotform.com/robots.txt
 http://francetvinfo.fr/robots.txt
 http://gal2.org/robots.txt
 http://galu.co.jp/robots.txt
-http://geraldtabios.com/robots.txt
 http://girlsridetoo.de/robots.txt
 http://govinfo.gov/robots.txt
 http://graphics8.nytimes.com/robots.txt
@@ -45858,10 +45927,10 @@ image/cache
 image/cache/*1400x1400
 image/detailprint
 image/new
+image/viewer.htm
 imageClipperUpload
 image_site
 imagelib
-imagelibrary
 imagelightbox?elementId
 imagenes
 imagenes/novias/20201021177551/vestido-novia-desmontable-viral-romancera/*/*.jpg
@@ -45898,8 +45967,6 @@ images/about/
 images/adspace/
 images/api
 images/application_uploads
-images/async/?
-images/async?
 images/campaign/
 images/carBlogsimages
 images/chart_gifs
@@ -45912,9 +45979,6 @@ images/dopfiles/DNT20170204041207.pdf
 images/email/
 images/ga_img_handler.php
 images/home/
-images/hosted/?
-images/hosted/search?
-images/hosted?
 images/interview_pdfs
 images/intfood/thermomix/HF_icon_thermomix.svg
 images/li.gif
@@ -45929,14 +45993,9 @@ images/pad/search?text=*&source=morda
 images/related
 images/resume
 images/scaled
-images/search%3F
-images/search%5C
-images/search/?
 images/search/details/twitterCards/twitter-card
-images/search?
 images/search?rpt=imageview
 images/search?text=*&source=morda
-images/searchbyimage
 images/showroomimages
 images/smart/$
 images/staff
@@ -45952,9 +46011,6 @@ images/verify_image.png
 images/viewconfig$
 images/visit/
 images/welcome_image.png
-images?
-images?*mmbaseline*
-images?*nr*
 imagesLinks
 imagine/
 imaging/appointments-and-referrals/email-signup-thankyou
@@ -46202,7 +46258,6 @@ inbox-talk
 inbox-user
 inbox.aspx
 inbox.php
-inboxcommerce
 inc
 inc/
 inc5000dev
@@ -46221,7 +46276,7 @@ include/
 includes
 includes
 includes/
-includes/
+includes/
 includes/*
 includes/*.asp$
 includes/*.js
@@ -46486,35 +46541,10 @@ info-service/api/v2/ev/
 info/
 info/*
 info/Review
-info/agb/anb-2018.html
 info/agb/api.html
 info/agb/starter-2018.html
-info/agb/teilnahmebedingungen.html
 info/china-license.html
 info/consent
-info/jobs/Development-Operations-Engineer.html
-info/jobs/Kooperations-Manager.html
-info/jobs/Social-Media-Online-Kommunikation.html
-info/jobs/UX-UI-Designer.html
-info/jobs/ba-medieninformatik.html
-info/jobs/buchhaltung-payment.html
-info/jobs/buchhaltung-personal.html
-info/jobs/compliance.html
-info/jobs/crowd-agentin.html
-info/jobs/frontend-entwickler.html
-info/jobs/kommunikationsdesign.html
-info/jobs/kommunikationsstrategin.html
-info/jobs/multimedia-journalistin.html
-info/jobs/product-owner.html
-info/jobs/projektberater.html
-info/jobs/scrum-master.html
-info/jobs/showmaster.html
-info/jobs/site-reliability-engineer.html
-info/jobs/social-media.html
-info/jobs/support.html
-info/jobs/texterin-uxwriterin.html
-info/jobs/werkstudentin-demokratie-botschafterin.html
-info/jobs/werkstudentin-social-media.html
 info/logistic
 info/schedule/includes
 info/search
@@ -46930,6 +46960,7 @@ intshippingpopup.aspx
 intuition
 inv
 inv_unsub
+inventory-detail/print.htm
 investcom/
 investing/FierceMarkets
 investing/businesswire
@@ -46942,6 +46973,8 @@ investors/de/stock/fixed-income/bond-documentation
 investors/de/stock/fixed-income/cp-documentation
 investors/en/stock/fixed-income/bond-documentation
 investors/en/stock/fixed-income/cp-documentation
+investors/shareholder-meetings/2021-agm-webcast
+investors/shareholder-meetings/2021-agm-webcast.html
 invitation
 invitation
 invitation-maker
@@ -47001,7 +47034,7 @@ ip/*zoom_level=10
 ipAcl
 ipAddress=
 ipad
-ipad/
+ipad/
 ipad/4/Contents/en/searchindex
 ipadconsent
 ipc
@@ -47510,7 +47543,7 @@ it/responsive-header
 it/retail/search$
 it/retail/search?
 it/ricerca
-it/richiesta-informazioni/
+it/richiesta-informazioni/
 it/scisoft/star
 it/search
 it/search*attr_*=
@@ -48922,18 +48955,154 @@ js/*.js*$
 js/*hotels*?
 js/*restaurants*?
 js/*things-to-do/*?
+js/AddHandler.js
+js/BidContainerPopUp.js
+js/BidContainerPopUp.js?date=16001231070000
+js/BidContainerPopUp2.js
+js/BidcontainerPopUp.js
+js/ChristiesLiveBidding_test.js
+js/FollowSaleLot.js
+js/FollowSaleLot.js?13072015
+js/LoadFastSignUp.js
+js/LoadFastSignUp.js?_=1512604800039
+js/LoadFastSignUp.js?_=1513036800038
+js/LoadFastSignUp.js?_=1513900800038
+js/LoadFastSignUp.js?_=1514073600037
+js/LoadFastSignUp.js?_=1514246400038
+js/LoadFastSignUp.js?_=1514332800038
+js/LoadFastSignUp.js?_=1514592000037
+js/LoadFastSignUp.js?_=1514592000038
+js/LoadFastSignUp.js?_=1514678400037
+js/LoadFastSignUp.js?_=1514764800037
+js/LoadFastSignUp.js?_=1514851200037
+js/LoadFastSignUp.js?_=1515024000038
+js/LoadFastSignUp.js?_=1515110400037
+js/ToolTip.js
 js/adobe/at.js
 js/ajaxconn
+js/chr.analytics.js
+js/chr.calendar.js
+js/chr.configs.analytics.js
+js/chr.configs.carousel.js
+js/chr.configs.carousel_2013.js
+js/chr.configs.carousel_2013_spinset.js
+js/chr.configs.share.js
+js/chr.configs.showhide.js
+js/chr.modules.crsl.js
+js/chr.share.js
+js/chr.util.calendar.js
+js/chr.util.showhide.js
+js/entersaleP2.js
 js/errors
+js/jAlertPopUp.js
+js/jAlertPopUp.js?010120180943
+js/jAlertPopUp.js?030120181509
+js/jAlertPopUp.js?040120181248
+js/jAlertPopUp.js?060120181545
+js/jAlertPopUp.js?070120181312
+js/jAlertPopUp.js?100120181322
+js/jAlertPopUp.js?120120181954
+js/jAlertPopUp.js?131220170707
+js/jAlertPopUp.js?131220171002
+js/jAlertPopUp.js?161220170733
+js/jAlertPopUp.js?211220170547
+js/jAlertPopUp.js?221120170127
+js/jAlertPopUp.js?221220170940
+js/jAlertPopUp.js?241220171114
+js/jAlertPopUp.js?251220171115
+js/jAlertPopUp.js?301220171323
+js/jCallWebService.js
+js/jSaleRegistration.js?010120180943
+js/jSaleRegistration.js?030120181509
+js/jSaleRegistration.js?040120181248
+js/jSaleRegistration.js?060120181545
+js/jSaleRegistration.js?070120181312
+js/jSaleRegistration.js?100120181322
+js/jSaleRegistration.js?120120181954
+js/jSaleRegistration.js?131220170707
+js/jSaleRegistration.js?131220171002
+js/jSaleRegistration.js?161220170733
+js/jSaleRegistration.js?211220170547
+js/jSaleRegistration.js?221120170127
+js/jSaleRegistration.js?221220170940
+js/jSaleRegistration.js?241220171114
+js/jSaleRegistration.js?251220171115
+js/jSaleRegistration.js?301220171323
+js/jcarousel.js
+js/jcarousellite_1.0.1.min.js
+js/jcarousellite_1.8.2.min
+js/jcarousellite_1.8.2.min.js
+js/jplayer/js/jquery.jplayer.min.js
+js/jplayer/skins/christies/christies.css
 js/jquery
+js/jquery.placeholder.js
 js/jquery/*
+js/jquery/jquery-1.7.1.min.js?date=09012010
+js/jquery/jquery-1.9.1.js
+js/jquery/jquery.cookie.js
+js/jquery/jquery.cookie.js?_=1512604800038
+js/jquery/jquery.cookie.js?_=1513036800037
+js/jquery/jquery.cookie.js?_=1513900800037
+js/jquery/jquery.cookie.js?_=1514073600036
+js/jquery/jquery.cookie.js?_=1514246400037
+js/jquery/jquery.cookie.js?_=1514332800037
+js/jquery/jquery.cookie.js?_=1514592000036
+js/jquery/jquery.cookie.js?_=1514592000037
+js/jquery/jquery.cookie.js?_=1514678400036
+js/jquery/jquery.cookie.js?_=1514764800036
+js/jquery/jquery.cookie.js?_=1514851200036
+js/jquery/jquery.cookie.js?_=1515024000037
+js/jquery/jquery.cookie.js?_=1515110400036
+js/jquery/jquery.min.js
+js/jquery/jquery.min.js?date=09012010
+js/jquery/jquery.plugins.min.js
+js/jquery/jquery.plugins.min.js?date=09012010
+js/jquery/modernizr-1.6.min.js
+js/jquery/site.jquery.events.js
+js/jquery/site.jqwidget.tooltip.js
+js/jquery/site.js
+js/jquery/site.pages.zoom_image.js
+js/jquery/site.sections.multimedia.cookie.js
+js/jquery/site.sections.multimedia.js
+js/jquery/site.sections.mygallery.js
+js/lib/lib.min.js?date=09012010
 js/main-*.js
 js/mbox.js
 js/powellsJQuery.validity.js?*
 js/powellsValidationData.js?*
 js/product_reveal_gdm_ajax.js
+js/sales_landing.js
+js/sales_landing_2013.js
+js/sales_landing_2014.js
+js/signup.js
+js/site.advanced_search.js
+js/site.controllers.collapse_panel.js
+js/site.controllers.field_hint.js?date=09012010
+js/site.controllers.list_tooltips.js
+js/site.controllers.sale_tooltips.js
+js/site.controllers.tooltips.js?date=09012010
+js/site.js?date=09012010
+js/site.lang.en.js?date=09012010
+js/site.legacy-utils.js
+js/site.page.js?date=09012010
+js/site.pages.about.googlemap.js
+js/site.pages.calendar_new.js
+js/site.pages.events.js
+js/site.pages.rss.js
+js/site.pages.specialists.js
+js/site.pages.timesbasedsalemessages.js
+js/site.widgets.calendar.js
+js/site.widgets.collapse_panel.js
+js/site.widgets.drop_down.js?date=09012010
+js/site.widgets.rich_checkbox.js
+js/site.widgets.rich_checkbox_list.js
+js/site.widgets.rich_select.js
+js/site.widgets.ticker.js?date=09012010
+js/site.widgets.tooltip.js
+js/site.widgets.tooltip.js?date=09012010
 js/spmsg-
 js/streamsense-*.min.js
+js/swfobject.js
 js/trash
 js/widget.js
 js2
@@ -48991,6 +49160,7 @@ json/property/estimates
 json/search
 jsonapi/*
 jsonp/Bdp.htm?
+jsonsearch
 jsp-templates
 jsp/events
 jsscripts/
@@ -49672,7 +49842,6 @@ kontakty
 kopl
 kopt
 koray
-korean
 korina
 korisauserzachrison
 koro
@@ -49702,7 +49871,6 @@ kperdue
 kperkins
 kpi
 kpjmckeon
-kpn
 kporter
 kqin
 kr
@@ -49927,7 +50095,7 @@ lan
 lancaster
 land
 land-browse-bff
-land/
+land/
 land/search.php
 landing
 landing-feed
@@ -49939,7 +50107,7 @@ landing-page/licensing
 landing-page/webinar/kessler-expertserieswebinar-mojo-assetdeliveryhttp://curbed.com/robots.txt
 landing-pages
 landing.php
-landing/
+landing/
 landing/braintree-paypal
 landing/brand/
 landing/brand/adam-eve-products$
@@ -50073,6 +50241,7 @@ latam/*
 latam/blog/
 latam/blog/*
 latest
+latest-amp-article
 latest-news/comodo-and-cybrhawk-announce-technology-partnership-cybrhawk-integrates-comodos-openedr-into-xdr-solution-set/?
 latest-playlist
 latest-talk
@@ -50087,7 +50256,7 @@ latinamerica/wp-admin
 latinamerica/wp-content/plugins
 latinamerica/wp-includes
 latoyaprincessjackson
-launch/
+launch/
 launch2005/
 launch_invitation
 launcher
@@ -50262,8 +50431,6 @@ learningarcade/
 learningcenter
 learnmore
 learnmore-transferhttp://mc.yandex.ru/robots.txt
-learnmorelayer
-learnmorelayertv
 leave
 leave-a-legacy/legacy-guide/
 leave-a-legacy/thank-you/
@@ -50702,6 +50869,7 @@ link.php
 link/
 link2
 link:
+link?
 link_preview_image
 link_to
 linked
@@ -50795,7 +50963,7 @@ listmanager/goodbye.php
 listmanager/notification1.php
 listmanager/notification2.php
 lists
-lists/
+lists/
 lists/search
 lists/search?q=
 listspider/
@@ -50851,6 +51019,7 @@ live/message
 live/report.rvt
 live_chat
 live_parties
+livebidding/
 liveblog
 liveblog/*
 liveblog/*/?cardId=
@@ -50930,7 +51099,6 @@ loans
 loc/volunteers
 loc=
 local
-local
 local-dev
 local-dev-old
 local-navigation/
@@ -51034,6 +51202,7 @@ login-callback
 login-for-zendesk
 login-messages
 login-mgt
+login-signup
 login-success
 login-v2?
 login.aspx
@@ -51100,10 +51269,7 @@ login_or_join/redirected?
 login_or_join?
 login_page
 login_with_redirect
-loginhelp
 loginhelp/
-loginhelp/lookup
-loginhelpsucess
 loginidentification/
 logo
 logo-maker/brief
@@ -51194,6 +51360,7 @@ lowFareDestinationAirport=
 lowFareOriginAirport=
 lowFareOutboundDateString=
 lowFareReturnDateString=
+loyalty-program
 loyalty?id=
 lp
 lp-
@@ -51856,9 +52023,7 @@ mail-okay
 mail.php
 mail/
 mail/$
-mail/subscribe
 mail/termsandconditions
-mail/valid-
 mail/view
 mail/webhook/open
 mail?
@@ -52033,7 +52198,7 @@ maoliangye
 map
 map-constructor/loader
 map-regular.xml
-map-review/
+map-review/
 map-search-flats/leningradskaya-oblast/$
 map-search-flats/moskovskaya-oblast/$
 map-search-flats/moskva/$
@@ -52067,7 +52232,6 @@ maps/*?lang=uk$
 maps/*?lang=uz$
 maps/*?ll=
 maps/about
-maps/adsendpoint
 maps/api/js
 maps/api/place/js
 maps/api/staticmap
@@ -52124,7 +52288,7 @@ margaretandrews
 margaretjenkins
 margarettroyer
 margotmoinester
-mari/
+mari/
 maria
 maria-garytrainingsite
 maria_acevedo
@@ -52430,7 +52594,6 @@ maxboland
 maxcdn.bootstrapcdn.com
 maxencecaillaud
 maximillian-soltysiak
-maxis
 maxkuheljbugaric
 maxnguyen
 maxp=
@@ -52524,6 +52687,8 @@ mbrady
 mbronstein
 mbsullivan
 mburkhauser
+mc-publication-test-folder
+mc-publication-test-folder.html
 mc.html
 mc/
 mc/all/ad.html
@@ -52543,7 +52708,6 @@ mcclintock
 mccom-admin/
 mccormack/forms/cameroon/thank-you/
 mccray
-mcd
 mcdowell
 mcgi
 mchan
@@ -52591,7 +52755,6 @@ mec-medicaid/
 mecha
 mechanic
 meconnect/
-medal/*
 meddev
 media
 media
@@ -52602,7 +52765,6 @@ media-library-links/
 media-library/sites/alspac/migrated/documents
 media-manager
 media/
-media/*
 media/*/analog-dialogue/
 media/*/company/sustainability-reports/
 media/*/edit
@@ -52649,7 +52811,6 @@ media/daodao/photo-t
 media/daodao/video-t
 media/daodao/video-v
 media/downloads/
-media/editimgecaption/*
 media/ethics-center/social-sector-ethics/
 media/fbavatar
 media/feeds
@@ -53208,7 +53369,6 @@ menu-test
 menu/*/feedback_form
 menu?
 menumachine
-meo
 meopa/search/node
 meps/*/eptvAjax
 meps/*/pdf*
@@ -53225,7 +53385,6 @@ merchant/directions
 merchant/imageUploadPreview
 merchant/loc
 merchant/moreProducts?
-merchant/reviews?
 merchant/update
 merchant_login
 merchant_shipping_policy.pdf
@@ -54076,7 +54235,6 @@ modern-formats-eng/ucp/static/*/client
 modern-formats-eng/ucp/static/*/csr
 modern-formats-eng/ucp/static/*/dev
 modern-formats-eng/ucp/static/framework_logging
-modernizr
 modes/driving/te-contact-us-make-an-enquiry-failure-26705
 modes/driving/te-contact-us-make-complaint-failure-*
 modify.php
@@ -54924,7 +55082,7 @@ mx/user
 mx/user$
 mx/ventureone
 mx/your
-mxf-final-cut-pro/
+mxf-final-cut-pro/
 mxhe
 my
 my*/blog-article-remove-faq-utms-*.js
@@ -55059,6 +55217,7 @@ mybestface
 myblog
 mybooking.html
 mybookoflists
+mycars
 mycart
 mydonate
 mye
@@ -55545,6 +55704,12 @@ newrelic
 news
 news$
 news*?
+news-and-insights/annual-publications
+news-and-insights/annual-publications.html
+news-and-insights/newsroom/news-and-media-releases/2023/shell-second-quarter-test-page
+news-and-insights/newsroom/news-and-media-releases/2023/shell-second-quarter-test-page.html
+news-and-insights/scenarios/a-better-life-with-a-healthy-planet
+news-and-insights/scenarios/a-better-life-with-a-healthy-planet.html
 news-app
 news-archive/*/*/?ap=
 news-archive/?
@@ -55558,7 +55723,7 @@ news-events/events-calendar/2019/05/tenth-annual-african-consumer-protection-dia
 news-ga
 news-insider$
 news-list
-news-manager/
+news-manager/
 news-marketing-collateral/press-releases/
 news-mobile
 news-redirect/*
@@ -55865,7 +56030,7 @@ newsletter-templates/weather
 newsletter-templates/weather-weekly
 newsletter-templates/weekly-messenger
 newsletter/
-newsletter/
+newsletter/
 newsletter/$http://digital-strategy.ec.europa.eu/robots.txt
 newsletter/*
 newsletter/archive/
@@ -57570,7 +57735,7 @@ non-searchable/
 nonConfig/api/wpa
 nonacm
 nonbot/
-nonce="_U70D0SmJsNA5xS_V4gQbw">if(google.cis
+nonce="_Qk4y8iG9CcJbaJ1PuSpkA">if(google.cis
 nonlystdomainredirect
 nonprofits-ngos
 nonprofits/account
@@ -57659,7 +57824,6 @@ not-found
 not-supported.html
 not_found
 not_recommended_reviews
-notamember
 note
 notebook/
 notebook/search?
@@ -57668,7 +57832,6 @@ notfound
 notfound.html
 notice/
 noticeboard
-notices
 noticias/20080416/destituyen-director-jefe-proteccion-asco-tras-fuga-radiactiva/28170.shtml
 noticias/envia_noticia.html
 noticias/live-fyre-blog-test*
@@ -57681,7 +57844,6 @@ notificationId
 notifications
 notifications
 notifications$
-notifications/
 notifications/*
 notify.js
 notifyMe.jsp
@@ -57736,7 +57898,6 @@ npr
 nqrwns
 nr/
 nraphael
-nrdapp
 nreed
 nrg
 nripsuta
@@ -58123,8 +58284,6 @@ offers/nonprofits$
 offers/nonprofits%
 offers/nonprofits?
 offers/preview/
-offers/proxy/dealsserver/api/log
-offers/proxy/dealsserver/buy
 offers/referral-terms-conditions
 offers/referral-terms-conditions$
 offers/referral-terms-conditions%
@@ -58462,10 +58621,7 @@ opssam1com
 opt-in-mobile
 opt-out
 opt-out-info.jsp
-opt-out-interface
 opt-out.jsp
-opt-out/http://youronlinechoices.eu/robots.txt
-opt-out/http://yourstory.com/robots.txt
 opt/widget-cache.php
 optInEmail
 opt_out
@@ -58487,7 +58643,6 @@ or/user
 or/users
 oral-and-maxillofacial-surgeon
 orange-anma
-orangefrance
 orcid-search
 order
 order
@@ -58621,6 +58776,7 @@ other/presentation
 otherContacts
 other_files
 otherlookup
+others
 otomobil-motor-aksesuar
 otp-failed-push-notif
 ottovonfeigenblatt
@@ -58664,7 +58820,6 @@ override
 override/
 overview
 overview.php
-overviewblockseeother
 owasow
 owenmartin
 owner-estimate
@@ -59305,7 +59460,6 @@ pamr/issues_experts/featured_experts_archive/index.html
 pamr/issues_experts/issues_experts_archive/index.html
 pamr/media_releases/index.html
 pamr/media_releases/media_releases_archive/index.html
-panasonic
 panda/jira
 panel
 panel
@@ -60049,7 +60203,6 @@ phone-fax-purchases.html
 phone/compare/?
 phonebook
 phoneinfo
-phonenumber
 phot-browse.asp*
 photo
 photo-
@@ -60080,6 +60233,7 @@ photo4/*/embed
 photo_albums
 photo_postcomment.php
 photo_request
+photobooks/about
 photogallery
 photogallery/*.xml$
 photogallery/6*.html
@@ -60183,7 +60337,6 @@ pierredegalbert
 pift/retest
 pigeon
 pimages
-pin
 pin-builder
 pin-communities
 pin-creation-tool
@@ -60457,7 +60610,7 @@ pl/pl/product-finder/internal-hard-drives/
 pl/pl/promos/
 pl/pl/promos/lyve-cloud-for-startups/
 pl/player
-pl/popros-o-informacje/
+pl/popros-o-informacje/
 pl/renew.html
 pl/retail/search$
 pl/retail/search?
@@ -60683,7 +60836,6 @@ playlist.xml
 playlists/*/user
 playlists/flag
 plays
-playstation
 plcz
 plde
 plehmann
@@ -61188,7 +61340,6 @@ populate
 populate/
 popup
 popup.php
-popup/codewhatisthis
 popup/emailconfirmation.xhtml
 popup=
 popup=true
@@ -61196,8 +61347,6 @@ popup?
 popup_salesforce.html
 popup_security.jsp
 popupclarin.asp
-popupdetails
-popupprivacypolicy
 popups/
 popups/mediaplayer
 popups/popup_gallery
@@ -61288,6 +61437,7 @@ post-rental-listings?zpid=
 post-style/*
 post/
 postQuestion.htm
+postRender
 post_comment
 post_forum_message.html
 post_id=*
@@ -61693,7 +61843,6 @@ privacy/request-data
 privacy/zoho-group.html
 privacy_policy
 privacypolicy/
-privacypolicychanges
 private
 private
 private-promo
@@ -61743,7 +61892,6 @@ pro/tools
 pro/upgrade
 pro/waze/report/*?*&hash=
 pro2
-proFile/history?
 proMatch
 proProjects
 proTeXt
@@ -61831,7 +61979,6 @@ product/prd-5193422
 product/productreviews/
 product/recommended
 product/review/add
-product/reviews?
 product/summer-plant-based-recipe-cookbook
 product/valentines-chocolate-vegan-cookbook
 product/vegan-desserts-cookbook
@@ -62028,6 +62175,7 @@ profile/*
 profile/*/Advice
 profile/*/Reviews-Written
 profile/*/Rss
+profile/*/blog
 profile/*/followers
 profile/*/following
 profile/*/map$
@@ -62038,8 +62186,10 @@ profile/*/recommendations/bymember
 profile/*/recommendations/member
 profile/*/recommendations/memberbooks
 profile/*/recommendations/unsuggester
+profile/*/reviews
 profile/*/stats
 profile/*/stats/library
+profile/*/trophies
 profile/*article*.ece
 profile/Rhizomer
 profile/author/$
@@ -62050,7 +62200,6 @@ profile/contributor/?
 profile/downloads_limit
 profile/edit
 profile/gravlady
-profile/history?
 profile/license
 profile/manage
 profile/messages
@@ -62359,10 +62508,8 @@ provi/check
 providers
 provost/articulation
 proximic
-proximus
 proxy
 proxy
-proxy/*
 proxy/accounts
 proxy/accounts/v1
 proxy/accounts/v1/groups
@@ -62775,7 +62922,7 @@ ptsv
 pttr
 ptype=
 ptzh
-pu
+pu/
 pu_all
 pu_echo/*.htaccess
 pu_echo/*.js
@@ -63092,7 +63239,6 @@ qa.www.spglobal.com
 qa.www.spglobal.com/*
 qa/
 qadministrator/mac/4.0/en/appleqadministrator/usermanual
-qantas
 qantas-group/delivering-today/fy18-highlights.html
 qbw
 qc_campaign
@@ -63423,7 +63569,6 @@ raters/ap-reader-chat/ap-reader-chat
 ratgeber/check/*
 rating
 rating/
-rating/average
 rating=*
 ratings
 ratings$
@@ -63673,16 +63818,9 @@ rechercher
 rechercher/
 recipe-box
 recipe-portal/search
-recipe/*/*/nutrition
-recipe/*/*/print
-recipe/*/bitflag
-recipe/*/feedback
 recipe/box
 recipe/edit
-recipe/email*
-recipe/link/*
 recipe/view
-recipe/zapcache/*
 recipe_parser
 recipes
 recipes/cooking
@@ -63816,6 +63954,7 @@ referencework
 referer
 referer=
 referral
+referral-program
 referral/
 referrals
 referrer=
@@ -63972,7 +64111,6 @@ relocator
 relpage
 remartin
 remember-the-titans
-rememberme
 remingtonhill
 remixed/rules/
 remo
@@ -64852,6 +64990,7 @@ reservations
 reservations/
 reservations/*/notifyme
 reservations/book-reservations.html
+reserve-it-now/form.htm
 reset
 reset-password
 reset-password
@@ -65136,7 +65275,6 @@ rewards/$
 rewards/*/membership
 rewards/*/status-matching
 rewards/bebas-ongkir
-rewardsapp/
 rex/recommendation/*
 rezepte/suche
 rf=*
@@ -65819,7 +65957,7 @@ ru/promotions
 ru/random
 ru/redirect-notification/
 ru/renew.html
-ru/request-information/
+ru/request-information/
 ru/ru/materials/*
 ru/ru/product-finder/external-hard-drives/
 ru/ru/product-finder/home-entertainment/
@@ -66036,6 +66174,7 @@ s3rd
 s=
 s=p
 s?
+s?
 s?*rh=n%3A1380045031
 s?k=
 s_id=
@@ -66108,7 +66247,7 @@ sale/uomo-*
 sale=
 salehizadeh
 salernolisa
-sales-kit/
+sales-kit/
 sales/digital
 sales/guest/form
 sales/includes
@@ -66162,14 +66301,12 @@ samlCertificates
 saml_login
 saml_sso
 samlee
-sample
 sample-
 samplesViewSingle
 samplescholar
 sampson
 samr
 samrosenfeld
-samsung
 samsus
 samsweney
 samuel_s_urlacher
@@ -66495,7 +66632,7 @@ scr5net
 scratch
 scratch_1.4
 scratchr2/static
-screen-flow/
+screen-flow/
 screen-time-strategies-you-can-start-today
 screen.js
 screen/image
@@ -66509,20 +66646,6 @@ screenshots-raw
 scrip.html
 script
 script/
-script/AdServe/Redir.aspx
-script/Articles/ArticleVersion.aspx
-script/Articles/Delete.aspx
-script/Articles/Download.aspx
-script/Articles/ListVersions.aspx
-script/Articles/Preview.aspx
-script/Articles/SubmissionWizard.aspx
-script/Articles/ViewHtml.aspx
-script/Catalog/Redir.aspx
-script/Forums/SetOptions.aspx
-script/Membership/LogOn.aspx
-script/Membership/OAuthLogOn.aspx
-script/Membership/SendPassword.aspx
-script/News/Redir.aspx
 script/ScriptServlet
 scriptPath
 scripteditorr1
@@ -66567,9 +66690,6 @@ sdfe/pdf/download/read/am4ir/noindex/pii
 sdfe/pdf/download/read/index/pii
 sdfe/pdf/download/read/noindex/pii
 sdfe/reader/pii
-sdg-newsletter.html
-sdg-newsletter/sdg-newsletter-abbestellen.html
-sdg-newsletter/sdg-newsletter-bestaetigen.html
 sdg4education2030/ar/event/*/ical
 sdg4education2030/ar/explore
 sdg4education2030/ar/search
@@ -67468,6 +67588,7 @@ series
 series/
 series/*
 series/optout
+series?q=
 seriesBeta
 seriesBetahttp://freecodecamp.org/robots.txt
 serietv/schede/cerca/?query=
@@ -67785,6 +67906,7 @@ sette/13_luglio_29/2013-31-fiaschetti-falsi-autori_9becf36c-f84e-11e2-a59e-96a50
 sette/speciale-presidenziali-usa-2024-harris-contro-trump/rss/*.xml
 settings
 settings
+settings.html
 settings/*
 settings/deactivated
 settings/linkaccount
@@ -68041,11 +68163,22 @@ shelemay
 shelf/user_shelves
 shell
 shell/
+shellbitumen100
+shellbitumen100.html
+shellenergy/marketingandtrading/shell-energy-europe/eworld
+shellenergy/marketingandtrading/shell-energy-europe/eworld.html
+shellenergy/othersolutions/voluntary-carbon-market-insights-and-trends-webinar/voluntary-carbon-market-webinar-2023-watch-recording
+shellenergy/othersolutions/voluntary-carbon-market-insights-and-trends-webinar/voluntary-carbon-market-webinar-2023-watch-recording.html
+shellenergy/progesstogether/case-studies-alimerka
+shellenergy/progesstogether/case-studies-alimerka.html
+shellenergy/progesstogether/case-studies-schneider-electric
+shellenergy/progesstogether/case-studies-schneider-electric.html
+shellenergy/tech/dc-whitepaper
+shellenergy/tech/dc-whitepaper.html
 shelleyxliu
 shellysimana
 shen
 sheng
-shenghuo
 shenghuo/search?
 shenyang
 sheraz
@@ -68061,6 +68194,7 @@ sherwin_special_offers
 shi
 shian
 shibboleth
+shifen/
 shihui
 shihui?
 shijiawei
@@ -68090,7 +68224,6 @@ shoelaces
 shomiller
 shootouts
 shop
-shop$
 shop-all
 shop-cart.shtml
 shop-data
@@ -68140,7 +68273,6 @@ shop/browse/overlay
 shop/browsing_history
 shop/checkout
 shop/claimcredit
-shop/collections
 shop/edit
 shop/eligible-for-special-financing/?
 shop/flattering-fitted-maxi-dress-5152169
@@ -68178,7 +68310,6 @@ shop/unified/availability.html?destination_page=BYOB
 shop/unified/availability/calltoorder.html
 shop/unified/availability/results.html
 shop/wishlist
-shop?
 shop_name_search_service
 shop_pins
 shopby
@@ -68439,7 +68570,6 @@ signup_modal
 signup_twitter.php
 signup_verify.php
 signups
-signurl
 sigou-interdisciplinary-blog
 sijang
 sijumonkunjachan
@@ -69388,7 +69518,6 @@ social/
 social/our_partners/
 social/photosui/images
 social/photosui/sitemap
-social/search?
 social/share_modal
 social/social-security-directory/cai
 social_media
@@ -69523,12 +69652,6 @@ son/user
 son/users
 song
 song/data
-song/delete_user_review
-song/fetch_overview_review
-song/fetch_review_view
-song/report_inappropriate
-song/review_thumbs
-song/save_user_review
 song/search
 songqi
 songs/
@@ -69542,7 +69665,6 @@ soniapujol
 sons-mp3-us
 sonumehta
 sonusubudhi
-sony
 soojin
 soojinpark
 sophia_lee
@@ -69628,7 +69750,6 @@ sparshasaha
 spartan
 spaul
 spb
-spbasic
 spbot
 spc-source=
 spc/amazon/en/landingPage.html
@@ -69673,6 +69794,7 @@ specialoffers
 specialprojects
 specials
 specials/north-america-2
+specials/trims.htm
 specialties?
 specialties?search=
 specialtopic
@@ -69817,7 +69939,6 @@ sports_fp_ms
 spotlight
 spp/
 spreadsheet
-spresults
 sprillaman
 sprince
 spring
@@ -70066,6 +70187,7 @@ static/*.pdf
 static/*.png
 static//static/fonts
 static/__sdemo__
+static/ads
 static/appdata
 static/buyer_research_inquiries
 static/chunks/*
@@ -70464,8 +70586,6 @@ storeclick
 storefront
 storepicker
 storerequest
-stores$
-stores/
 stores/page/*visitId=
 stories
 stories/shop
@@ -70768,7 +70888,6 @@ subscription/*?*source=
 subscription/subscribe
 subscription/subscribe$
 subscriptionDetails.htm
-subscriptioncancel
 subscriptions
 subscriptions/manage/
 subscriptions/securejson/
@@ -71033,6 +71152,24 @@ suspended_tickets
 suspension-appeal
 sustainability
 sustainability-as-an-efficient-business-strategy
+sustainability/nature/respecting-nature
+sustainability/nature/respecting-nature.html
+sustainability/our-approach/environmental-and-community-partners
+sustainability/our-approach/environmental-and-community-partners.html
+sustainability/our-approach/impact-assessment
+sustainability/our-approach/impact-assessment.html
+sustainability/our-climate-target/shell-energy-transition-strategy/ets-2024
+sustainability/our-climate-target/shell-energy-transition-strategy/ets-2024.html
+sustainability/people/access-to-energy/access-to-energy-in-communities/case-study
+sustainability/people/access-to-energy/access-to-energy-in-communities/case-study.html
+sustainability/people/access-to-energy/access-to-energy-in-communities/clean-cooking-solutions
+sustainability/people/access-to-energy/access-to-energy-in-communities/clean-cooking-solutions.html
+sustainability/safety/transport-safety/community-road-safety
+sustainability/safety/transport-safety/community-road-safety.html
+sustainability/safety/transport-safety/driver-safety
+sustainability/safety/transport-safety/driver-safety.html
+sustainability/transparency-and-sustainability-reporting/advocacy-and-political-activity/advocacy-releases
+sustainability/transparency-and-sustainability-reporting/advocacy-and-political-activity/advocacy-releases.html
 sustainable-development-goals/sustainable-cities
 sustainable-is-attainable-fest/*
 sustainable-is-attainable-news18-and-tata-power-initiative-2021/*
@@ -71474,6 +71611,8 @@ tag
 tag
 tag-manager
 tag-manager-360
+tag-search
+tag-search.html
 tag/
 tag/*
 tag/*
@@ -71792,7 +71931,6 @@ talk_to_us/?
 talkread
 talks2
 talkscreen.bml
-talktalk
 tallarita
 tally-loader-react/static-1.65
 tallyman/context
@@ -71833,7 +71971,6 @@ tasawarjalali
 tasima
 tasks
 tasks/
-tastesurvey
 tax-center
 tax.php
 tax_countries.php
@@ -71963,6 +72100,7 @@ tc/
 tc/accept
 tcarroll
 tcastellano
+tcd
 tcfv2
 tchayirli
 tcheng
@@ -71982,7 +72120,6 @@ td/profiles
 tdangelo
 tdaroundme
 tdautourdemoi
-tdc
 tdevine
 tdid=*
 tdir/p/kids-back-to-school
@@ -72132,7 +72269,6 @@ teleconnections/*.php?
 telegram998com
 telegraph
 telehealth-for-women
-telenor
 television/entretenimiento
 television/teletexto/incibe
 television/teletexto/noticias
@@ -72143,7 +72279,6 @@ television/tve-internacional/asia/?
 television/tve-internacional/europa/?
 television/tve-internacional/parrilla/?
 television/tve-internacional/programacion-tv/?
-telfort
 tell-friend
 tellAFriend.do
 telsearch
@@ -72345,7 +72480,6 @@ terms/looker/legal/suppliers/partner-assignment-faq
 terms/looker/legal/suppliers/supplier-code-of-conduct
 terms_of_use
 termsandprivacy.aspx
-termsofusechanges
 tervala
 tessibrahimchah
 tesswallace
@@ -72374,7 +72508,7 @@ test.html
 test.jsp
 test.umb.edu/
 test/
-test/
+test/
 test/action~
 test/action~agenda
 test/action~default
@@ -72502,7 +72636,6 @@ tge
 tgibson
 tgkkk6681
 th
-th$
 th*/blog-article-remove-faq-utms-*.js
 th*/cdn-cgi/challenge-platform
 th*/step
@@ -72651,7 +72784,6 @@ th/user
 th/users
 th/ventureone
 th/ส่วนประกอบ_widget
-th?
 th_
 th_th/templates/M
 thailandgolf.
@@ -72794,6 +72926,7 @@ themillerlab
 theming
 thenationalconversation
 thenesoyamartin
+theory/theory
 thequint/assets
 theresabyrne
 therese
@@ -72967,7 +73100,6 @@ tilda/scroll
 till
 tilleczek
 tilton
-tim
 tim-miller
 tim/index.xhtml
 timanderson
@@ -73370,7 +73502,7 @@ topicsearch?q=
 topo-summit-2019
 toprailapp
 tops
-topsecret/
+topsecret/
 topsy
 topup
 tor
@@ -73678,7 +73810,7 @@ trade_directory
 trademgr
 tradeoffer
 traffic
-traffic-manager/
+traffic-manager/
 traffic/status*?
 trafficbot.life
 trafficbot.live
@@ -73771,7 +73903,6 @@ translation
 translation-not-found
 translations
 translations/
-translator/?ref=
 translators
 transmute/static-1.85
 transmute/static-2.6
@@ -73835,18 +73966,10 @@ travel/airlines/main/au/en
 travel/airlines/main/us/en
 travel/airlines/pfs*
 travel/airlines/premium-travel-arranger/au/en
-travel/api
 travel/booking/en_gb#/*
 travel/clk
-travel/css
 travel/entity
 travel/event/search
-travel/flight-search
-travel/flight/flightSearch
-travel/flight/flightSearchAction
-travel/flight/search/?
-travel/flight/search?
-travel/flights
 travel/flights/s
 travel/globalgateway.jsp/global/public/de_
 travel/globalgateway.jsp/global/public/en_
@@ -73875,24 +73998,15 @@ travel/home/public/pt_*
 travel/home/public/ru_*
 travel/home/public/sv_*
 travel/home/public/zh_*
-travel/hotel-search
 travel/hotel/e
-travel/hotel/hotelMiniSearchRequest
-travel/hotel/hotelSearch
-travel/hotels/checkout
-travel/hotels/detail
 travel/hotels/entity
 travel/hotels/hotel-finder
 travel/hotels/hotel-finder/$
-travel/hotels/search/?
-travel/hotels/search?
 travel/hotels/stories
 travel/passport
 travel/pkomatch/
 travel/s
-travel/scripts
 travel/search
-travel/secure
 travel/test$
 travel/wp-admin
 travelDate=
@@ -73900,6 +74014,7 @@ travelmap
 travelnevadahighway95tc
 travisbenson
 travismoore
+trax
 trck
 trcz
 trde
@@ -74142,7 +74257,6 @@ tto
 ttobin
 ttong
 tucritica
-tudum
 tue
 tufengen
 tune*retpath=
@@ -74303,7 +74417,6 @@ tvmag
 tvopham
 tvschedule
 tvschedules
-tvsignup
 tvtee
 tw
 tw*/blog-article-remove-faq-utms-*.js
@@ -74793,6 +74906,7 @@ ukraina_
 ul/P/umse/0
 ulatan
 uli/trac
+ulink?
 ulk/i
 ulk/sch
 ulk/usr
@@ -74958,10 +75072,9 @@ until=
 up
 upcoming-parties
 upcomingeventdetails.html
-upcomingevents
 upcomingevents.html?
 update
-update-check/wirecast/
+update-check/wirecast/
 update-your-browser
 update.bml
 update.php
@@ -74998,6 +75111,7 @@ upload
 upload-document?
 upload-image-lens-history
 upload-lens-image
+upload.fotodom.by
 upload/
 upload/*.doc
 upload/*.pdf
@@ -75054,7 +75168,6 @@ ur/users
 urayama
 urchin_test
 url
-url
 url-shortner
 url.asp
 url=
@@ -75823,7 +75936,6 @@ utility/trac
 utility/vote
 utils
 utils/
-utils/*
 utkarsh
 utl/
 utm
@@ -75963,13 +76075,13 @@ vanessa_knutson
 vanessarodriguez
 vanilla
 vansorat
-vantage-cloud/
-vantage-enterprise/
-vantage-transcoding/
-vantageCloudManager/
-vantageCloudTrial/
-vantageDomainServer/
-vantageTrancodeServer/
+vantage-cloud/
+vantage-enterprise/
+vantage-transcoding/
+vantageCloudManager/
+vantageCloudTrial/
+vantageDomainServer/
+vantageTrancodeServer/
 vantage_point/archive
 var
 var/
@@ -76070,7 +76182,6 @@ verification/source
 verificationofemployment
 verified-modal
 verify
-verify$
 verify-email
 verify-error
 verify-mobile
@@ -76082,7 +76193,6 @@ verify/*http://soundcloud.com/robots.txt
 verify_age
 verify_your_email$
 verify_your_email?
-verifyidentity
 verizon
 verizon-connecta-la-banda*
 veronica_benavides
@@ -76297,7 +76407,7 @@ video
 video-
 video-alert
 video-c.php
-video-cue/
+video-cue/
 video-gallery/
 video-keywords
 video-landing-pages/
@@ -76433,45 +76543,28 @@ videorequest
 videos
 videos/*.feedback.html
 videos/*?
-videos/?
 videos/api
 videos/assets
-videos/browsing
 videos/e
 videos/expander
-videos/explore
-videos/favorites
-videos/feed
-videos/feedcontent
 videos/get_video_by_uid/
 videos/hardware-in-the-loop-simulation-with-simulink-real-time-using-plcs-and-ethernetip-protocol-for-validating-industrial-automation-control-algorithms-96697.html
 videos/iframe
 videos/images
 videos/js
-videos/music
-videos/music/
-videos/music/?
-videos/music?
 videos/new
 videos/raw-ethernet-communication-support-from-simulink-real-time-96619.html
 videos/real-time-udp-communication-support-99895.html
-videos/riverview/relatedvideo
-videos/riverview/relatedvideo/?
-videos/riverview/relatedvideo?
 videos/search
 videos/search.html
 videos/search.php
-videos/search/?
 videos/search?
 videos/show_trailer
 videos/slider
 videos/transcripts
-videos/trending
-videos/trendingcontent
 videos/video-feedback-confirmation.html
 videos/video-feedback.html
 videos/viewmeta/
-videos?
 videosearch
 videosearch?
 videotutorials
@@ -76518,7 +76611,6 @@ viewconfig$
 viewer
 viewfile.php
 viewfound.php
-viewingactivity
 viewlocale=zh_CN
 viewonline.php
 vieworder
@@ -76583,7 +76675,6 @@ vionnieyu
 vip
 vip_club
 viral-picks
-virginaustralia
 virginialovison
 virtual
 virtual-open-day
@@ -76716,9 +76807,7 @@ vocabolario/ricerca/*量
 vocabolario/tag
 vocabularies
 vod
-vodafone
 vodafone-foundation/focus-areas/dreamlab-app/Lesotho
-vodafonedemobilelaunch
 voguevoices/thankyou
 voguevoices/thankyou2
 voiceover
@@ -99081,7 +99170,17 @@ what-makes-up-your-digital-self
 what-now/card-details/
 what-s-cooking-with-julie-chrisley
 what-to-expect-when-you-re-expecting
+what-we-do/digitalisation/news-room
+what-we-do/digitalisation/news-room.html
 what-we-do/investment-management/private-wealth-management/intellectual-capital/isg-outlook-2013.pdf
+what-we-do/major-projects
+what-we-do/major-projects.html
+what-we-do/oil-and-natural-gas/liquefied-natural-gas-lng/lng-for-road
+what-we-do/oil-and-natural-gas/liquefied-natural-gas-lng/lng-for-road.html
+what-we-do/our-approach/powering-progress-in-action
+what-we-do/our-approach/powering-progress-in-action.html
+what-we-do/renewable-power/wind/supplier-registration
+what-we-do/renewable-power/wind/supplier-registration.html
 whatis
 whats-new
 whats-next/planes-trains-and-analytics
@@ -99116,10 +99215,13 @@ whitneypolk23
 whois/captcha.php
 whois/refresh.php
 whois/suggest.php
+whole-house-air-purifiers
 wholesale*
 wholesale-page-*.html
 wholesale.html$
 wholesalecheckout
+wholesalefuel-test-am
+wholesalefuel-test-am.html
 whsmiths
 why-2019-will-be-the-year-of-stricter-privacy-regulation
 why-am-i-seeing-this-page/
@@ -99129,19 +99231,14 @@ why-does-digital-privacy-matter-i-have-nothing-to-hide
 why-ghostery-midnight
 why-liveperson
 why-sell?inquirySource=482&
-whysecure
 whythisad
 wicket:interface*http://springerlink.com/robots.txt
 widget
 widget-landing-page.html
 widget-uploads
 widget.html
-widget/cr
-widget/entity/search/?
 widget/head/login
 widget/player
-widget/render
-widget/snapshot
 widget/v2
 widget_auth.php
 widget_chunks
@@ -100168,7 +100265,7 @@ wip
 wip_index.php
 wipostad/
 wire-
-wire-cast/
+wire-cast/
 wirecutter
 wirecutter/*&xid=
 wirecutter/*.csv$
@@ -100282,7 +100379,6 @@ wordpress/wp-admin
 wordpress/wp-admin/admin-ajax.php
 words
 work
-work
 work$
 work-at-ucl/redeployment
 work-info
@@ -100538,21 +100634,22 @@ wp-admin/admin-ajax.php
 wp-admin/admin-ajax.php?action=icl_mpp_search
 wp-admin/admin-ajax.phphttp://agrotop.co.il/robots.txt
 wp-admin/admin-ajax.phphttp://akismet.com/robots.txt
+wp-admin/admin-ajax.phphttp://aljazeera.com/robots.txt
 wp-admin/admin-ajax.phphttp://ask.com/robots.txt
 wp-admin/admin-ajax.phphttp://aweber.com/robots.txt
 wp-admin/admin-ajax.phphttp://blogs.lse.ac.uk/robots.txt
-wp-admin/admin-ajax.phphttp://buy.garmin.com/robots.txt
 wp-admin/admin-ajax.phphttp://findbyplate.com/robots.txt
 wp-admin/admin-ajax.phphttp://foodandwine.com/robots.txt
 wp-admin/admin-ajax.phphttp://goldensun.get.com.tw/robots.txt
 wp-admin/admin-ajax.phphttp://hopemoon.com/robots.txt
 wp-admin/admin-ajax.phphttp://johnallman.uk/robots.txt
 wp-admin/admin-ajax.phphttp://law.cornell.edu/robots.txt
-wp-admin/admin-ajax.phphttp://openssl.org/robots.txt
 wp-admin/admin-ajax.phphttp://publish.get.com.tw/robots.txt
+wp-admin/admin-ajax.phphttp://royalfurniture.com/robots.txt
 wp-admin/admin-ajax.phphttp://skaterki.ru/robots.txt
 wp-admin/admin-ajax.phphttp://smugmug.com/robots.txt
 wp-admin/admin-ajax.phphttp://toucharcade.com/robots.txt
+wp-admin/admin-ajax.phphttp://vox.com/robots.txt
 wp-admin/admin-ajax.phphttp://wera-heilbronn.de/robots.txt
 wp-admin/admin-ajax.phphttp://zenodo.org/robots.txt
 wp-admin/admin-ajax.phphttp://zoho.com/robots.txt
@@ -100596,7 +100693,6 @@ wp-content/compsub
 wp-content/dbsub
 wp-content/gallery
 wp-content/mu-plugins
-wp-content/mu-plugins/http://blogs.discovermagazine.com/robots.txt
 wp-content/mu_plugins
 wp-content/plugins
 wp-content/plugins
@@ -100743,7 +100839,7 @@ wp-content/uploads/woocommerce_uploads
 wp-content/uploads/woocommerce_uploads/
 wp-content/uploads/wp-import-export-lite
 wp-content/uploads/wp-import-export-lite/
-wp-content/uploads/wp-import-export-lite/http://aa.com.tr/robots.txt
+wp-content/uploads/wp-import-export-lite/http://aa.com/robots.txt
 wp-content/uploads/wp-import-export-lite/http://amnh.org/robots.txt
 wp-content/uploads/wp-import-export-lite/http://bigthink.com/robots.txt
 wp-content/uploads/wp-import-export-lite/http://realmsoftrinity.com/robots.txt
@@ -100993,7 +101089,6 @@ x?
 xArchive/Archive/Illegal-liquor-export-20010319
 xavier
 xb/en/
-xboxone
 xbp/search
 xc/
 xclick-auction
@@ -101009,7 +101104,6 @@ xd/health/services/family-medicine/clinic/nike-employee.cfm
 xdong
 xdoor
 xfilter=1
-xfinity
 xfx7
 xh/forums
 xh/kb/*/discuss
@@ -101092,7 +101186,6 @@ xorouter
 xpert/xcloud
 xqli
 xs
-xs4all
 xsanR1
 xsanR2
 xsanR3
@@ -101265,8 +101358,6 @@ your-library-invoices.asp*
 your-lyst
 your-orders
 your-shop
-yourAccount
-youraccount
 youraccount/
 yourarchives
 youreurope/businessACC
@@ -101891,7 +101982,6 @@ ziad
 zibell
 zichenhms
 zickler
-ziggo
 zihaowang
 zillowarticles
 zimmermann
@@ -102319,7 +102409,6 @@ zzz_systest
 2016/09/22/thailand-s-king-bhumibol-the-peoples-monarch-has-died
 2016/page/*/?rs=
 2016/page/11/?email=
-2016/paypal-payments.html
 2017-world-environment-day
 2017/04/03/food-money-sex-cheese-souffle-morning-sex-bf
 2017/05/07/billionaire-flood-tragedy-transformed-greenbrier-resort-blue-collar-town-depended