Merge pull request #933 from molangning/patch-5

Added a script to automatically update trickest wordlist

.bin/trickest-patcher.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python3
+
+import os,shutil
+
+print("[+] trickest wordlist patcher")
+
+ROOT=".working_space"
+INPUT_TECHNOLOGIES=os.path.join(ROOT,"wordlists/technologies/")
+INPUT_ROBOTS=os.path.join(ROOT,"wordlists/robots/")
+OUTPUT_TECHNOLOGIES="Discovery/Web-Content/CMS/trickest-cms-wordlist/"
+OUTPUT_ROBOTS="Discovery/Web-Content/trickest-robots-disallowed-wordlists/"
+
+if not os.path.isdir(".working_space"):
+    print("[!] Working dir not found!")
+    exit(2)
+
+if not os.path.isdir(os.path.join(ROOT,"wordlists")):
+    print("[!] wordlists dir not found!")
+    exit(2)
+
+if not os.path.isdir(OUTPUT_TECHNOLOGIES):
+    os.makedirs(OUTPUT_TECHNOLOGIES)
+
+if not os.path.isdir(OUTPUT_ROBOTS):
+    os.makedirs(OUTPUT_ROBOTS)
+
+for i in os.listdir(INPUT_TECHNOLOGIES):
+    path=os.path.join(INPUT_TECHNOLOGIES,i)
+
+    if os.path.isfile(path):
+        shutil.copy(path,OUTPUT_TECHNOLOGIES)
+    else:
+        shutil.copytree(path,OUTPUT_TECHNOLOGIES,dirs_exist_ok=True)
+
+for i in os.listdir(INPUT_ROBOTS):
+    path=os.path.join(INPUT_ROBOTS,i)
+
+    if os.path.isfile(path):
+        shutil.copy(path,OUTPUT_ROBOTS)
+    else:
+        shutil.copytree(path,OUTPUT_ROBOTS,dirs_exist_ok=True)
+
+print("[+] Copied all the files")
+for i in [OUTPUT_ROBOTS,OUTPUT_TECHNOLOGIES]:
+    for root,_,file_list in os.walk(i):
+        for file in file_list:
+            
+            path=os.path.join(root,file)
+            contents=open(path,"rb").read()
+
+            if contents.endswith(b"\n"):
+                print("[!] %s ends with new line"%(path))
+                contents=contents[:-1]
+                open(path,"wb").write(contents)
+
+            patch_content=[]
+            counter=0
+            for content in contents.split(b"\n"):
+                counter+=1
+                if not content:
+                    print("[+] %s has an empty line at %i"%(path,counter))
+                    continue
+                patch_content.append(content)
+                
+            if len(contents)!=len(patch_content):
+                open(path,"wb").write(b"\n".join(patch_content))
+
+
+

.bin/trickest-updater.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/bash
+
+set -e 
+set -o pipefail
+
+mkdir -p .working_space
+cd .working_space
+git clone --depth=1 https://github.com/trickest/wordlists.git
+cd ../
+
+./.bin/trickest-patcher.py
+rm -rf .working_space

.github/workflows/wordlist-updater_trickest-wordlists.yml

@@ -0,0 +1,32 @@
+name: Wordlist Updater - Trickest wordlists updater
+
+on:
+  schedule:
+  - cron: 0 0 * * *
+  
+  workflow_dispatch:
+
+jobs:
+  update-files:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Update lists
+        run: ./.bin/trickest-updater.sh
+          
+      - name: Commit files if changed
+        run: |
+          git add -N .
+
+          if [ -z "$(git ls-files --modified)" ]; then
+              echo "[+] No files were changed"
+          else
+              echo "[+] Files were changed! Pushing changed..."
+              git add -A
+              git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
+              git config --local user.email "example@github.com"
+              git config --local user.name "GitHub Action"
+              git commit -m "[Github Action] Automated trickest wordlists update."
+              git push
+          fi

.gitignore

@@ -2,3 +2,4 @@
 .*.icloud
 .gitkeep
 .idea
+.working_space/