diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index beb3c5fcab..94d8145045 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -46,10 +46,10 @@ jobs:
pip install poetry build
poetry self add "poetry-dynamic-versioning[plugin]"
- name: Build Pypi package
- if: github.ref == 'refs/heads/stable'
+ if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'
run: python -m build
- name: Publish Pypi package
- if: github.ref == 'refs/heads/stable'
+ if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'
uses: pypa/gh-action-pypi-publish@release/v1.5
with:
password: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/README.md b/README.md
index 9d62adbcdc..d7c54d8c0f 100644
--- a/README.md
+++ b/README.md
@@ -9,30 +9,32 @@ pip install bbot
[](https://www.python.org) [](https://github.com/psf/black) [](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [](https://codecov.io/gh/blacklanternsecurity/bbot)
-
+
-### **BBOT** is a **recursive**, **modular** OSINT framework inspired by Spiderfoot and written in Python.
+### **BBOT** is a **recursive**, **modular** OSINT framework inspired by Spiderfoot.
-Capable of executing the entire OSINT process in a single command, BBOT does subdomain enumeration, port scanning, web screenshots (with its `gowitness` module), vulnerability scanning (with `nuclei`), and much more.
+BBOT can execute the entire OSINT process in a single command: subdomain enumeration, port scans, web screenshots (with `gowitness`), vulnerability scanning (with `nuclei`), and much more. BBOT has over **80 modules** and counting.
-BBOT currently has over **70 modules** and counting.
-
-### [Subdomain Enumeration Face-off](https://blog.blacklanternsecurity.com/p/subdomain-enumeration-tool-face-off)
+Read our [blog post](https://blog.blacklanternsecurity.com/p/subdomain-enumeration-tool-face-off) to find out why BBOT is the most thorough subdomain enumeration tool available.
-## Installation (pip)
+## Installation ([pip](https://pypi.org/project/bbot/))
+Note: installing in a virtualenv (e.g. via `pipx`) is recommended
~~~bash
-# note: installing in a virtualenv is recommended
+# stable version
pip install bbot
+# bleeding edge (dev branch)
+pip install --pre bbot
+
bbot --help
~~~
Prerequisites:
-- Linux (Windows including WSL is not supported)
+- Linux (Windows, including WSL is not supported)
- Python 3.9 or newer
-## [Installation (Docker)](https://hub.docker.com/r/blacklanternsecurity/bbot)
+## Installation ([Docker](https://hub.docker.com/r/blacklanternsecurity/bbot))
~~~bash
# bleeding edge (dev)
docker run -it blacklanternsecurity/bbot --help
@@ -50,58 +52,71 @@ See also: [Release History](https://github.com/blacklanternsecurity/bbot/wiki/Re
## Scanning with BBOT
-Note: the `httpx` module is recommended in most scans because it is [used by BBOT to visit webpages](https://github.com/blacklanternsecurity/bbot/wiki#note-on-the-httpx-module).
-
### Examples
~~~bash
-# list modules
-bbot -l
-
-# subdomain enumeration
-bbot --flags subdomain-enum --modules httpx --targets evilcorp.com
+# subdomains
+bbot -t evilcorp.com -f subdomain-enum
-# passive modules only
-bbot --flags passive --targets evilcorp.com
+# subdomains (passive only)
+bbot -t evilcorp.com -f subdomain-enum -rf passive
-# web screenshots with gowitness
-bbot -m naabu httpx gowitness --name my_scan --output-dir . -t subdomains.txt
+# subdomains + port scan + web screenshots
+bbot -t evilcorp.com -f subdomain-enum -m naabu gowitness -n my_scan -o .
-# web scan
-bbot -f web-basic -t www.evilcorp.com
+# subdomains + basic web scan (wappalyzer, robots.txt, iis shortnames, etc.)
+bbot -t evilcorp.com -f subdomain-enum web-basic
-# web spider (search for emails, etc.)
-bbot -m httpx -c web_spider_distance=2 web_spider_depth=2 -t www.evilcorp.com
+# subdomains + web spider (search for emails, etc.)
+bbot -t evilcorp.com -f subdomain-enum -c web_spider_distance=2 web_spider_depth=2
# everything at once because yes
-bbot -f subdomain-enum email-enum cloud-enum web-basic -m naabu gowitness nuclei --allow-deadly -t evilcorp.com
+# subdomains + emails + cloud + port scan + non-intrusive web + web screenshots + nuclei
+bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m naabu gowitness nuclei --allow-deadly
+
+# list modules
+bbot -l
~~~
### Targets
-In BBOT, targets are used to seed a scan. You can specify any number of targets, and if you require more granular control over scope, you can also use whitelists and blacklists.
+Targets seed a scan with initial data. You can specify an unlimited number of targets, either directly on the command line or in files (or both!). Targets can be any of the following:
+
+- DNS_NAME (`evilcorp.com`)
+- IP_ADDRESS (`1.2.3.4`)
+- IP_RANGE (`1.2.3.0/24`)
+- URL (`https://www.evilcorp.com`)
+- EMAIL_ADDRESS (`bob@evilcorp.com`)
+
+For example, the following scan is totally valid:
~~~bash
# multiple targets
-bbot -t evilcorp.com evilcorp.co.uk 1.2.3.0/24 targets.txt
+bbot -t evilcorp.com evilcorp.co.uk http://www.evilcorp.cn 1.2.3.0/24 other_targets.txt
+~~~
+#### Whitelists / Blacklists
+
+BBOT's whitelist determines what's considered to be in-scope. By default, the whitelist is simply your target. But if you want more granular scope control, you can override it with `--whitelist` (or add a `--blacklist`).
+
+~~~bash
# seed a scan with two domains, but only consider assets to be in scope if they are inside 1.2.3.0/24
-bbot -t evilcorp.com evilcorp.co.uk --whitelist 1.2.3.0/24 --blacklist test.evilcorp.com 1.2.3.4
+bbot -t evilcorp.com evilcorp.co.uk --whitelist 1.2.3.0/24 --blacklist test.evilcorp.com 1.2.3.4 blacklist.txt
~~~
-Visit the wiki for more [tips and tricks](https://github.com/blacklanternsecurity/bbot/wiki#tips-and-tricks), including details on how BBOT handles scope, and how to tweak it if you need to.
+Visit the wiki for more [tips and tricks](https://github.com/blacklanternsecurity/bbot/wiki#tips-and-tricks).
## Using BBOT as a Python library
~~~python
from bbot.scanner import Scanner
# any number of targets can be specified
-scan = Scanner("evilcorp.com", "1.2.3.0/24", modules=["naabu"])
+scan = Scanner("evilcorp.com", "1.2.3.0/24", modules=["httpx", "sslcert"])
for event in scan.start():
- print(event)
+ print(event.json())
~~~
# Output
-BBOT can output to TXT, JSON, CSV, Neo4j, and more with `--output-module`. You can output to multiple formats simultaneously.
+By default, BBOT saves its output in TXT, JSON, and CSV formats. To enable more output modules, you can use `--output-module`.
~~~bash
# tee to a file
bbot -f subdomain-enum -t evilcorp.com | tee evilcorp.txt
@@ -109,10 +124,10 @@ bbot -f subdomain-enum -t evilcorp.com | tee evilcorp.txt
# output to JSON
bbot --output-module json -f subdomain-enum -t evilcorp.com | jq
-# output to CSV, TXT, and JSON, in current directory
-bbot -o . --output-module human csv json -f subdomain-enum -t evilcorp.com
+# output asset inventory in current directory
+bbot -o . --output-module asset_inventory -f subdomain-enum -t evilcorp.com
~~~
-For every scan, BBOT generates a unique and mildly-entertaining name like `fuzzy_gandalf`. Output for that scan, including the word cloud and any gowitness screenshots, etc., are saved to a folder by that name in `~/.bbot/scans`. The most recent 20 scans are kept, and older ones are removed. You can change the location of BBOT's output with `--output`, and you can also pick a custom scan name with `--name`.
+For every scan, BBOT generates a unique and mildly-entertaining name like `demonic_jimmy`. Output for that scan, including the word cloud and any gowitness screenshots, etc., are saved to a folder by that name in `~/.bbot/scans`. The most recent 20 scans are kept, and older ones are removed. You can change the location of BBOT's output with `--output`, and you can also pick a custom scan name with `--name`.
If you reuse a scan name, it will append to its original output files and leverage the previous word cloud.
@@ -127,16 +142,17 @@ docker run -p 7687:7687 -p 7474:7474 -v "$(pwd)/data/:/data/" --env NEO4J_AUTH=n
~~~
- After that, run bbot with `--output-modules neo4j`
~~~bash
-bbot -f subdomain-enum -t evilcorp.com --output-modules human neo4j
+bbot -f subdomain-enum -t evilcorp.com --output-modules neo4j
~~~
- Browse data at http://localhost:7474
# Usage
~~~
$ bbot --help
-usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-n SCAN_NAME] [-m MODULE [MODULE ...]] [-l] [-em MODULE [MODULE ...]]
- [-f FLAG [FLAG ...]] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [-om MODULE [MODULE ...]] [-o DIR] [-c [CONFIG ...]] [--allow-deadly] [-v] [-d] [-s] [--force] [-y] [--dry-run] [--current-config]
- [--save-wordcloud FILE] [--load-wordcloud FILE] [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [-a] [--version]
+usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-n SCAN_NAME] [-m MODULE [MODULE ...]] [-l]
+ [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [-om MODULE [MODULE ...]] [-o DIR] [-c [CONFIG ...]] [--allow-deadly] [-v] [-d] [-s]
+ [--force] [-y] [--dry-run] [--current-config] [--save-wordcloud FILE] [--load-wordcloud FILE]
+ [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [-a] [--version]
Bighuge BLS OSINT Tool
@@ -146,18 +162,18 @@ options:
-n SCAN_NAME, --name SCAN_NAME
Name of scan (default: random)
-m MODULE [MODULE ...], --modules MODULE [MODULE ...]
- Modules to enable. Choices: affiliates,anubisdb,asn,aspnet_viewstate,azure_tenant,bevigil,binaryedge,bucket_aws,bucket_azure,bucket_gcp,builtwith,bypass403,c99,censys,certspotter,cookie_brute,crobat,crt,dnscommonsrv,dnsdumpster,dnszonetransfer,emailformat,ffuf,ffuf_shortnames,fullhunt,generic_ssrf,getparam_brute,github,gowitness,hackertarget,header_brute,host_header,httpx,hunt,hunterio,iis_shortnames,ipneighbor,leakix,massdns,naabu,ntlm,nuclei,otx,passivetotal,pgp,rapiddns,riddler,securitytrails,shodan_dns,skymem,smuggler,sslcert,sublist3r,telerik,threatminer,url_manipulation,urlscan,vhost,viewdns,virustotal,wappalyzer,wayback,zoomeye
+ Modules to enable. Choices: affiliates,anubisdb,asn,azure_tenant,badsecrets,bevigil,binaryedge,bucket_aws,bucket_azure,bucket_digitalocean,bucket_gcp,builtwith,bypass403,c99,censys,certspotter,crobat,crt,dnscommonsrv,dnsdumpster,dnszonetransfer,emailformat,ffuf,ffuf_shortnames,fingerprintx,fullhunt,generic_ssrf,github,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,ipneighbor,ipstack,leakix,masscan,massdns,naabu,ntlm,nuclei,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,rapiddns,riddler,robots,securitytrails,shodan_dns,skymem,smuggler,sslcert,subdomain_hijack,sublist3r,telerik,threatminer,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye
-l, --list-modules List available modules.
-em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...]
Exclude these modules.
-f FLAG [FLAG ...], --flags FLAG [FLAG ...]
- Enable modules by flag. Choices: active,affiliates,aggressive,brute-force,cloud-enum,deadly,email-enum,iis-shortnames,passive,portscan,report,safe,slow,subdomain-enum,web-advanced,web-basic,web-paramminer,web-screenshots
+ Enable modules by flag. Choices: active,affiliates,aggressive,cloud-enum,deadly,email-enum,iis-shortnames,passive,portscan,report,safe,service-enum,slow,subdomain-enum,subdomain-hijack,web-advanced,web-basic,web-paramminer,web-screenshots
-rf FLAG [FLAG ...], --require-flags FLAG [FLAG ...]
Disable modules that don't have these flags (e.g. --require-flags passive)
-ef FLAG [FLAG ...], --exclude-flags FLAG [FLAG ...]
Disable modules with these flags. (e.g. --exclude-flags brute-force)
-om MODULE [MODULE ...], --output-modules MODULE [MODULE ...]
- Output module(s). Choices: asset_inventory,csv,http,human,json,neo4j,python,websocket
+ Output module(s). Choices: asset_inventory,csv,http,human,json,neo4j,python,web_report,websocket
-o DIR, --output-dir DIR
-c [CONFIG ...], --config [CONFIG ...]
custom config file, or configuration options in key=value format: 'modules.shodan.api_key=1234'
@@ -233,204 +249,221 @@ For explanations of config options, see `defaults.yml` or the [wiki](https://git
### Note: You can find more fun and interesting modules at the [Module Playground](https://github.com/blacklanternsecurity/bbot-module-playground). For instructions on how to install these other modules, see the [wiki](https://github.com/blacklanternsecurity/bbot/wiki#module-playground).
-To view a full list of module config options, use `--help-all`.
+To see modules' options (how to change wordlists, thread count, etc.), use `--help-all`.
~~~
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| Module | Type | Needs | Description | Flags | Produced Events |
-| | | API | | | |
-| | | Key | | | |
-+==================+==========+=========+==========================================+=========================================+==========================================+
-| aspnet_viewstate | scan | | Parse web pages for viewstates and check | active,safe,web-basic | VULNERABILITY |
-| | | | them against blacklist3r | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| bucket_aws | scan | | Check for S3 buckets related to target | active,cloud-enum,safe | STORAGE_BUCKET |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| bucket_azure | scan | | Check for Azure storage blobs related to | active,cloud-enum,safe | STORAGE_BUCKET |
-| | | | target | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| bucket_gcp | scan | | Check for Google object storage related | active,cloud-enum,safe | STORAGE_BUCKET |
-| | | | to target | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| bypass403 | scan | | Check 403 pages for common bypasses | active,aggressive,web-advanced | FINDING |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| cookie_brute | scan | | Check for common HTTP cookie parameters | active,aggressive,brute-force,slow,web- | FINDING |
-| | | | | paramminer | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| dnszonetransfer | scan | | Attempt DNS zone transfers | active,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| ffuf | scan | | A fast web fuzzer written in Go | active,aggressive,brute- | URL |
-| | | | | force,deadly,web-advanced | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| ffuf_shortnames | scan | | Use ffuf in combination IIS shortnames | active,aggressive,brute-force,iis- | URL |
-| | | | | shortnames,web-advanced | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| generic_ssrf | scan | | Check for generic SSRFs | active,aggressive,web-advanced | VULNERABILITY |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| getparam_brute | scan | | Check for common HTTP GET parameters | active,aggressive,brute-force,slow,web- | FINDING |
-| | | | | paramminer | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| gowitness | scan | | Take screenshots of webpages | active,safe,web-screenshots | SCREENSHOT |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| header_brute | scan | | Check for common HTTP header parameters | active,aggressive,brute-force,slow,web- | FINDING |
-| | | | | paramminer | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| host_header | scan | | Try common HTTP Host header spoofing | active,aggressive,web-advanced | FINDING |
-| | | | techniques | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| httpx | scan | | Visit webpages. Many other modules rely | active,safe,web-basic | HTTP_RESPONSE,URL |
-| | | | on httpx | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| hunt | scan | | Watch for commonly-exploitable HTTP | active,safe,web-advanced | FINDING |
-| | | | parameters | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| iis_shortnames | scan | | Check for IIS shortname vulnerability | active,iis-shortnames,safe,web-basic | URL_HINT |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| naabu | scan | | Execute port scans with naabu | active,aggressive,portscan | OPEN_TCP_PORT |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| ntlm | scan | | Watch for HTTP endpoints that support | active,safe,web-basic | DNS_NAME,FINDING |
-| | | | NTLM authentication | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| nuclei | scan | | Fast and customisable vulnerability | active,aggressive,deadly,web-advanced | FINDING,VULNERABILITY |
-| | | | scanner | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| smuggler | scan | | Check for HTTP smuggling | active,aggressive,brute-force,slow,web- | FINDING |
-| | | | | advanced | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| sslcert | scan | | Visit open ports and retrieve SSL | active,affiliates,email- | DNS_NAME,EMAIL_ADDRESS |
-| | | | certificates | enum,safe,subdomain-enum | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| telerik | scan | | Scan for critical Telerik | active,aggressive,slow,web-basic | FINDING,VULNERABILITY |
-| | | | vulnerabilities | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| url_manipulation | scan | | Attempt to identify URL parsing/routing | active,aggressive,web-advanced | FINDING |
-| | | | based vulnerabilities | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| vhost | scan | | Fuzz for virtual hosts | active,aggressive,brute- | DNS_NAME,VHOST |
-| | | | | force,deadly,slow,web-advanced | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| wappalyzer | scan | | Extract technologies from web responses | active,safe,web-basic | TECHNOLOGY |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| affiliates | scan | | Summarize affiliate domains at the end | passive,report,safe | |
-| | | | of a scan | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| anubisdb | scan | | Query jldc.me's database for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| asn | scan | | Query ripe and bgpview.io for ASNs | passive,report,safe,subdomain-enum | ASN |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| azure_tenant | scan | | Query Azure for tenant sister domains | affiliates,passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| bevigil | scan | X | Retrieve OSINT data from mobile | passive,safe,subdomain-enum | DNS_NAME,URL_UNVERIFIED |
-| | | | applications using BeVigil | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| binaryedge | scan | X | Query the BinaryEdge API | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| builtwith | scan | X | Query Builtwith.com for subdomains | affiliates,passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| c99 | scan | X | Query the C99 API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| censys | scan | X | Query the Censys API | email-enum,passive,safe,subdomain-enum | DNS_NAME,EMAIL_ADDRESS,IP_ADDRESS,OPEN_P |
-| | | | | | ORT,PROTOCOL |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| certspotter | scan | | Query Certspotter's API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| crobat | scan | | Query Project Crobat for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| crt | scan | | Query crt.sh (certificate transparency) | passive,safe,subdomain-enum | DNS_NAME |
-| | | | for subdomains | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| dnscommonsrv | scan | | Check for common SRV records | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| dnsdumpster | scan | | Query dnsdumpster for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| emailformat | scan | | Query email-format.com for email | email-enum,passive,safe | EMAIL_ADDRESS |
-| | | | addresses | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| fullhunt | scan | X | Query the fullhunt.io API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| github | scan | X | Query Github's API for related | passive,safe,subdomain-enum | URL_UNVERIFIED |
-| | | | repositories | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| hackertarget | scan | | Query the hackertarget.com API for | passive,safe,subdomain-enum | DNS_NAME |
-| | | | subdomains | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| hunterio | scan | X | Query hunter.io for emails | email-enum,passive,safe,subdomain-enum | DNS_NAME,EMAIL_ADDRESS,URL_UNVERIFIED |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| ipneighbor | scan | | Look beside IPs in their surrounding | aggressive,passive,subdomain-enum | IP_ADDRESS |
-| | | | subnet | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| leakix | scan | | Query leakix.net for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| massdns | scan | | Brute-force subdomains with massdns | aggressive,brute- | DNS_NAME |
-| | | | (highly effective) | force,passive,slow,subdomain-enum | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| otx | scan | | Query otx.alienvault.com for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| passivetotal | scan | X | Query the PassiveTotal API for | passive,safe,subdomain-enum | DNS_NAME |
-| | | | subdomains | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| pgp | scan | | Query common PGP servers for email | email-enum,passive,safe | EMAIL_ADDRESS |
-| | | | addresses | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| rapiddns | scan | | Query rapiddns.io for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| riddler | scan | | Query riddler.io for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| securitytrails | scan | X | Query the SecurityTrails API for | passive,safe,subdomain-enum | DNS_NAME |
-| | | | subdomains | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| shodan_dns | scan | X | Query Shodan for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| skymem | scan | | Query skymem.info for email addresses | email-enum,passive,safe | EMAIL_ADDRESS |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| sublist3r | scan | | Query sublist3r's API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| threatminer | scan | | Query threatminer's API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| urlscan | scan | | Query urlscan.io for subdomains | passive,safe,subdomain-enum | DNS_NAME,URL_UNVERIFIED |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| viewdns | scan | | Query viewdns.info's reverse whois for | affiliates,passive,safe | DNS_NAME |
-| | | | related domains | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| virustotal | scan | X | Query VirusTotal's API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| wayback | scan | | Query archive.org's API for subdomains | passive,safe,subdomain-enum | DNS_NAME,URL_UNVERIFIED |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| zoomeye | scan | X | Query ZoomEye's API for subdomains | affiliates,passive,safe,subdomain-enum | DNS_NAME |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| asset_inventory | output | | Output to an asset inventory style | | |
-| | | | flattened CSV file | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| csv | output | | Output to CSV | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| http | output | | Output to HTTP | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| human | output | | Output to text | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| json | output | | Output to JSON | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| neo4j | output | | Output to Neo4j | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| python | output | | Output via Python API | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| websocket | output | | Output to websockets | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| aggregate | internal | | Report on scan statistics | passive,safe | SUMMARY |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| excavate | internal | | Passively extract juicy tidbits from | passive | URL_UNVERIFIED |
-| | | | scan data | | |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
-| speculate | internal | | Derive certain event types from others | passive | DNS_NAME,FINDING,IP_ADDRESS,OPEN_TCP_POR |
-| | | | by common sense | | T |
-+------------------+----------+---------+------------------------------------------+-----------------------------------------+------------------------------------------+
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| Module | Type | Needs | Description | Flags | Produced Events |
+| | | API | | | |
+| | | Key | | | |
++======================+==========+=========+==========================================+========================================+==========================================+
+| badsecrets | scan | | Library for detecting known or weak | active,safe,web-basic | FINDING,VULNERABILITY |
+| | | | secrets across many web frameworks | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| bucket_aws | scan | | Check for S3 buckets related to target | active,cloud-enum,safe,web-basic | FINDING,STORAGE_BUCKET |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| bucket_azure | scan | | Check for Azure storage blobs related to | active,cloud-enum,safe,web-basic | FINDING,STORAGE_BUCKET |
+| | | | target | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| bucket_digitalocean | scan | | Check for DigitalOcean spaces related to | active,cloud-enum,safe,web-basic | FINDING,STORAGE_BUCKET |
+| | | | target | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| bucket_gcp | scan | | Check for Google object storage related | active,cloud-enum,safe,web-basic | FINDING,STORAGE_BUCKET |
+| | | | to target | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| bypass403 | scan | | Check 403 pages for common bypasses | active,aggressive | FINDING |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| dnszonetransfer | scan | | Attempt DNS zone transfers | active,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| ffuf | scan | | A fast web fuzzer written in Go | active,aggressive,deadly,web-advanced | URL_UNVERIFIED |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| ffuf_shortnames | scan | | Use ffuf in combination IIS shortnames | active,aggressive,iis-shortnames | URL_UNVERIFIED |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| fingerprintx | scan | | Fingerprint exposed services like RDP, | active,safe,service-enum,slow | PROTOCOL |
+| | | | SSH, MySQL, etc. | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| generic_ssrf | scan | | Check for generic SSRFs | active,aggressive | VULNERABILITY |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| gowitness | scan | | Take screenshots of webpages | active,safe,web-screenshots | SCREENSHOT |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| host_header | scan | | Try common HTTP Host header spoofing | active,aggressive | FINDING |
+| | | | techniques | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| httpx | scan | | Visit webpages. Many other modules rely | active,safe,subdomain-enum,web-basic | HTTP_RESPONSE,URL |
+| | | | on httpx | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| hunt | scan | | Watch for commonly-exploitable HTTP | active,safe,web-basic | FINDING |
+| | | | parameters | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| iis_shortnames | scan | | Check for IIS shortname vulnerability | active,iis-shortnames,safe,web-basic | URL_HINT |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| masscan | scan | | Port scan IP subnets with masscan | active,aggressive,portscan | OPEN_TCP_PORT |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| naabu | scan | | Execute port scans with naabu | active,aggressive,portscan | OPEN_TCP_PORT |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| ntlm | scan | | Watch for HTTP endpoints that support | active,safe,web-basic | DNS_NAME,FINDING |
+| | | | NTLM authentication | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| nuclei | scan | | Fast and customisable vulnerability | active,aggressive,deadly,web-advanced | FINDING,VULNERABILITY |
+| | | | scanner | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| paramminer_cookies | scan | | Check for common HTTP cookie parameters | active,aggressive,slow,web-paramminer | FINDING |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| paramminer_getparams | scan | | Use smart brute-force to check for | active,aggressive,slow,web-paramminer | FINDING |
+| | | | common HTTP GET parameters | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| paramminer_headers | scan | | Use smart brute-force to check for | active,aggressive,slow,web-paramminer | FINDING |
+| | | | common HTTP header parameters | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| robots | scan | | Look for and parse robots.txt | active,safe,web-basic | URL_UNVERIFIED |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| smuggler | scan | | Check for HTTP smuggling | active,aggressive,slow | FINDING |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| sslcert | scan | | Visit open ports and retrieve SSL | active,affiliates,email- | DNS_NAME,EMAIL_ADDRESS |
+| | | | certificates | enum,safe,subdomain-enum,web-basic | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| subdomain_hijack | scan | | Detect hijackable subdomains | active,cloud-enum,safe,subdomain- | FINDING |
+| | | | | enum,subdomain-hijack,web-basic | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| telerik | scan | | Scan for critical Telerik | active,aggressive,slow | FINDING,VULNERABILITY |
+| | | | vulnerabilities | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| url_manipulation | scan | | Attempt to identify URL parsing/routing | active,aggressive | FINDING |
+| | | | based vulnerabilities | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| vhost | scan | | Fuzz for virtual hosts | active,aggressive,deadly,slow,web- | DNS_NAME,VHOST |
+| | | | | advanced | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| wafw00f | scan | | Web Application Firewall Fingerprinting | active,aggressive | WAF |
+| | | | Tool | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| wappalyzer | scan | | Extract technologies from web responses | active,safe,web-basic | TECHNOLOGY |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| affiliates | scan | | Summarize affiliate domains at the end | passive,report,safe | |
+| | | | of a scan | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| anubisdb | scan | | Query jldc.me's database for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| asn | scan | | Query ripe and bgpview.io for ASNs | passive,report,safe,subdomain-enum | ASN |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| azure_tenant | scan | | Query Azure for tenant sister domains | affiliates,passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| bevigil | scan | X | Retrieve OSINT data from mobile | passive,safe,subdomain-enum | DNS_NAME,URL_UNVERIFIED |
+| | | | applications using BeVigil | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| binaryedge | scan | X | Query the BinaryEdge API | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| builtwith | scan | X | Query Builtwith.com for subdomains | affiliates,passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| c99 | scan | X | Query the C99 API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| censys | scan | X | Query the Censys API | email-enum,passive,safe,subdomain-enum | DNS_NAME,EMAIL_ADDRESS,IP_ADDRESS,OPEN_P |
+| | | | | | ORT,PROTOCOL |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| certspotter | scan | | Query Certspotter's API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| crobat | scan | | Query Project Crobat for subdomains | passive,safe | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| crt | scan | | Query crt.sh (certificate transparency) | passive,safe,subdomain-enum | DNS_NAME |
+| | | | for subdomains | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| dnscommonsrv | scan | | Check for common SRV records | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| dnsdumpster | scan | | Query dnsdumpster for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| emailformat | scan | | Query email-format.com for email | email-enum,passive,safe | EMAIL_ADDRESS |
+| | | | addresses | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| fullhunt | scan | X | Query the fullhunt.io API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| github | scan | X | Query Github's API for related | passive,safe,subdomain-enum | URL_UNVERIFIED |
+| | | | repositories | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| hackertarget | scan | | Query the hackertarget.com API for | passive,safe,subdomain-enum | DNS_NAME |
+| | | | subdomains | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| hunterio | scan | X | Query hunter.io for emails | email-enum,passive,safe,subdomain-enum | DNS_NAME,EMAIL_ADDRESS,URL_UNVERIFIED |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| ipneighbor | scan | | Look beside IPs in their surrounding | aggressive,passive,subdomain-enum | IP_ADDRESS |
+| | | | subnet | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| ipstack | scan | X | Query IPStack's API for GeoIP | passive,safe | GEOLOCATION |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| leakix | scan | | Query leakix.net for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| massdns | scan | | Brute-force subdomains with massdns | aggressive,passive,slow,subdomain-enum | DNS_NAME |
+| | | | (highly effective) | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| otx | scan | | Query otx.alienvault.com for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| passivetotal | scan | X | Query the PassiveTotal API for | passive,safe,subdomain-enum | DNS_NAME |
+| | | | subdomains | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| pgp | scan | | Query common PGP servers for email | email-enum,passive,safe | EMAIL_ADDRESS |
+| | | | addresses | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| rapiddns | scan | | Query rapiddns.io for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| riddler | scan | | Query riddler.io for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| securitytrails | scan | X | Query the SecurityTrails API for | passive,safe,subdomain-enum | DNS_NAME |
+| | | | subdomains | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| shodan_dns | scan | X | Query Shodan for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| skymem | scan | | Query skymem.info for email addresses | email-enum,passive,safe | EMAIL_ADDRESS |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| sublist3r | scan | | Query sublist3r's API for subdomains | passive,safe | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| threatminer | scan | | Query threatminer's API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| urlscan | scan | | Query urlscan.io for subdomains | passive,safe,subdomain-enum | DNS_NAME,URL_UNVERIFIED |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| viewdns | scan | | Query viewdns.info's reverse whois for | affiliates,passive,safe | DNS_NAME |
+| | | | related domains | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| virustotal | scan | X | Query VirusTotal's API for subdomains | passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| wayback | scan | | Query archive.org's API for subdomains | passive,safe,subdomain-enum | DNS_NAME,URL_UNVERIFIED |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| zoomeye | scan | X | Query ZoomEye's API for subdomains | affiliates,passive,safe,subdomain-enum | DNS_NAME |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| asset_inventory | output | | Output to an asset inventory style | | |
+| | | | flattened CSV file | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| csv | output | | Output to CSV | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| http | output | | Send every event to a custom URL via a | | |
+| | | | web request | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| human | output | | Output to text | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| json | output | | Output to JSON | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| neo4j | output | | Output to Neo4j | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| python | output | | Output via Python API | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| web_report | output | | Create a markdown report with web assets | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| websocket | output | | Output to websockets | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| aggregate | internal | | Report on scan statistics | passive,safe | SUMMARY |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| excavate | internal | | Passively extract juicy tidbits from | passive | URL_UNVERIFIED |
+| | | | scan data | | |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
+| speculate | internal | | Derive certain event types from others | passive | DNS_NAME,FINDING,IP_ADDRESS,OPEN_TCP_POR |
+| | | | by common sense | | T |
++----------------------+----------+---------+------------------------------------------+----------------------------------------+------------------------------------------+
~~~
# Credit
-BBOT is written by @TheTechromancer. Web hacking in BBOT is made possible by @liquidsec, who wrote most of the web-oriented modules and helpers.
+BBOT is written by @TheTechromancer. Web hacking in BBOT is made possible by @liquidsec, who wrote most of the web modules and helpers.
Very special thanks to the following people who made BBOT possible:
- @kerrymilan for his Neo4j and Ansible expertise
-- Steve Micallef (@smicallef) for creating Spiderfoot, by which BBOT is heavily inspired
+- Steve Micallef (@smicallef) for creating Spiderfoot
- Aleksei Kornev (@alekseiko) for allowing us ownership of the bbot Pypi repository <3
diff --git a/bbot/agent/agent.py b/bbot/agent/agent.py
index ada7e2916c..b377054fbf 100644
--- a/bbot/agent/agent.py
+++ b/bbot/agent/agent.py
@@ -60,7 +60,7 @@ def send(self, message):
break
except Exception as e:
log.warning(f"Error sending message: {e}, retrying")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
sleep(1)
continue
@@ -143,7 +143,7 @@ def stop_scan(self):
return {"success": msg, "scan_id": scan_id}
except Exception as e:
log.warning(f"Error while stopping scan: {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
finally:
self.scan = None
self.thread = None
@@ -171,15 +171,15 @@ def err_handle(callback, *args, **kwargs):
except Exception as e:
msg = f"Error in {callback.__qualname__}(): {e}"
log.error(msg)
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
return {"error": msg}
def _start_scan(self, scan):
try:
- scan.start()
+ scan.start_without_generator()
except bbot.core.errors.ScanError as e:
log.error(f"Scan error: {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
except Exception:
log.critical(f"Encountered error: {traceback.format_exc()}")
self.on_scan_status("FAILED", scan.id)
diff --git a/bbot/cli.py b/bbot/cli.py
index 58bcb30633..9c6b1c490e 100755
--- a/bbot/cli.py
+++ b/bbot/cli.py
@@ -3,6 +3,7 @@
import os
import sys
import logging
+import threading
import traceback
from omegaconf import OmegaConf
from contextlib import suppress
@@ -11,7 +12,7 @@
sys.stdout.reconfigure(line_buffering=True)
# logging
-from bbot.core.logger import init_logging, get_log_level
+from bbot.core.logger import init_logging, get_log_level, toggle_log_level
logging_queue, logging_handlers = init_logging()
@@ -20,6 +21,7 @@
from bbot.modules import module_loader
from bbot.core.configurator.args import parser
from bbot.core.helpers.logger import log_to_stderr
+from bbot.core.configurator import ensure_config_files
log = logging.getLogger("bbot.cli")
sys.stdout.reconfigure(line_buffering=True)
@@ -32,12 +34,12 @@
def main():
-
err = False
scan_name = ""
- try:
+ ensure_config_files()
+ try:
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
@@ -94,8 +96,20 @@ def main():
log.verbose(f'Enabling {m} because it has flag "{f}"')
modules.add(m)
+ default_output_modules = ["human", "json", "csv"]
+
+ # Make a list of the modules which can be output to the console
+ consoleable_output_modules = [
+ k for k, v in module_loader.preloaded(type="output").items() if "console" in v["config"]
+ ]
+
+ # If no options are specified, use the default set
if not options.output_modules:
- options.output_modules = ["human"]
+ options.output_modules = default_output_modules
+
+ # if none of the output modules provided on the command line are consoleable, don't turn off the defaults. Instead, just add the one specified to the defaults.
+ elif not any(o in consoleable_output_modules for o in options.output_modules):
+ options.output_modules += default_output_modules
scanner = Scanner(
*options.targets,
@@ -166,20 +180,23 @@ def main():
log.verbose(
f"Removing {m} because it does not have the required flags: {'+'.join(options.require_flags)}"
)
- modules.remove(m)
+ with suppress(KeyError):
+ modules.remove(m)
# excluded flags
for m in scanner._scan_modules:
flags = module_loader._preloaded.get(m, {}).get("flags", [])
if any(f in flags for f in options.exclude_flags):
log.verbose(f"Removing {m} because of excluded flag: {','.join(options.exclude_flags)}")
- modules.remove(m)
+ with suppress(KeyError):
+ modules.remove(m)
# excluded modules
for m in options.exclude_modules:
if m in modules:
log.verbose(f"Removing {m} because it is excluded")
- modules.remove(m)
+ with suppress(KeyError):
+ modules.remove(m)
scanner._scan_modules = list(modules)
log_fn = log.info
@@ -210,15 +227,42 @@ def main():
return
module_list = module_loader.filter_modules(modules=modules)
- deadly_modules = [
- m[0] for m in module_list if "deadly" in m[-1]["flags"] and m[0] in scanner._scan_modules
- ]
- if scanner._scan_modules and deadly_modules:
- if not options.allow_deadly:
+ deadly_modules = []
+ active_modules = []
+ active_aggressive_modules = []
+ slow_modules = []
+ for m in module_list:
+ if m[0] in scanner._scan_modules:
+ if "deadly" in m[-1]["flags"]:
+ deadly_modules.append(m[0])
+ if "active" in m[-1]["flags"]:
+ active_modules.append(m[0])
+ if "aggressive" in m[-1]["flags"]:
+ active_aggressive_modules.append(m[0])
+ if "slow" in m[-1]["flags"]:
+ slow_modules.append(m[0])
+ if scanner._scan_modules:
+ if deadly_modules and not options.allow_deadly:
log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}")
log.hugewarning(f"Deadly modules are highly intrusive")
log.hugewarning(f"Please specify --allow-deadly to continue")
return
+ if active_modules:
+ if active_modules:
+ if active_aggressive_modules:
+ log.hugewarning(
+ "This is an (aggressive) active scan! Intrusive connections will be made to target"
+ )
+ else:
+ log.warning(
+ "This is a (safe) active scan. Non-intrusive connections will be made to target"
+ )
+ else:
+ log.hugeinfo("This is a passive scan. No connections will be made to target")
+ if slow_modules:
+ log.hugewarning(
+ f"You have enabled the following slow modules: {','.join(slow_modules)}. Scan may take longer than usual"
+ )
scanner.helpers.word_cloud.load(options.load_wordcloud)
@@ -229,6 +273,15 @@ def main():
log.hugesuccess(f"Scan ready. Press enter to execute {scanner.name}")
input()
+ def keyboard_listen():
+ while 1:
+ keyboard_input = input()
+ if not keyboard_input:
+ toggle_log_level(logger=log)
+
+ keyboard_listen_thread = threading.Thread(target=keyboard_listen, daemon=True)
+ keyboard_listen_thread.start()
+
scanner.start_without_generator()
except bbot.core.errors.ScanError as e:
@@ -258,12 +311,12 @@ def main():
finally:
# save word cloud
- with suppress(Exception):
+ with suppress(BaseException):
save_success, filename = scanner.helpers.word_cloud.save(options.save_wordcloud)
if save_success:
log_to_stderr(f"Saved word cloud ({len(scanner.helpers.word_cloud):,} words) to {filename}")
# remove output directory if empty
- with suppress(Exception):
+ with suppress(BaseException):
scanner.home.rmdir()
if err:
os._exit(1)
diff --git a/bbot/core/configurator/__init__.py b/bbot/core/configurator/__init__.py
index 9ca76cca1e..d29d335794 100644
--- a/bbot/core/configurator/__init__.py
+++ b/bbot/core/configurator/__init__.py
@@ -11,16 +11,18 @@
# cached sudo password
bbot_sudo_pass = None
+modules_config = OmegaConf.create(
+ {
+ "modules": module_loader.configs(type="scan"),
+ "output_modules": module_loader.configs(type="output"),
+ "internal_modules": module_loader.configs(type="internal"),
+ }
+)
+
try:
config = OmegaConf.merge(
# first, pull module defaults
- OmegaConf.create(
- {
- "modules": module_loader.configs(type="scan"),
- "output_modules": module_loader.configs(type="output"),
- "internal_modules": module_loader.configs(type="internal"),
- }
- ),
+ modules_config,
# then look in .yaml files
files.get_config(),
# finally, pull from CLI arguments
@@ -33,25 +35,39 @@
config = environ.prepare_environment(config)
-# ensure bbot.yml
-if not files.config_filename.exists():
- log_to_stderr(f"Creating BBOT config at {files.config_filename}")
- no_secrets_config = OmegaConf.to_object(config)
- no_secrets_config = clean_dict(no_secrets_config, "api_key", "username", "password", "token", fuzzy=True)
- yaml = OmegaConf.to_yaml(no_secrets_config)
- yaml = "\n".join(f"# {line}" for line in yaml.splitlines())
- with open(str(files.config_filename), "w") as f:
- f.write(yaml)
-
-# ensure secrets.yml
-if not files.secrets_filename.exists():
- log_to_stderr(f"Creating BBOT secrets at {files.secrets_filename}")
- secrets_only_config = OmegaConf.to_object(config)
- secrets_only_config = filter_dict(
- secrets_only_config, "api_key", "username", "password", "token", "secret", "_id", fuzzy=True
- )
- yaml = OmegaConf.to_yaml(secrets_only_config)
- yaml = "\n".join(f"# {line}" for line in yaml.splitlines())
- with open(str(files.secrets_filename), "w") as f:
- f.write(yaml)
- files.secrets_filename.chmod(0o600)
+def ensure_config_files():
+ default_config = OmegaConf.merge(files.default_config, modules_config)
+
+ secrets_strings = ["api_key", "username", "password", "token", "secret", "_id"]
+ exclude_keys = ["modules", "output_modules", "internal_modules"]
+
+ # ensure bbot.yml
+ if not files.config_filename.exists():
+ log_to_stderr(f"Creating BBOT config at {files.config_filename}")
+ no_secrets_config = OmegaConf.to_object(default_config)
+ no_secrets_config = clean_dict(
+ no_secrets_config,
+ *secrets_strings,
+ fuzzy=True,
+ exclude_keys=exclude_keys,
+ )
+ yaml = OmegaConf.to_yaml(no_secrets_config)
+ yaml = "\n".join(f"# {line}" for line in yaml.splitlines())
+ with open(str(files.config_filename), "w") as f:
+ f.write(yaml)
+
+ # ensure secrets.yml
+ if not files.secrets_filename.exists():
+ log_to_stderr(f"Creating BBOT secrets at {files.secrets_filename}")
+ secrets_only_config = OmegaConf.to_object(default_config)
+ secrets_only_config = filter_dict(
+ secrets_only_config,
+ *secrets_strings,
+ fuzzy=True,
+ exclude_keys=exclude_keys,
+ )
+ yaml = OmegaConf.to_yaml(secrets_only_config)
+ yaml = "\n".join(f"# {line}" for line in yaml.splitlines())
+ with open(str(files.secrets_filename), "w") as f:
+ f.write(yaml)
+ files.secrets_filename.chmod(0o600)
diff --git a/bbot/core/configurator/args.py b/bbot/core/configurator/args.py
index dc21e57f19..c068dff9c8 100644
--- a/bbot/core/configurator/args.py
+++ b/bbot/core/configurator/args.py
@@ -59,30 +59,27 @@ def error(self, message):
epilog = """EXAMPLES
- list modules:
- bbot -l
-
- subdomain enumeration:
- bbot -t evilcorp.com -f subdomain-enum -m httpx
+ Subdomains:
+ bbot -t evilcorp.com -f subdomain-enum
- passive modules only:
- bbot -t evilcorp.com -f passive
+ Subdomains (passive only):
+ bbot -t evilcorp.com -f subdomain-enum -rf passive
- subdomains + web screenshots:
- bbot -t targets.txt -f subdomain-enum -m httpx gowitness --name my_scan --output-dir .
+ Subdomains + port scan + web screenshots:
+ bbot -t evilcorp.com -f subdomain-enum -m naabu gowitness -n my_scan -o .
- subdomains + basic web scanning:
+ Subdomains + basic web scan (wappalyzer, robots.txt, iis shortnames, etc.):
bbot -t evilcorp.com -f subdomain-enum web-basic
- single module:
- bbot -t evilcorp.com -m github -c modules.github.api_key=deadbeef
-
- web spider + advanced web scan:
- bbot -t www.evilcorp.com -m httpx -f web-basic web-advanced -c web_spider_distance=2 web_spider_depth=2
+ Subdomains + web spider (search for emails, etc.):
+ bbot -t evilcorp.com -f subdomain-enum -c web_spider_distance=2 web_spider_depth=2
- subdomains + emails + cloud buckets + portscan + screenshots + nuclei:
+ Subdomains + emails + cloud + port scan + non-intrusive web + web screenshots + nuclei:
bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m naabu gowitness nuclei --allow-deadly
+ List modules:
+ bbot -l
+
"""
@@ -148,7 +145,7 @@ def error(self, message):
"-om",
"--output-modules",
nargs="+",
- default=["human"],
+ default=["human", "json", "csv"],
help=f'Output module(s). Choices: {",".join(output_module_choices)}',
metavar="MODULE",
)
diff --git a/bbot/core/configurator/environ.py b/bbot/core/configurator/environ.py
index 79d99bea7d..70e8520f92 100644
--- a/bbot/core/configurator/environ.py
+++ b/bbot/core/configurator/environ.py
@@ -5,6 +5,7 @@
from . import args
from ...modules import module_loader
+from ..helpers.misc import cpu_architecture, os_platform, os_platform_friendly
def flatten_config(config, base="bbot"):
@@ -52,6 +53,11 @@ def prepare_environment(bbot_config):
bbot_lib = home / "lib"
os.environ["BBOT_LIB"] = str(bbot_lib)
+ # platform variables
+ os.environ["BBOT_OS_PLATFORM"] = os_platform()
+ os.environ["BBOT_OS"] = os_platform_friendly()
+ os.environ["BBOT_CPU_ARCH"] = cpu_architecture()
+
# exchange certain options between CLI args and config
if args.cli_options is not None:
# deps
diff --git a/bbot/core/configurator/files.py b/bbot/core/configurator/files.py
index 6a8442ac64..1d96f34ad4 100644
--- a/bbot/core/configurator/files.py
+++ b/bbot/core/configurator/files.py
@@ -10,6 +10,7 @@
mkdir(config_dir)
config_filename = (config_dir / "bbot.yml").resolve()
secrets_filename = (config_dir / "secrets.yml").resolve()
+default_config = None
def _get_config(filename, name="config", notify=True):
@@ -26,9 +27,10 @@ def _get_config(filename, name="config", notify=True):
def get_config():
-
+ global default_config
+ default_config = _get_config(defaults_filename, name="defaults")
return OmegaConf.merge(
- _get_config(defaults_filename, name="defaults"),
+ default_config,
_get_config(config_filename, name="config"),
_get_config(secrets_filename, name="secrets"),
)
diff --git a/bbot/core/errors.py b/bbot/core/errors.py
index 4b16b7ac15..fd6fa5ebb0 100644
--- a/bbot/core/errors.py
+++ b/bbot/core/errors.py
@@ -51,3 +51,7 @@ class WordlistError(BBOTError):
class DNSError(BBOTError):
pass
+
+
+class CurlError(BBOTError):
+ pass
diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index f4c5dd1c83..358decd784 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1,6 +1,7 @@
import json
import logging
import ipaddress
+import traceback
from typing import Optional
from datetime import datetime
from contextlib import suppress
@@ -22,6 +23,8 @@
smart_decode,
get_file_extension,
validators,
+ smart_decode_punycode,
+ tagify,
)
@@ -29,11 +32,12 @@
class BaseEvent:
-
+ # Always emit this event type even if it's not in scope
+ _always_emit = False
+ # Always emit events with these tags even if they're not in scope
+ _always_emit_tags = ["affiliate"]
# Exclude from output modules
_omit = False
- # Priority, 1-5, lower numbers == higher priority
- _priority = 3
# Disables certain data validations
_dummy = False
# Data validation, if data is a dictionary
@@ -53,12 +57,13 @@ def __init__(
_dummy=False,
_internal=None,
):
-
self._id = None
self._hash = None
self.__host = None
self._port = None
self.__words = None
+ self._priority = None
+ self._module_priority = None
self._resolved_hosts = set()
self._made_internal = False
@@ -67,12 +72,12 @@ def __init__(
self.timestamp = datetime.utcnow()
- if tags is None:
- tags = set()
+ self._tags = set()
+ if tags is not None:
+ self._tags = set(tagify(s) for s in tags)
self._data = None
self.type = event_type
- self.tags = set(tags)
self.confidence = int(confidence)
# for creating one-off events without enforcing source requirement
@@ -102,9 +107,7 @@ def __init__(
try:
self.data = self._sanitize_data(data)
except Exception as e:
- import traceback
-
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
raise ValidationError(f'Error sanitizing event data "{data}" for type "{self.type}": {e}')
if not self.data:
@@ -165,6 +168,13 @@ def host(self):
@property
def port(self):
self.host
+ if getattr(self, "parsed", None):
+ if self.parsed.port is not None:
+ return self.parsed.port
+ elif self.parsed.scheme == "https":
+ return 443
+ elif self.parsed.scheme == "http":
+ return 80
return self._port
@property
@@ -187,6 +197,27 @@ def words(self):
def _words(self):
return set()
+ @property
+ def tags(self):
+ return self._tags
+
+ @tags.setter
+ def tags(self, tags):
+ if isinstance(tags, str):
+ tags = (tags,)
+ self._tags = set(tagify(s) for s in tags)
+
+ def add_tag(self, tag):
+ self._tags.add(tagify(tag))
+
+ def remove_tag(self, tag):
+ with suppress(KeyError):
+ self._tags.remove(tagify(tag))
+
+ @property
+ def always_emit(self):
+ return self._always_emit or any(t in self.tags for t in self._always_emit_tags)
+
@property
def id(self):
if self._id is None:
@@ -210,8 +241,8 @@ def scope_distance(self, scope_distance):
self._scope_distance = new_scope_distance
for t in list(self.tags):
if t.startswith("distance-"):
- self.tags.remove(t)
- self.tags.add(f"distance-{new_scope_distance}")
+ self.remove_tag(t)
+ self.add_tag(f"distance-{new_scope_distance}")
@property
def source(self):
@@ -248,7 +279,7 @@ def get_source(self):
def make_internal(self):
if not self._made_internal:
self._internal = True
- self.tags.add("internal")
+ self.add_tag("internal")
self._made_internal = True
def unmake_internal(self, set_scope_distance=None, force_output=False):
@@ -257,7 +288,7 @@ def unmake_internal(self, set_scope_distance=None, force_output=False):
if set_scope_distance is not None:
self.scope_distance = set_scope_distance
self._internal = False
- self.tags.remove("internal")
+ self.remove_tag("internal")
if force_output:
self._force_output = True
self._made_internal = False
@@ -280,7 +311,7 @@ def make_in_scope(self, set_scope_distance=0):
source_trail = self.unmake_internal(set_scope_distance=set_scope_distance, force_output=True)
self.scope_distance = set_scope_distance
if set_scope_distance == 0:
- self.tags.add("in-scope")
+ self.add_tag("in-scope")
return source_trail
def _host(self):
@@ -292,6 +323,7 @@ def _sanitize_data(self, data):
if not isinstance(data, dict):
raise ValidationError(f"data is not of type dict: {data}")
data = self._data_validator(**data).dict()
+ data = {k: v for k, v in data.items() if v is not None}
return self.sanitize_data(data)
def sanitize_data(self, data):
@@ -324,18 +356,25 @@ def _data_id(self):
return self.data
@property
- def data_graph(self):
+ def pretty_string(self):
"""
Graph representation of event.data
"""
- return self._data_graph()
+ return self._pretty_string()
- def _data_graph(self):
+ def _pretty_string(self):
if isinstance(self.data, dict):
with suppress(Exception):
return json.dumps(self.data, sort_keys=True)
return smart_decode(self.data)
+ @property
+ def data_graph(self):
+ """
+ Representation of event.data for neo4j graph nodes
+ """
+ return self.pretty_string
+
@property
def data_json(self):
"""
@@ -405,12 +444,26 @@ def json(self, mode="json"):
def from_json(j):
return event_from_json(j)
+ @property
+ def module_priority(self):
+ if self._module_priority is None:
+ module = getattr(self, "module", None)
+ self._module_priority = int(max(1, min(5, getattr(module, "priority", 3))))
+ return self._module_priority
+
+ @module_priority.setter
+ def module_priority(self, priority):
+ self._module_priority = int(max(1, min(5, priority)))
+
@property
def priority(self):
- self_priority = int(max(1, min(5, self._priority)))
- mod_priority = int(max(1, min(5, getattr(self.module, "priority", 1))))
- timestamp = self.timestamp.timestamp()
- return self_priority + mod_priority + (1 / timestamp)
+ if self._priority is None:
+ timestamp = self.timestamp.timestamp()
+ if self.source.timestamp == self.timestamp:
+ self._priority = (timestamp,)
+ else:
+ self._priority = getattr(self.source, "priority", ()) + (timestamp,)
+ return self._priority
def __iter__(self):
"""
@@ -422,13 +475,13 @@ def __lt__(self, other):
"""
For queue sorting
"""
- return self.priority < int(getattr(other, "priority", 5))
+ return self.priority < getattr(other, "priority", (0,))
def __gt__(self, other):
"""
For queue sorting
"""
- return self.priority > int(getattr(other, "priority", 5))
+ return self.priority > getattr(other, "priority", (0,))
def __eq__(self, other):
try:
@@ -450,12 +503,28 @@ def __repr__(self):
return str(self)
+class FINISHED(BaseEvent):
+ """
+ Special signal event to indicate end of scan
+ """
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._priority = (999999999999999999999,)
+
+
class DefaultEvent(BaseEvent):
def sanitize_data(self, data):
return data
class DictEvent(BaseEvent):
+ def sanitize_data(self, data):
+ url = data.get("url", "")
+ if url:
+ self.parsed = validators.validate_url_parsed(url)
+ return data
+
def _data_human(self):
return json.dumps(self.data, sort_keys=True)
@@ -479,7 +548,7 @@ def _host(self):
self.parsed = validators.validate_url_parsed(self.data["url"])
return make_ip_type(self.parsed.hostname)
- def _data_graph(self):
+ def _pretty_string(self):
return self.data["url"]
@@ -487,9 +556,9 @@ class IP_ADDRESS(BaseEvent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
ip = ipaddress.ip_address(self.data)
- self.tags.add(f"ipv{ip.version}")
+ self.add_tag(f"ipv{ip.version}")
if ip.is_private:
- self.tags.add("private")
+ self.add_tag("private")
self.dns_resolve_distance = getattr(self.source, "dns_resolve_distance", 0)
def sanitize_data(self, data):
@@ -513,14 +582,14 @@ def __init__(self, *args, **kwargs):
self.dns_resolve_distance = getattr(source, "dns_resolve_distance", 0)
if source_module_type == "DNS":
self.dns_resolve_distance += 1
- # self.tags.add(f"resolve-distance-{self.dns_resolve_distance}")
+ # self.add_tag(f"resolve-distance-{self.dns_resolve_distance}")
class IP_RANGE(DnsEvent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
net = ipaddress.ip_network(self.data, strict=False)
- self.tags.add(f"ipv{net.version}")
+ self.add_tag(f"ipv{net.version}")
def sanitize_data(self, data):
return str(ipaddress.ip_network(str(data), strict=False))
@@ -530,14 +599,12 @@ def _host(self):
class DNS_NAME(DnsEvent):
- _priority = 2
-
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if is_subdomain(self.data):
- self.tags.add("subdomain")
+ self.add_tag("subdomain")
elif is_domain(self.data):
- self.tags.add("domain")
+ self.add_tag("domain")
def sanitize_data(self, data):
return validators.validate_host(data)
@@ -579,9 +646,9 @@ def sanitize_data(self, data):
# tag as dir or endpoint
if str(self.parsed.path).endswith("/"):
- self.tags.add("dir")
+ self.add_tag("dir")
else:
- self.tags.add("endpoint")
+ self.add_tag("endpoint")
parsed_path_lower = str(self.parsed.path).lower()
@@ -594,11 +661,11 @@ def sanitize_data(self, data):
extension = get_file_extension(parsed_path_lower)
if extension:
- self.tags.add(f"extension-{extension}")
+ self.add_tag(f"extension-{extension}")
if extension in url_extension_blacklist:
- self.tags.add("blacklisted")
+ self.add_tag("blacklisted")
if extension in url_extension_httpx_only:
- self.tags.add("httpx-only")
+ self.add_tag("httpx-only")
self._omit = True
data = self.parsed.geturl()
@@ -617,15 +684,6 @@ def _words(self):
def _host(self):
return make_ip_type(self.parsed.hostname)
- @property
- def port(self):
- if self.parsed.port is not None:
- return self.parsed.port
- elif self.parsed.scheme == "https":
- return 443
- elif self.parsed.scheme == "http":
- return 80
-
class URL(URL_UNVERIFIED):
def sanitize_data(self, data):
@@ -639,11 +697,13 @@ def sanitize_data(self, data):
def resolved_hosts(self):
return [i.split("-")[1] for i in self.tags if i.startswith("ip-")]
+ @property
+ def pretty_string(self):
+ return self.data
-class STORAGE_BUCKET(URL_UNVERIFIED, DictEvent):
- def sanitize_data(self, data):
- self.parsed = validators.validate_url_parsed(data["url"])
- return data
+
+class STORAGE_BUCKET(DictEvent, URL_UNVERIFIED):
+ _always_emit = True
class _data_validator(BaseModel):
name: str
@@ -671,8 +731,6 @@ def _words(self):
class HTTP_RESPONSE(URL_UNVERIFIED, DictEvent):
- _priority = 2
-
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.web_spider_distance = getattr(self.source, "web_spider_distance", 0)
@@ -699,13 +757,15 @@ def sanitize_data(self, data):
def _words(self):
return set()
+ def _pretty_string(self):
+ return f'{self.data["hash"]["header_mmh3"]}:{self.data["hash"]["body_mmh3"]}'
+
class VULNERABILITY(DictHostEvent):
- _priority = 1
+ _always_emit = True
- def _sanitize_data(self, data):
- data = super()._sanitize_data(data)
- self.tags.add(data["severity"].lower())
+ def sanitize_data(self, data):
+ self.add_tag(data["severity"].lower())
return data
class _data_validator(BaseModel):
@@ -716,12 +776,12 @@ class _data_validator(BaseModel):
_validate_host = validator("host", allow_reuse=True)(validators.validate_host)
_validate_severity = validator("severity", allow_reuse=True)(validators.validate_severity)
- def _data_graph(self):
+ def _pretty_string(self):
return f'[{self.data["severity"]}] {self.data["description"]}'
class FINDING(DictHostEvent):
- _priority = 1
+ _always_emit = True
class _data_validator(BaseModel):
host: str
@@ -729,20 +789,22 @@ class _data_validator(BaseModel):
url: Optional[str]
_validate_host = validator("host", allow_reuse=True)(validators.validate_host)
- def _data_graph(self):
+ def _pretty_string(self):
return self.data["description"]
class TECHNOLOGY(DictHostEvent):
- _priority = 2
-
class _data_validator(BaseModel):
host: str
technology: str
url: Optional[str]
_validate_host = validator("host", allow_reuse=True)(validators.validate_host)
- def _data_graph(self):
+ def _data_id(self):
+ tech = self.data.get("technology", "")
+ return f"{self.host}:{self.port}:{tech}"
+
+ def _pretty_string(self):
return self.data["technology"]
@@ -753,7 +815,7 @@ class _data_validator(BaseModel):
url: Optional[str]
_validate_host = validator("host", allow_reuse=True)(validators.validate_host)
- def _data_graph(self):
+ def _pretty_string(self):
return self.data["vhost"]
@@ -761,13 +823,21 @@ class PROTOCOL(DictHostEvent):
class _data_validator(BaseModel):
host: str
protocol: str
- _validate_host = validator("host", allow_reuse=True)(validators.validate_open_port)
+ port: Optional[int]
+ banner: Optional[str]
+ _validate_host = validator("host", allow_reuse=True)(validators.validate_host)
+ _validate_port = validator("port", allow_reuse=True)(validators.validate_port)
- def _host(self):
- host, self._port = split_host_port(self.data["host"])
- return host
+ def sanitize_data(self, data):
+ new_data = dict(data)
+ new_data["protocol"] = data.get("protocol", "").upper()
+ return new_data
+
+ @property
+ def port(self):
+ return self.data.get("port", None)
- def _data_graph(self):
+ def _pretty_string(self):
return self.data["protocol"]
@@ -802,11 +872,11 @@ def make_event(
return data
else:
if event_type is None:
+ if isinstance(data, str):
+ data = smart_decode_punycode(data)
event_type = get_event_type(data)
if not dummy:
log.debug(f'Autodetected event type "{event_type}" based on data: "{data}"')
- if event_type is None:
- raise ValidationError(f'Unable to autodetect event type from "{data}"')
event_type = str(event_type).strip().upper()
@@ -820,6 +890,7 @@ def make_event(
try:
data = validators.validate_host(data)
except Exception as e:
+ log.trace(traceback.format_exc())
raise ValidationError(f'Error sanitizing event data "{data}" for type "{event_type}": {e}')
data_is_ip = is_ip(data)
if event_type == "DNS_NAME" and data_is_ip:
diff --git a/bbot/core/event/helpers.py b/bbot/core/event/helpers.py
index 872283038e..5820183ec2 100644
--- a/bbot/core/event/helpers.py
+++ b/bbot/core/event/helpers.py
@@ -2,7 +2,8 @@
import ipaddress
from contextlib import suppress
-from bbot.core.helpers import sha1, smart_decode
+from bbot.core.errors import ValidationError
+from bbot.core.helpers import sha1, smart_decode, smart_decode_punycode
from bbot.core.helpers.regexes import event_type_regexes, event_id_regex, _hostname_regex
@@ -14,7 +15,7 @@ def get_event_type(data):
Attempt to divine event type from data
"""
- data = smart_decode(data).strip()
+ data = smart_decode_punycode(smart_decode(data).strip())
# IP address
with suppress(Exception):
@@ -38,6 +39,8 @@ def get_event_type(data):
if _hostname_regex.match(data):
return "DNS_NAME"
+ raise ValidationError(f'Unable to autodetect event type from "{data}"')
+
def is_event_id(s):
if event_id_regex.match(str(s)):
diff --git a/bbot/core/helpers/cloud/aws.py b/bbot/core/helpers/cloud/aws.py
index 4f96dccd94..e88bf57a4b 100644
--- a/bbot/core/helpers/cloud/aws.py
+++ b/bbot/core/helpers/cloud/aws.py
@@ -2,7 +2,6 @@
class AWS(BaseCloudProvider):
-
domains = [
"amazon-dss.com",
"amazonaws.com",
diff --git a/bbot/core/helpers/cloud/azure.py b/bbot/core/helpers/cloud/azure.py
index a4fd9eee2b..9f70817673 100644
--- a/bbot/core/helpers/cloud/azure.py
+++ b/bbot/core/helpers/cloud/azure.py
@@ -3,20 +3,22 @@
class Azure(BaseCloudProvider):
domains = [
- "windows.net",
- "azure.com",
"azmk8s.io",
"azure-api.net",
"azure-mobile.net",
+ "azure.com",
+ "azure.net",
"azurecontainer.io",
"azurecr.io",
+ "azuredatalakestore.net",
"azureedge.net",
"azurefd.net",
+ "azurehdinsight.net",
"azurewebsites.net",
"cloudapp.net",
+ "windows.net",
"onmicrosoft.com",
"trafficmanager.net",
- "vault.azure.net",
"visualstudio.com",
"vo.msecnd.net",
]
diff --git a/bbot/core/helpers/cloud/base.py b/bbot/core/helpers/cloud/base.py
index b818c8a4d4..0cce30028d 100644
--- a/bbot/core/helpers/cloud/base.py
+++ b/bbot/core/helpers/cloud/base.py
@@ -5,7 +5,6 @@
class BaseCloudProvider:
-
domains = []
regexes = {}
@@ -59,8 +58,19 @@ def is_valid_bucket(self, bucket_name):
return self.bucket_name_regex.match(bucket_name)
def tag_event(self, event):
- if event.host and isinstance(event.host, str):
- for r in self.domain_regexes:
- if r.match(event.host):
+ # tag the event if
+ if event.host:
+ # its host directly matches this cloud provider's domains
+ if isinstance(event.host, str) and self.domain_match(event.host):
+ event.tags.update(self.base_tags)
+ return
+ # or it has a CNAME that matches this cloud provider's domains
+ for rh in event.resolved_hosts:
+ if not self.parent_helper.is_ip(rh) and self.domain_match(rh):
event.tags.update(self.base_tags)
- break
+
+ def domain_match(self, s):
+ for r in self.domain_regexes:
+ if r.match(s):
+ return True
+ return False
diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py
index c44f62adef..c79be6e09a 100644
--- a/bbot/core/helpers/command.py
+++ b/bbot/core/helpers/command.py
@@ -34,12 +34,20 @@ def run_live(self, command, *args, **kwargs):
if not "stderr" in kwargs:
kwargs["stderr"] = subprocess.PIPE
_input = kwargs.pop("input", "")
+ sudo = kwargs.pop("sudo", False)
input_msg = ""
if _input:
kwargs["stdin"] = subprocess.PIPE
input_msg = " (with stdin)"
command = [str(s) for s in command]
+ env = kwargs.get("env", os.environ)
+ if sudo:
+ self.depsinstaller.ensure_root()
+ env["SUDO_ASKPASS"] = str((self.tools_dir / self.depsinstaller.askpass_filename).resolve())
+ env["BBOT_SUDO_PASS"] = self.depsinstaller._sudo_password
+ kwargs["env"] = env
+ command = ["sudo", "-A"] + command
log.hugeverbose(f"run_live{input_msg}: {' '.join(command)}")
try:
with catch(subprocess.Popen, command, *args, **kwargs) as process:
@@ -77,8 +85,16 @@ def run(self, command, *args, **kwargs):
kwargs["stderr"] = subprocess.PIPE
if not "text" in kwargs:
kwargs["text"] = True
+ sudo = kwargs.pop("sudo", False)
command = [str(s) for s in command]
+ env = kwargs.get("env", os.environ)
+ if sudo:
+ self.depsinstaller.ensure_root()
+ env["SUDO_ASKPASS"] = str((self.tools_dir / self.depsinstaller.askpass_filename).resolve())
+ env["BBOT_SUDO_PASS"] = self.depsinstaller._sudo_password
+ kwargs["env"] = env
+ command = ["sudo", "-A"] + command
log.hugeverbose(f"run: {' '.join(command)}")
result = catch(subprocess.run, command, *args, **kwargs)
@@ -97,15 +113,15 @@ def catch(callback, *args, **kwargs):
return callback(*args, **kwargs)
except FileNotFoundError as e:
log.warning(f"{e} - missing executable?")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
except BrokenPipeError as e:
log.warning(f"Error in subprocess: {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
def tempfile(self, content, pipe=True):
"""
- tempfile("temp\nfile\ncontent") --> Path("/home/user/.bbot/temp/pgxml13bov87oqrvjz7a")
+ tempfile(["temp", "file", "content"]) --> Path("/home/user/.bbot/temp/pgxml13bov87oqrvjz7a")
if "pipe" is True (the default), a named pipe is used instead of
a true file, which allows python data to be piped directly into the
@@ -126,7 +142,7 @@ def tempfile(self, content, pipe=True):
f.write(line)
except Exception as e:
log.error(f"Error creating temp file: {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
return filename
@@ -159,7 +175,7 @@ def _feed_pipe(self, pipe, content, text=True):
self.scan.stop()
except Exception as e:
log.error(f"Error in _feed_pipe(): {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
def feed_pipe(self, pipe, content, text=True):
@@ -179,7 +195,7 @@ def tempfile_tail(self, callback):
t.start()
except Exception as e:
log.error(f"Error setting up tail for file {filename}: {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
return
return filename
@@ -192,4 +208,4 @@ def tail(filename, callback):
callback(line)
except Exception as e:
log.error(f"Error tailing file {filename}: {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index fd3de34031..17f81e4b08 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -1,18 +1,20 @@
import os
import sys
+import stat
import json
import shutil
import getpass
import logging
from time import sleep
+from pathlib import Path
from itertools import chain
from contextlib import suppress
from ansible_runner.interface import run
from subprocess import CalledProcessError
-from bbot.modules import module_loader
-from ..misc import can_sudo_without_password
from bbot.core import configurator
+from bbot.modules import module_loader
+from ..misc import can_sudo_without_password, os_platform
log = logging.getLogger("bbot.core.helpers.depsinstaller")
@@ -25,6 +27,7 @@ def __init__(self, parent_helper):
http_timeout = self.parent_helper.config.get("http_timeout", 30)
os.environ["ANSIBLE_TIMEOUT"] = str(http_timeout)
+ self.askpass_filename = "sudo_askpass.py"
self._sudo_password = os.environ.get("BBOT_SUDO_PASS", None)
if self._sudo_password is None:
if configurator.bbot_sudo_pass is not None:
@@ -167,14 +170,16 @@ def apt_install(self, packages):
packages_str = ",".join(packages)
log.info(f"Installing the following OS packages: {packages_str}")
args = {"name": packages_str, "state": "present"} # , "update_cache": True, "cache_valid_time": 86400}
- success, err = self.ansible_run(
- module="package",
- args=args,
- ansible_args={
- "ansible_become": True,
- "ansible_become_method": "sudo",
- },
- )
+ kwargs = {}
+ # don't sudo brew
+ if os_platform() != "darwin":
+ kwargs = {
+ "ansible_args": {
+ "ansible_become": True,
+ "ansible_become_method": "sudo",
+ }
+ }
+ success, err = self.ansible_run(module="package", args=args, **kwargs)
if success:
log.info(f'Successfully installed OS packages "{packages_str}"')
else:
@@ -226,6 +231,14 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
log.debug(f"ansible_run(module={module}, args={args}, ansible_args={ansible_args})")
playbook = None
if tasks:
+ for task in tasks:
+ if "package" in task:
+ # special case for macos
+ if os_platform() == "darwin":
+ # don't sudo brew
+ task["become"] = False
+ # brew doesn't support update_cache
+ task["package"].pop("update_cache", "")
playbook = {"hosts": "all", "tasks": tasks}
log.debug(json.dumps(playbook, indent=2))
if self._sudo_password is not None:
@@ -289,11 +302,16 @@ def ensure_root(self, message=""):
log.warning("Incorrect password")
def install_core_deps(self):
+ to_install = set()
+ # install custom askpass script
+ askpass_src = Path(__file__).resolve().parent / self.askpass_filename
+ askpass_dst = self.parent_helper.tools_dir / self.askpass_filename
+ shutil.copy(askpass_src, askpass_dst)
+ askpass_dst.chmod(askpass_dst.stat().st_mode | stat.S_IEXEC)
# ensure tldextract data is cached
self.parent_helper.tldextract("evilcorp.co.uk")
# command: package_name
core_deps = {"unzip": "unzip", "curl": "curl"}
- to_install = set()
for command, package_name in core_deps.items():
if not self.parent_helper.which(command):
to_install.add(package_name)
diff --git a/bbot/core/helpers/depsinstaller/sudo_askpass.py b/bbot/core/helpers/depsinstaller/sudo_askpass.py
new file mode 100644
index 0000000000..42eccc1674
--- /dev/null
+++ b/bbot/core/helpers/depsinstaller/sudo_askpass.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+import os
+
+print(os.environ.get("BBOT_SUDO_PASS", ""), end="")
diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py
index 5b267ad646..43f668dfd8 100644
--- a/bbot/core/helpers/diff.py
+++ b/bbot/core/helpers/diff.py
@@ -10,18 +10,23 @@
class HttpCompare:
- def __init__(self, baseline_url, parent_helper, method="GET", allow_redirects=False):
-
+ def __init__(self, baseline_url, parent_helper, method="GET", allow_redirects=False, include_cache_buster=True):
self.parent_helper = parent_helper
self.baseline_url = baseline_url
+ self.include_cache_buster = include_cache_buster
# vanilla URL
- url_1 = self.parent_helper.add_get_params(self.baseline_url, self.gen_cache_buster()).geturl()
+ if self.include_cache_buster:
+ url_1 = self.parent_helper.add_get_params(self.baseline_url, self.gen_cache_buster()).geturl()
+ else:
+ url_1 = self.baseline_url
baseline_1 = self.parent_helper.request(url_1, allow_redirects=allow_redirects, method=method)
sleep(1)
# put random parameters in URL, headers, and cookies
- get_params = self.gen_cache_buster()
- get_params.update({self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)})
+ get_params = {self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}
+
+ if self.include_cache_buster:
+ get_params.update(self.gen_cache_buster())
url_2 = self.parent_helper.add_get_params(self.baseline_url, get_params).geturl()
baseline_2 = self.parent_helper.request(
url_2,
@@ -70,14 +75,13 @@ def __init__(self, baseline_url, parent_helper, method="GET", allow_redirects=Fa
]
dynamic_headers = self.compare_headers(baseline_1.headers, baseline_2.headers)
- self.baseline_ignore_headers += dynamic_headers
+ self.baseline_ignore_headers += [x.lower() for x in dynamic_headers]
self.baseline_body_distance = self.compare_body(baseline_1_json, baseline_2_json)
def gen_cache_buster(self):
return {self.parent_helper.rand_string(6): "1"}
def compare_headers(self, headers_1, headers_2):
-
differing_headers = []
for i, headers in enumerate((headers_1, headers_2)):
@@ -99,7 +103,6 @@ def compare_headers(self, headers_1, headers_2):
return differing_headers
def compare_body(self, content_1, content_2):
-
if content_1 == content_2:
return True
@@ -124,8 +127,11 @@ def compare(
"""
reflection = False
- cache_key, cache_value = list(self.gen_cache_buster().items())[0]
- url = self.parent_helper.add_get_params(subject, {cache_key: cache_value}).geturl()
+ if self.include_cache_buster:
+ cache_key, cache_value = list(self.gen_cache_buster().items())[0]
+ url = self.parent_helper.add_get_params(subject, {cache_key: cache_value}).geturl()
+ else:
+ url = subject
subject_response = self.parent_helper.request(
url, headers=headers, cookies=cookies, allow_redirects=allow_redirects, method=method
)
diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py
index 9e81307c9d..400c870c47 100644
--- a/bbot/core/helpers/dns.py
+++ b/bbot/core/helpers/dns.py
@@ -2,14 +2,16 @@
import json
import logging
import ipaddress
+import traceback
+import cloudcheck
import dns.resolver
import dns.exception
from threading import Lock
from contextlib import suppress
from concurrent.futures import ThreadPoolExecutor
+from .threadpool import NamedLock
from .regexes import dns_name_regex
-from .threadpool import ThreadPoolWrapper, NamedLock
from bbot.core.errors import ValidationError, DNSError
from .misc import is_ip, is_domain, domain_parents, parent_domain, rand_string
@@ -25,7 +27,6 @@ class DNSHelper:
all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"]
def __init__(self, parent_helper):
-
self.parent_helper = parent_helper
try:
self.resolver = dns.resolver.Resolver()
@@ -34,7 +35,7 @@ def __init__(self, parent_helper):
self.timeout = self.parent_helper.config.get("dns_timeout", 5)
self.retries = self.parent_helper.config.get("dns_retries", 1)
self.abort_threshold = self.parent_helper.config.get("dns_abort_threshold", 5)
- self.dns_resolve_distance = self.parent_helper.config.get("dns_resolve_distance", 4)
+ self.max_dns_resolve_distance = self.parent_helper.config.get("max_dns_resolve_distance", 4)
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout
self._resolver_list = None
@@ -58,8 +59,7 @@ def __init__(self, parent_helper):
# we need our own threadpool because using the shared one can lead to deadlocks
max_workers = self.parent_helper.config.get("max_dns_threads", 100)
- executor = ThreadPoolExecutor(max_workers=max_workers)
- self._thread_pool = ThreadPoolWrapper(executor, max_workers=max_workers)
+ self._thread_pool = ThreadPoolExecutor(max_workers=max_workers)
self._debug = self.parent_helper.config.get("dns_debug", False)
@@ -91,7 +91,7 @@ def resolve(self, query, **kwargs):
"""
results = set()
raw_results, errors = self.resolve_raw(query, **kwargs)
- for (rdtype, answers) in raw_results:
+ for rdtype, answers in raw_results:
for answer in answers:
for _, t in self.extract_targets(answer):
results.add(t)
@@ -134,6 +134,12 @@ def resolve_raw(self, query, **kwargs):
return (results, errors)
+ def submit_task(self, *args, **kwargs):
+ try:
+ return self._thread_pool.submit(*args, **kwargs)
+ except RuntimeError as e:
+ log.debug(f"Error submitting DNS thread task: {e}")
+
def _resolve_hostname(self, query, **kwargs):
self.debug(f"Resolving {query} with kwargs={kwargs}")
results = []
@@ -146,16 +152,16 @@ def _resolve_hostname(self, query, **kwargs):
parent_hash = hash(f"{parent}:{rdtype}")
dns_cache_hash = hash(f"{query}:{rdtype}")
while tries_left > 0:
- error_count = self._errors.get(parent_hash, 0)
- if error_count >= self.abort_threshold:
- log.verbose(
- f'Aborting query "{query}" because failed {rdtype} queries for "{parent}" ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})'
- )
- return results, errors
try:
try:
results = self._dns_cache[dns_cache_hash]
except KeyError:
+ error_count = self._errors.get(parent_hash, 0)
+ if error_count >= self.abort_threshold:
+ log.verbose(
+ f'Aborting query "{query}" because failed {rdtype} queries for "{parent}" ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})'
+ )
+ return results, errors
results = list(self._catch(self.resolver.resolve, query, **kwargs))
if cache_result:
self._dns_cache[dns_cache_hash] = results
@@ -169,10 +175,10 @@ def _resolve_hostname(self, query, **kwargs):
self._errors[parent_hash] += 1
except KeyError:
self._errors[parent_hash] = 1
- log.verbose(
- f'DNS error or timeout for {rdtype} query "{query}" ({self._errors[parent_hash]:,} so far): {e}'
- )
- errors.append(e)
+ log.verbose(
+ f'DNS error or timeout for {rdtype} query "{query}" ({self._errors[parent_hash]:,} so far): {e}'
+ )
+ errors.append(e)
# don't retry if we get a SERVFAIL
if isinstance(e, dns.resolver.NoNameservers):
break
@@ -215,60 +221,78 @@ def _resolve_ip(self, query, **kwargs):
self.debug(f"Results for {query} with kwargs={kwargs}: {results}")
return results, errors
- def resolve_event(self, event):
- result = self._resolve_event(event)
+ def resolve_event(self, event, minimal=False):
+ result = self._resolve_event(event, minimal=minimal)
# if it's a wildcard, go again with _wildcard.{domain}
if len(result) == 2:
event, wildcard_rdtypes = result
- return self._resolve_event(event, wildcard_rdtypes)
+ return self._resolve_event(event, minimal=minimal, _wildcard_rdtypes=wildcard_rdtypes)
# else we're good
else:
return result
- def _resolve_event(self, event, _wildcard_rdtypes=None):
+ def _resolve_event(self, event, minimal=False, _wildcard_rdtypes=None):
"""
Tag event with appropriate dns record types
Optionally create child events from dns resolutions
"""
event_tags = set()
- try:
- if not event.host or event.type in ("IP_RANGE",):
- return [], set(), False, False, set()
- children = []
- event_host = str(event.host)
-
- event_whitelisted = False
- event_blacklisted = False
-
- resolved_hosts = set()
-
- # wildcard check first
+ if not event.host or event.type in ("IP_RANGE",):
+ return [], set(), False, False, set()
+ children = []
+ event_host = str(event.host)
+
+ event_whitelisted = False
+ event_blacklisted = False
+
+ resolved_hosts = set()
+
+ # wildcard checks
+ if not is_ip(event.host):
+ # check if this domain is using wildcard dns
+ for hostname, wildcard_domain_rdtypes in self.is_wildcard_domain(event_host).items():
+ if wildcard_domain_rdtypes:
+ event_tags.add("wildcard-domain")
+ for rdtype, ips in wildcard_domain_rdtypes.items():
+ event_tags.add(f"{rdtype.lower()}-wildcard-domain")
+ # check if the dns name itself is a wildcard entry
if _wildcard_rdtypes is None:
wildcard_rdtypes = self.is_wildcard(event_host)
else:
wildcard_rdtypes = _wildcard_rdtypes
- for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items():
- wildcard_tag = "error"
- if is_wildcard == True:
- event_tags.add("wildcard")
- wildcard_tag = "wildcard"
- event_tags.add(f"{rdtype.lower()}-{wildcard_tag}")
-
- # lock to ensure resolution of the same host doesn't start while we're working here
- with self._event_cache_locks.get_lock(event_host):
- # try to get data from cache
- _event_tags, _event_whitelisted, _event_blacklisted, _resolved_hosts = self.event_cache_get(event_host)
- event_tags.update(_event_tags)
- # if we found it, return it
- if _event_whitelisted is not None:
- return children, event_tags, _event_whitelisted, _event_blacklisted, _resolved_hosts
-
- # then resolve
- if event.type == "DNS_NAME":
- types = "any"
- else:
- types = ("A", "AAAA")
- resolved_raw, errors = self.resolve_raw(event_host, type=types, cache_result=True)
+ for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items():
+ wildcard_tag = "error"
+ if is_wildcard == True:
+ event_tags.add("wildcard")
+ wildcard_tag = "wildcard"
+ event_tags.add(f"{rdtype.lower()}-{wildcard_tag}")
+
+ # lock to ensure resolution of the same host doesn't start while we're working here
+ with self._event_cache_locks.get_lock(event_host):
+ # try to get data from cache
+ _event_tags, _event_whitelisted, _event_blacklisted, _resolved_hosts = self.event_cache_get(event_host)
+ event_tags.update(_event_tags)
+ # if we found it, return it
+ if _event_whitelisted is not None:
+ return children, event_tags, _event_whitelisted, _event_blacklisted, _resolved_hosts
+
+ # then resolve
+ if event.type == "DNS_NAME" and not minimal:
+ types = self.all_rdtypes
+ else:
+ types = ("A", "AAAA")
+
+ futures = {}
+ for t in types:
+ future = self.submit_task(
+ self._catch_keyboardinterrupt, self.resolve_raw, event_host, type=t, cache_result=True
+ )
+ if future is None:
+ break
+ futures[future] = t
+
+ for future in self.parent_helper.as_completed(futures):
+ resolved_raw, errors = future.result()
for rdtype, e in errors:
event_tags.add(f"{rdtype.lower()}-error")
for rdtype, records in resolved_raw:
@@ -280,7 +304,7 @@ def _resolve_event(self, event, _wildcard_rdtypes=None):
for r in records:
for _, t in self.extract_targets(r):
if t:
- if rdtype in ("A", "AAAA"):
+ if rdtype in ("A", "AAAA", "CNAME"):
ip = self.parent_helper.make_ip_type(t)
with suppress(ValidationError):
@@ -296,40 +320,51 @@ def _resolve_event(self, event, _wildcard_rdtypes=None):
continue
children.append((rdtype, t))
- # if the host resolves and we haven't checked wildcards yet
- if children and _wildcard_rdtypes is None:
- # these are the rdtypes that successfully resolve
- resolved_rdtypes = set([c[0].upper() for c in children])
- # these are the rdtypes that have wildcards
- wildcard_rdtypes_set = set(wildcard_rdtypes)
- # consider the event a full wildcard if all its records are wildcards
- event_is_wildcard = all(r in wildcard_rdtypes_set for r in resolved_rdtypes)
- if event_is_wildcard and event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."):
- wildcard_parent = self.parent_helper.parent_domain(event_host)
- for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items():
- if _is_wildcard:
- wildcard_parent = _parent_domain
- break
- wildcard_data = f"_wildcard.{wildcard_parent}"
- if wildcard_data != event.data:
- log.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"')
- event.data = wildcard_data
- return (event, wildcard_rdtypes)
-
- if "resolved" not in event_tags:
- event_tags.add("unresolved")
- for ip in resolved_hosts:
- try:
- ip = ipaddress.ip_address(ip)
- if ip.is_private:
- event_tags.add("private-ip")
- except ValueError:
- continue
+ # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com)
+ if not is_ip(event.host) and children and _wildcard_rdtypes is None:
+ # these are the rdtypes that successfully resolve
+ resolved_rdtypes = set([c[0].upper() for c in children])
+ # these are the rdtypes that have wildcards
+ wildcard_rdtypes_set = set(wildcard_rdtypes)
+ # consider the event a full wildcard if all its records are wildcards
+ event_is_wildcard = all(r in wildcard_rdtypes_set for r in resolved_rdtypes)
+ if event_is_wildcard and event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."):
+ wildcard_parent = self.parent_helper.parent_domain(event_host)
+ for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items():
+ if _is_wildcard:
+ wildcard_parent = _parent_domain
+ break
+ wildcard_data = f"_wildcard.{wildcard_parent}"
+ if wildcard_data != event.data:
+ log.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"')
+ event.data = wildcard_data
+ return (event, wildcard_rdtypes)
+
+ if not self.parent_helper.in_tests:
+ ips = set()
+ if event.type == "IP_ADDRESS":
+ ips.add(event.data)
+ for rdtype, target in children:
+ if rdtype in ("A", "AAAA"):
+ ips.add(target)
+ for ip in ips:
+ provider, subnet = cloudcheck.check(ip)
+ if provider:
+ event_tags.add(f"cloud-{provider.lower()}")
+
+ if "resolved" not in event_tags:
+ event_tags.add("unresolved")
+ for ip in resolved_hosts:
+ try:
+ ip = ipaddress.ip_address(ip)
+ if ip.is_private:
+ event_tags.add("private-ip")
+ except ValueError:
+ continue
- self._event_cache[event_host] = (event_tags, event_whitelisted, event_blacklisted, resolved_hosts)
- return children, event_tags, event_whitelisted, event_blacklisted, resolved_hosts
- finally:
- event._resolved.set()
+ self._event_cache[event_host] = (event_tags, event_whitelisted, event_blacklisted, resolved_hosts)
+
+ return children, event_tags, event_whitelisted, event_blacklisted, resolved_hosts
def event_cache_get(self, host):
try:
@@ -346,7 +381,9 @@ def resolve_batch(self, queries, **kwargs):
"""
futures = dict()
for query in queries:
- future = self._thread_pool.submit_task(self._catch_keyboardinterrupt, self.resolve, query, **kwargs)
+ future = self.submit_task(self._catch_keyboardinterrupt, self.resolve, query, **kwargs)
+ if future is None:
+ break
futures[future] = query
for future in self.parent_helper.as_completed(futures):
query = futures[future]
@@ -414,106 +451,6 @@ def get_valid_resolvers(self, min_reliability=0.99):
resolver_list = self.verify_nameservers(nameservers)
return resolver_list
- @property
- def resolvers(self):
- """
- Returns set() of valid DNS servers from public-dns.info
- """
- if self._resolver_list is None:
- file_content = self.parent_helper.cache_get("resolver_list", cache_hrs=24 * 30)
- if file_content is not None:
- self._resolver_list = set([l for l in file_content.splitlines() if l])
- if not self._resolver_list:
- resolvers = self.get_valid_resolvers()
- if resolvers:
- self._resolver_list = resolvers
- self.parent_helper.cache_put("resolver_list", "\n".join(self._resolver_list))
- else:
- return set()
- return self._resolver_list
-
- @property
- def mass_resolver_file(self):
- self.resolvers
- return self.parent_helper.cache_filename("resolver_list")
-
- def verify_nameservers(self, nameservers, timeout=2):
- """Check each resolver to make sure it can actually resolve DNS names
-
- Args:
- nameservers (list): nameservers to verify
- timeout (int): timeout for dns query
- """
- log.info(f"Verifying {len(nameservers):,} public nameservers. Please be patient, this may take a while.")
- futures = []
- for nameserver in nameservers:
- # don't use the system nameservers
- if nameserver in self.system_resolvers:
- continue
- futures.append(
- self._thread_pool.submit_task(self._catch_keyboardinterrupt, self.verify_nameserver, nameserver)
- )
-
- valid_nameservers = set()
- for future in self.parent_helper.as_completed(futures):
- nameserver, error = future.result()
- if error is None:
- self.debug(f'Nameserver "{nameserver}" is valid')
- valid_nameservers.add(nameserver)
- else:
- self.debug(str(error))
- if not valid_nameservers:
- log.hugewarning(
- "Unable to reach any nameservers. Please check your internet connection and ensure that DNS is not blocked outbound."
- )
- else:
- log.info(f"Successfully verified {len(valid_nameservers):,}/{len(nameservers):,} nameservers")
-
- return valid_nameservers
-
- def verify_nameserver(self, nameserver, timeout=2):
- """Validate a nameserver by making a sample query and a garbage query
-
- Args:
- nameserver (str): nameserver to verify
- timeout (int): timeout for dns query
- """
- self.debug(f'Verifying nameserver "{nameserver}"')
- error = None
-
- resolver = dns.resolver.Resolver()
- resolver.timeout = timeout
- resolver.lifetime = timeout
- resolver.nameservers = [nameserver]
-
- # first, make sure it can resolve a valid hostname
- try:
- a_results = [str(r) for r in list(resolver.resolve("dns.google", "A"))]
- aaaa_results = [str(r) for r in list(resolver.resolve("dns.google", "AAAA"))]
- if not ("2001:4860:4860::8888" in aaaa_results and "8.8.8.8" in a_results):
- error = f"Nameserver {nameserver} failed to resolve basic query"
- except Exception:
- error = f"Nameserver {nameserver} failed to resolve basic query within {timeout} seconds"
-
- # then, make sure it isn't feeding us garbage data
- randhost = f"www-m.{rand_string(9, digits=False)}.{rand_string(10, digits=False)}.com"
- if error is None:
- try:
- a_results = list(resolver.resolve(randhost, "A"))
- error = f"Nameserver {nameserver} returned garbage data"
- except dns.exception.DNSException:
- pass
- # Garbage query to nameserver failed successfully ;)
- if error is None:
- try:
- a_results = list(resolver.resolve(randhost, "AAAA"))
- error = f"Nameserver {nameserver} returned garbage data"
- except dns.exception.DNSException:
- pass
- # Garbage query to nameserver failed successfully ;)
-
- return nameserver, error
-
def _catch(self, callback, *args, **kwargs):
try:
return callback(*args, **kwargs)
@@ -528,22 +465,24 @@ def _catch(self, callback, *args, **kwargs):
log.warning(f"Error in {callback.__qualname__}() with args={args}, kwargs={kwargs}")
return list()
- def is_wildcard(self, query, ips=None):
+ def is_wildcard(self, query, ips=None, rdtype=None):
"""
Use this method to check whether a *host* is a wildcard entry
- This can reliably tell the difference between a valid DNS record and a wildcard entry in a wildcard domain.
+ This can reliably tell the difference between a valid DNS record and a wildcard inside a wildcard domain.
If you want to know whether a domain is using wildcard DNS, use is_wildcard_domain() instead.
Returns a dictionary in the following format:
{rdtype: (is_wildcard, wildcard_parent)}
- is_wildcard("www.github.io" --> {"A": (True, "github.io"), "AAAA": (True, "github.io")})
+ is_wildcard("www.github.io") --> {"A": (True, "github.io"), "AAAA": (True, "github.io")}
Note that is_wildcard can be True, False, or None (indicating that wildcard detection was inconclusive)
"""
result = {}
+ if rdtype is None:
+ rdtype = "ANY"
query = self._clean_dns_record(query)
# skip check if it's an IP
@@ -560,107 +499,133 @@ def is_wildcard(self, query, ips=None):
parent = parent_domain(query)
parents = list(domain_parents(query))
- for rdtype in self.all_rdtypes:
- # resolve the base query
- if ips is None:
- query_ips = set()
- raw_results, errors = self.resolve_raw(query, type=rdtype, cache_result=True)
+ futures = []
+ base_query_ips = dict()
+ # if the caller hasn't already done the work of resolving the IPs
+ if ips is None:
+ # then resolve the query for all rdtypes
+ for _rdtype in self.all_rdtypes:
+ # resolve the base query
+ future = self.submit_task(
+ self._catch_keyboardinterrupt, self.resolve_raw, query, type=_rdtype, cache_result=True
+ )
+ if future is None:
+ break
+ futures.append(future)
+
+ for future in self.parent_helper.as_completed(futures):
+ raw_results, errors = future.result()
if errors and not raw_results:
- self.debug(f"Failed to resolve {query} ({rdtype}) during wildcard detection")
- result[rdtype] = (None, parent)
+ self.debug(f"Failed to resolve {query} ({_rdtype}) during wildcard detection")
+ result[_rdtype] = (None, parent)
continue
- for (rdtype, answers) in raw_results:
+ for _rdtype, answers in raw_results:
+ base_query_ips[_rdtype] = set()
for answer in answers:
for _, t in self.extract_targets(answer):
- query_ips.add(t)
- else:
- query_ips = set([self._clean_dns_record(ip) for ip in ips])
+ base_query_ips[_rdtype].add(t)
+ else:
+ # otherwise, we can skip all that
+ base_query_ips[rdtype] = set([self._clean_dns_record(ip) for ip in ips])
+ if not base_query_ips:
+ return result
+
+ # once we've resolved the base query and have IP addresses to work with
+ # we can compare the IPs to the ones we have on file for wildcards
+ # for every rdtype
+ for _rdtype in self.all_rdtypes:
+ # get the IPs from above
+ query_ips = base_query_ips.get("ANY", base_query_ips.get(_rdtype, set()))
if not query_ips:
continue
-
+ # for every parent domain, starting with the longest
for host in parents[::-1]:
host_hash = hash(host)
- # if host_hash not in self._wildcard_cache:
+ # make sure we've checked that domain for wildcards
self.is_wildcard_domain(host)
- # if we've seen this domain before
if host_hash in self._wildcard_cache:
+ # then get its IPs from our wildcard cache
wildcard_rdtypes = self._wildcard_cache[host_hash]
- # otherwise check to see if the dns name matches the wildcard IPs
- if rdtype in wildcard_rdtypes:
- wildcard_ips = wildcard_rdtypes[rdtype]
- # if the results are the same as the wildcard IPs, then ladies and gentlemen we have a wildcard
- is_wildcard = all(r in wildcard_ips for r in query_ips)
+ # then check to see if our IPs match the wildcard ones
+ if _rdtype in wildcard_rdtypes:
+ wildcard_ips = wildcard_rdtypes[_rdtype]
+ # if our IPs match the wildcard ones, then ladies and gentlemen we have a wildcard
+ is_wildcard = any(r in wildcard_ips for r in query_ips)
if is_wildcard:
- result[rdtype] = (True, host)
+ result[_rdtype] = (True, host)
break
return result
- def is_wildcard_domain(self, domain, retries=5):
+ def is_wildcard_domain(self, domain):
"""
Check whether a domain is using wildcard DNS
Returns a dictionary containing any DNS record types that are wildcards, and their associated IPs
is_wildcard_domain("github.io") --> {"A": {"1.2.3.4",}, "AAAA": {"dead::beef",}}
"""
+ wildcard_domain_results = {}
domain = self._clean_dns_record(domain)
+
# make a list of its parents
parents = list(domain_parents(domain, include_self=True))
- num_parents = len(parents)
- # and check each of them, beginning with the highest parent (e.g. evilcorp.com)
+ # and check each of them, beginning with the highest parent (i.e. the root domain)
for i, host in enumerate(parents[::-1]):
# have we checked this host before?
host_hash = hash(host)
with self._wildcard_lock.get_lock(host_hash):
# if we've seen this host before
if host_hash in self._wildcard_cache:
- # return true if it's a wildcard
- if self._wildcard_cache[host_hash]:
- return self._wildcard_cache[host_hash]
- # return false if it's not a wildcard and it's the last one we're checking
- elif i + 1 == num_parents:
- return {}
- # otherwise keep going
- else:
- continue
+ wildcard_domain_results[host] = self._wildcard_cache[host_hash]
+ continue
+
# determine if this is a wildcard domain
wildcard_futures = {}
# resolve a bunch of random subdomains of the same parent
for rdtype in self.all_rdtypes:
- wildcard_futures[rdtype] = []
+ # continue if a wildcard was already found for this rdtype
+ # if rdtype in self._wildcard_cache[host_hash]:
+ # continue
for _ in range(self.wildcard_tests):
rand_query = f"{rand_string(digits=False, length=10)}.{host}"
- future = self._thread_pool.submit_task(
- self._catch_keyboardinterrupt, self.resolve, rand_query, type=rdtype, retries=retries
+ future = self.submit_task(
+ self._catch_keyboardinterrupt,
+ self.resolve,
+ rand_query,
+ type=rdtype,
+ cache_result=False,
)
- wildcard_futures[rdtype].append(future)
+ if future is None:
+ break
+ wildcard_futures[future] = rdtype
# combine the random results
- wildcard_rdtypes = {}
- for rdtype, futures in wildcard_futures.items():
- wildcard_results = set()
- for future in self.parent_helper.as_completed(futures):
- results = future.result()
- if results:
- wildcard_results.update(results)
- if wildcard_results:
- wildcard_rdtypes[rdtype] = wildcard_results
-
- self._wildcard_cache.update({host_hash: wildcard_rdtypes})
- if wildcard_rdtypes:
- wildcard_rdtypes_str = ",".join([t.upper() for t in wildcard_rdtypes])
+ is_wildcard = False
+ wildcard_results = dict()
+ for future in self.parent_helper.as_completed(wildcard_futures):
+ results = future.result()
+ rdtype = wildcard_futures[future]
+ if results:
+ is_wildcard = True
+ if results:
+ if not rdtype in wildcard_results:
+ wildcard_results[rdtype] = set()
+ wildcard_results[rdtype].update(results)
+
+ self._wildcard_cache.update({host_hash: wildcard_results})
+ wildcard_domain_results.update({host: wildcard_results})
+ if is_wildcard:
+ wildcard_rdtypes_str = ",".join(sorted([t.upper() for t, r in wildcard_results.items() if r]))
log.info(f"Encountered domain with wildcard DNS ({wildcard_rdtypes_str}): {host}")
- return wildcard_rdtypes
- return {}
+
+ return wildcard_domain_results
def _catch_keyboardinterrupt(self, callback, *args, **kwargs):
try:
return callback(*args, **kwargs)
except Exception as e:
- import traceback
-
log.error(f"Error in {callback.__qualname__}(): {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
except KeyboardInterrupt:
if self.parent_helper.scan:
self.parent_helper.scan.stop()
diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py
index c88057036c..949f9dfe76 100644
--- a/bbot/core/helpers/helper.py
+++ b/bbot/core/helpers/helper.py
@@ -41,8 +41,6 @@ def __init__(self, config, scan=None):
self.mkdir(self.temp_dir)
self.mkdir(self.tools_dir)
self.mkdir(self.lib_dir)
- # holds requests CachedSession() objects for duration of scan
- self.cache_sessions = dict()
self._futures = set()
self._future_lock = Lock()
@@ -57,9 +55,8 @@ def __init__(self, config, scan=None):
def interactsh(self):
return Interactsh(self)
- def http_compare(self, url, allow_redirects=False):
-
- return HttpCompare(url, self, allow_redirects=allow_redirects)
+ def http_compare(self, url, allow_redirects=False, include_cache_buster=True):
+ return HttpCompare(url, self, allow_redirects=allow_redirects, include_cache_buster=include_cache_buster)
def temp_filename(self):
"""
@@ -81,13 +78,13 @@ def scan(self):
@property
def in_tests(self):
- return os.environ["BBOT_TESTING"] == "True"
+ return os.environ.get("BBOT_TESTING", "") == "True"
@staticmethod
def as_completed(*args, **kwargs):
return as_completed(*args, **kwargs)
- def _make_dummy_module(self, name, _type):
+ def _make_dummy_module(self, name, _type="scan"):
"""
Construct a dummy module, for attachment to events
"""
@@ -119,6 +116,8 @@ def __getattribute__(self, attr):
class DummyModule(BaseModule):
+ _priority = 4
+
def __init__(self, *args, **kwargs):
self._name = kwargs.pop("name")
self._type = kwargs.pop("_type")
diff --git a/bbot/core/helpers/interactsh.py b/bbot/core/helpers/interactsh.py
index 52b1573898..5d065b118e 100644
--- a/bbot/core/helpers/interactsh.py
+++ b/bbot/core/helpers/interactsh.py
@@ -29,7 +29,6 @@ def __init__(self, parent_helper):
self._thread = None
def register(self, callback=None):
-
rsa = RSA.generate(1024)
self.public_key = rsa.publickey().exportKey()
@@ -84,7 +83,6 @@ def register(self, callback=None):
return self.domain
def deregister(self):
-
if not self.server or not self.correlation_id or not self.secret:
raise InteractshError(f"Missing required information to deregister")
@@ -99,7 +97,6 @@ def deregister(self):
raise InteractshError(f"Failed to de-register with interactsh server {self.server}")
def poll(self):
-
if not self.server or not self.correlation_id or not self.secret:
raise InteractshError(f"Missing required information to poll")
@@ -116,7 +113,6 @@ def poll(self):
aes_key = r.json()["aes_key"]
for data in data_list:
-
decrypted_data = self.decrypt(aes_key, data)
yield decrypted_data
@@ -133,7 +129,7 @@ def _poll_loop(self, callback):
data_list = list(self.poll())
except InteractshError as e:
log.warning(e)
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
if not data_list:
sleep(10)
continue
diff --git a/bbot/core/helpers/logger.py b/bbot/core/helpers/logger.py
index c22c5d24c2..1c7a126a10 100644
--- a/bbot/core/helpers/logger.py
+++ b/bbot/core/helpers/logger.py
@@ -2,6 +2,7 @@
loglevel_mapping = {
"DEBUG": "DBUG",
+ "TRACE": "TRCE",
"VERBOSE": "VERB",
"HUGEVERBOSE": "VERB",
"INFO": "INFO",
@@ -15,6 +16,7 @@
}
color_mapping = {
"DEBUG": 242, # grey
+ "TRACE": 242, # red
"VERBOSE": 242, # grey
"INFO": 69, # blue
"HUGEINFO": 69, # blue
diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index e577314e29..6293a47467 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -10,6 +10,7 @@
import signal
import string
import logging
+import platform
import ipaddress
import wordninja
import subprocess as sp
@@ -25,7 +26,8 @@
from .url import * # noqa F401
from . import regexes
from .. import errors
-from .names_generator import random_name # noqa F401
+from .punycode import * # noqa F401
+from .names_generator import random_name, names, adjectives # noqa F401
log = logging.getLogger("bbot.core.helpers.misc")
@@ -332,7 +334,9 @@ def kill_children(parent_pid=None, sig=signal.SIGTERM):
try:
child.send_signal(sig)
except psutil.NoSuchProcess:
- log.debug(f"No such PID: {parent_pid}")
+ log.debug(f"No such PID: {child.pid}")
+ except psutil.AccessDenied:
+ log.debug(f"Error killing PID: {child.pid} - access denied")
def str_or_file(s):
@@ -364,8 +368,8 @@ def chain_lists(l, try_files=False, msg=None):
f_path = Path(f).resolve()
if try_files and f_path.is_file():
if msg is not None:
- msg = str(msg).format(filename=f_path)
- log.info(msg)
+ new_msg = str(msg).format(filename=f_path)
+ log.info(new_msg)
for line in str_or_file(f):
final_list[line] = None
else:
@@ -471,32 +475,42 @@ def search_format_dict(d, **kwargs):
return d
-def filter_dict(d, *key_names, fuzzy=False, invert=False):
+def filter_dict(d, *key_names, fuzzy=False, invert=False, exclude_keys=None, prev_key=None):
"""
Recursively filter a dictionary based on key names
- filter_dict({"key1": "test", "key2": "asdf"}, key_name=key2)
+ filter_dict({"key1": "test", "key2": "asdf"}, "key2")
--> {"key2": "asdf"}
"""
+ if exclude_keys is None:
+ exclude_keys = []
+ if isinstance(exclude_keys, str):
+ exclude_keys = [exclude_keys]
ret = {}
if isinstance(d, dict):
for key in d:
if key in key_names or (fuzzy and any(k in key for k in key_names)):
- ret[key] = copy.deepcopy(d[key])
+ if not prev_key in exclude_keys:
+ ret[key] = copy.deepcopy(d[key])
elif isinstance(d[key], list) or isinstance(d[key], dict):
- child = filter_dict(d[key], *key_names, fuzzy=fuzzy)
+ child = filter_dict(d[key], *key_names, fuzzy=fuzzy, prev_key=key, exclude_keys=exclude_keys)
if child:
ret[key] = child
return ret
-def clean_dict(d, *key_names, fuzzy=False):
+def clean_dict(d, *key_names, fuzzy=False, exclude_keys=None, prev_key=None):
+ if exclude_keys is None:
+ exclude_keys = []
+ if isinstance(exclude_keys, str):
+ exclude_keys = [exclude_keys]
d = copy.deepcopy(d)
if isinstance(d, dict):
for key, val in list(d.items()):
if key in key_names or (fuzzy and any(k in key for k in key_names)):
- d.pop(key)
+ if prev_key not in exclude_keys:
+ d.pop(key)
else:
- d[key] = clean_dict(val, *key_names, fuzzy=fuzzy)
+ d[key] = clean_dict(val, *key_names, fuzzy=fuzzy, prev_key=key, exclude_keys=exclude_keys)
return d
@@ -755,3 +769,46 @@ def human_timedelta(d):
if seconds:
result.append(f"{seconds:,} second" + ("s" if seconds > 1 else ""))
return ", ".join(result)
+
+
+def cpu_architecture():
+ """
+ Returns the CPU architecture, e.g. "amd64, "armv7", "arm64", etc.
+ """
+ uname = platform.uname()
+ arch = uname.machine.lower()
+ if arch.startswith("aarch"):
+ return "arm64"
+ elif arch == "x86_64":
+ return "amd64"
+ return arch
+
+
+def os_platform():
+ """
+ Returns the OS platform, e.g. "linux", "darwin", "windows", etc.
+ """
+ return platform.system().lower()
+
+
+def os_platform_friendly():
+ """
+ Returns the OS platform in a more human-friendly format, because apple is indecisive
+ """
+ p = os_platform()
+ if p == "darwin":
+ return "macOS"
+ return p
+
+
+tag_filter_regex = re.compile(r"[^a-z0-9]+")
+
+
+def tagify(s):
+ """
+ Sanitize a string into a tag-friendly format
+
+ tagify("HTTP Web Title") --> "http-web-title"
+ """
+ ret = str(s).lower()
+ return tag_filter_regex.sub("-", ret).strip("-")
diff --git a/bbot/core/helpers/modules.py b/bbot/core/helpers/modules.py
index 1df2b5c99a..ea4d3dd349 100644
--- a/bbot/core/helpers/modules.py
+++ b/bbot/core/helpers/modules.py
@@ -5,7 +5,7 @@
from omegaconf import OmegaConf
from contextlib import suppress
-from .misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table
+from .misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform
class ModuleLoader:
@@ -137,6 +137,12 @@ def preload_module(self, module_file):
# ansible playbook
elif any([target.id == "deps_ansible" for target in class_attr.targets]):
ansible_tasks = ast.literal_eval(class_attr.value)
+ for task in ansible_tasks:
+ if not "become" in task:
+ task["become"] = False
+ # don't sudo brew
+ elif os_platform() == "darwin" and ("package" in task and task.get("become", False) == True):
+ task["become"] = False
preloaded_data = {
"watched_events": watched_events,
"produced_events": produced_events,
@@ -217,6 +223,8 @@ def recommend_dependencies(self, modules):
missing_deps = {e: not self.check_dependency(e, modname, produced) for e in watched_events}
if all(missing_deps.values()):
for event_type in watched_events:
+ if event_type == "SCAN":
+ continue
choices = produced_all.get(event_type, [])
choices = set(choices)
with suppress(KeyError):
diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py
index c8aee35497..a87d26ecd2 100644
--- a/bbot/core/helpers/names_generator.py
+++ b/bbot/core/helpers/names_generator.py
@@ -3,7 +3,6 @@
adjectives = [
"abnormal",
"acrophobic",
- "adhesive",
"adorable",
"adversarial",
"affectionate",
@@ -21,6 +20,7 @@
"blazed",
"bloodshot",
"brown",
+ "carbonated",
"cheeky",
"childish",
"chiseled",
@@ -40,11 +40,13 @@
"cute",
"dark",
"dastardly",
+ "decrypted",
"deep",
"delicious",
"demonic",
- "depressed",
"depraved",
+ "depressed",
+ "deranged",
"derogatory",
"despicable",
"devilish",
@@ -54,12 +56,15 @@
"difficult",
"dilapidated",
"dismal",
+ "distilled",
"disturbed",
"dramatic",
"drunk",
"effeminate",
+ "elden",
"eldritch",
"embarrassed",
+ "encrypted",
"enigmatic",
"enlightened",
"esoteric",
@@ -98,6 +103,8 @@
"hellish",
"hideous",
"hysterical",
+ "imaginary",
+ "immense",
"immoral",
"incomprehensible",
"inebriated",
@@ -117,9 +124,11 @@
"inventive",
"irritable",
"large",
+ "liquid",
"loveable",
"lovely",
"malevolent",
+ "malfunctioning",
"malicious",
"manic",
"masochistic",
@@ -127,6 +136,7 @@
"mediocre",
"melodramatic",
"moist",
+ "molten",
"monstrous",
"muscular",
"mushy",
@@ -135,8 +145,9 @@
"nefarious",
"negligent",
"neurotic",
- "normal",
"nihilistic",
+ "normal",
+ "overattached",
"overcompensating",
"overmedicated",
"overwhelming",
@@ -159,16 +170,16 @@
"premature",
"profound",
"promiscuous",
- "psychic",
"psychedelic",
+ "psychic",
"puffy",
"pure",
"queer",
"questionable",
"rabid",
"raging",
- "raving",
"rambunctious",
+ "raving",
"reckless",
"ripped",
"sadistic",
@@ -196,17 +207,17 @@
"strained",
"strenuous",
"stricken",
+ "stubborn",
"stuffed",
"stumped",
"subtle",
- "suggestive",
- "suicidal",
"sudden",
+ "suggestive",
"sunburned",
"surreal",
"suspicious",
- "sycophantic",
"sweet",
+ "sycophantic",
"tense",
"terrible",
"terrific",
@@ -226,6 +237,7 @@
"unmedicated",
"unmelted",
"unmitigated",
+ "unrelenting",
"unrestrained",
"unworthy",
"utmost",
@@ -279,7 +291,9 @@
"audrey",
"austin",
"baggins",
+ "bailey",
"barbara",
+ "bart",
"bellatrix",
"benjamin",
"betty",
@@ -289,6 +303,7 @@
"bobby",
"bombadil",
"bonnie",
+ "bonson",
"boromir",
"bradley",
"brandon",
@@ -328,6 +343,7 @@
"daniel",
"danielle",
"danny",
+ "data",
"david",
"dawn",
"deborah",
@@ -388,6 +404,7 @@
"galadriel",
"gandalf",
"gary",
+ "geordi",
"george",
"gerald",
"gimli",
@@ -408,6 +425,7 @@
"helen",
"henry",
"hermione",
+ "homer",
"howard",
"irene",
"isaac",
@@ -424,6 +442,7 @@
"jasmine",
"jason",
"jean",
+ "jean-luc",
"jeffrey",
"jennifer",
"jeremy",
@@ -459,6 +478,7 @@
"kelly",
"kenneth",
"kenobi",
+ "kerry",
"kevin",
"kimberly",
"kyle",
@@ -479,10 +499,12 @@
"lori",
"louis",
"louise",
+ "lucius",
"luis",
"luke",
"lupin",
"madison",
+ "magnus",
"margaret",
"maria",
"marie",
@@ -500,14 +522,18 @@
"melvin",
"merry",
"michael",
+ "micheal",
"michelle",
"mildred",
+ "milhouse",
"monica",
"nancy",
"natalie",
"nathan",
"nathaniel",
"nazgul",
+ "ned",
+ "nelson",
"nicholas",
"nicole",
"noah",
@@ -542,6 +568,7 @@
"ron",
"ronald",
"rose",
+ "ross",
"roy",
"ruby",
"russell",
@@ -603,7 +630,9 @@
"wendy",
"william",
"willie",
+ "worf",
"wormtongue",
+ "xavier",
"yoda",
"zachary",
]
diff --git a/bbot/core/helpers/punycode.py b/bbot/core/helpers/punycode.py
new file mode 100644
index 0000000000..bbebbafb87
--- /dev/null
+++ b/bbot/core/helpers/punycode.py
@@ -0,0 +1,27 @@
+import idna
+from contextlib import suppress
+
+
+def smart_decode_punycode(data):
+ """
+ xn--eckwd4c7c.xn--zckzah --> ドメイン.テスト
+ """
+ if not isinstance(data, str):
+ raise ValueError(f"data must be a string, not {type(data)}")
+ if "xn--" in data:
+ with suppress(UnicodeError):
+ parts = data.split("@")
+ return "@".join(idna.decode(p) for p in parts)
+ return data
+
+
+def smart_encode_punycode(data):
+ """
+ ドメイン.テスト --> xn--eckwd4c7c.xn--zckzah
+ """
+ if not isinstance(data, str):
+ raise ValueError(f"data must be a string, not {type(data)}")
+ with suppress(UnicodeError):
+ parts = data.split("@")
+ return "@".join(idna.encode(p).decode(errors="ignore") for p in parts)
+ return data
diff --git a/bbot/core/helpers/queueing.py b/bbot/core/helpers/queueing.py
new file mode 100644
index 0000000000..3b7674544c
--- /dev/null
+++ b/bbot/core/helpers/queueing.py
@@ -0,0 +1,102 @@
+import random
+from contextlib import suppress
+from queue import PriorityQueue, Empty
+
+
+class QueuedEvent(tuple):
+ """
+ Allows sorting of tuples in outgoing PriorityQueue
+ """
+
+ def __init__(self, item):
+ self.item = item
+
+ def __gt__(self, other):
+ return self.event > other.event
+
+ def __lt__(self, other):
+ return self.event < other.event
+
+ @property
+ def event(self):
+ return self._get_event(self.item)
+
+ @staticmethod
+ def _get_event(e):
+ try:
+ return e[0]
+ except Exception:
+ return e
+
+
+class EventQueue(PriorityQueue):
+ """
+ A "meta-queue" class that includes five queues, one for each priority
+
+ Events are taken from the queues in a weighted random fashion based
+ on the priority of their parent module.
+
+ This prevents complete exclusion of lower-priority events
+
+ This queue also tracks events by module and event type for stat purposes
+ """
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.event_types = dict()
+ self.modules = dict()
+ self._queues = dict()
+ self._priorities = (1, 2, 3, 4, 5)
+ self._weights = (10, 7, 5, 3, 1)
+ for priority in self._priorities:
+ q = PriorityQueue(*args, **kwargs)
+ self._queues[priority] = q
+
+ @property
+ def events(self):
+ for q in self._queues:
+ for e in q.queue:
+ yield e.event
+
+ def _qsize(self):
+ return sum(q._qsize() for q in self._queues.values())
+
+ def empty(self):
+ return all(q.empty() for q in self._queues.values())
+
+ def _put(self, item):
+ queued_event = QueuedEvent(item)
+ q = self._queues[queued_event.event.module_priority]
+ self._increment(self.event_types, queued_event.event.type)
+ self._increment(self.modules, str(queued_event.event.module))
+ q._put(queued_event)
+
+ def _get(self):
+ # first pick a (weighted) random queue
+ priority = self._random_priority()
+ try:
+ # and get an event from it
+ queued_event = self._queues[priority]._get()
+ # if that fails
+ except IndexError:
+ # try every queue
+ queues = [_ for _ in self._queues.values() if not _.empty()]
+ if not queues:
+ raise Empty
+ queued_event = queues[0]._get()
+ self._decrement(self.event_types, queued_event.event.type)
+ self._decrement(self.modules, str(queued_event.event.module))
+ return queued_event.item
+
+ def _random_priority(self):
+ return random.choices(self._priorities, weights=self._weights, k=1)[0]
+
+ def _increment(self, d, v):
+ try:
+ d[v] += 1
+ except KeyError:
+ d[v] = 1
+
+ def _decrement(self, d, v):
+ with suppress(KeyError):
+ d[v] = max(0, d[v] - 1)
diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py
index 1f1d277b24..716374f8cb 100644
--- a/bbot/core/helpers/regexes.py
+++ b/bbot/core/helpers/regexes.py
@@ -16,15 +16,21 @@
]
]
+# Designed to remove junk such as these from the beginning of a search string:
+# \nhttps://www.google.com
+# \x3dhttps://www.google.com
+# %a2https://www.google.com
+# \uac20https://www.google.com
+junk_remover = r"(?:\\x[a-fA-F0-9]{2}|%[a-fA-F0-9]{2}|\\u[a-fA-F0-9]{4}|\\[a-zA-Z])?"
word_regex = re.compile(r"[^\d\W_]+")
word_num_regex = re.compile(r"[^\W_]+")
num_regex = re.compile(r"\d+")
_ipv6_regex = r"[A-F0-9:]*:[A-F0-9:]*:[A-F0-9:]*"
ipv6_regex = re.compile(_ipv6_regex, re.I)
-_dns_name_regex = r"(?:(?:[\w-]+)\.)+(?:[a-z0-9]{2,20})"
+_dns_name_regex = r"(?:(?:[\w-]+)\.)+(?:[^\W_0-9]{2,20})"
_hostname_regex = re.compile(r"^[\w-]+$")
-_email_regex = r"(?:[a-zA-Z0-9][\w\-\.\+]{,100})@(?:[a-zA-Z0-9_][\w\-\._]{,100})\.(?:[a-zA-Z]{2,8})"
+_email_regex = r"(?:[^\W_][\w\-\.\+]{,100})@(?:\w[\w\-\._]{,100})\.(?:[^\W_0-9]{2,8})"
email_regex = re.compile(_email_regex, re.I)
event_type_regexes = OrderedDict(
@@ -42,14 +48,14 @@
(
"OPEN_TCP_PORT",
(
- r"^((?:[A-Z0-9_]|[A-Z0-9_][A-Z0-9\-_]*[A-Z0-9_])[\.]?)+(?:[A-Z0-9_][A-Z0-9\-_]*[A-Z0-9_]|[A-Z0-9_]):[0-9]{1,5}$",
+ r"^((?:\w|\w[\w\-]*\w)[\.]?)+(?:\w[\w\-]*\w|\w):[0-9]{1,5}$",
r"^\[" + _ipv6_regex + r"\]:[0-9]{1,5}$",
),
),
(
"URL",
(
- r"https?://((?:[A-Z0-9_]|[A-Z0-9_][A-Z0-9\-_]*[A-Z0-9_])[\.]?)+(?:[A-Z0-9_][A-Z0-9\-_]*[A-Z0-9_]|[A-Z0-9_])(?::[0-9]{1,5})?.*$",
+ r"https?://((?:\w|\w[\w\-]*\w)[\.]?)+(?:\w[\w\-]*\w|\w)(?::[0-9]{1,5})?.*$",
r"https?://\[" + _ipv6_regex + r"\](?::[0-9]{1,5})?.*$",
),
),
diff --git a/bbot/core/helpers/threadpool.py b/bbot/core/helpers/threadpool.py
index dba634c90a..73a62a2363 100644
--- a/bbot/core/helpers/threadpool.py
+++ b/bbot/core/helpers/threadpool.py
@@ -1,6 +1,9 @@
import logging
import threading
-from time import sleep
+import traceback
+from datetime import datetime
+from queue import SimpleQueue, Full
+from concurrent.futures import ThreadPoolExecutor
log = logging.getLogger("bbot.core.helpers.threadpool")
@@ -8,44 +11,153 @@
from ...core.errors import ScanCancelledError
+def pretty_fn(a):
+ if callable(a):
+ return a.__qualname__
+ return a
+
+
+class ThreadPoolSimpleQueue(SimpleQueue):
+ def __init__(self, *args, **kwargs):
+ self._executor = kwargs.pop("_executor", None)
+
+ def get(self, *args, **kwargs):
+ work_item = super().get(*args, **kwargs)
+ thread_id = threading.get_ident()
+ self._executor._current_work_items[thread_id] = (work_item, datetime.now())
+ return work_item
+
+
+class BBOTThreadPoolExecutor(ThreadPoolExecutor):
+ """
+ Allows inspection of thread pool to determine which functions are currently executing
+ """
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._current_work_items = {}
+ self._work_queue = ThreadPoolSimpleQueue(_executor=self)
+
+ @property
+ def threads_status(self):
+ work_items = []
+ for thread_id, (work_item, start_time) in sorted(self._current_work_items.items()):
+ func = work_item.fn.__qualname__
+ func_index = 0
+ if work_item and not work_item.future.done():
+ for i, f in enumerate(list(work_item.args)):
+ if callable(f):
+ func = f.__qualname__
+ func_index = i + 1
+ else:
+ break
+ running_for = datetime.now() - start_time
+ wi_args = list(work_item.args)[func_index:]
+ wi_args = [pretty_fn(a) for a in wi_args]
+ wi_args = str(wi_args).strip("[]")
+ wi_kwargs = ", ".join(["{0}={1}".format(k, pretty_fn(v)) for k, v in work_item.kwargs.items()])
+ func_with_args = f"{func}({wi_args}" + (f", {wi_kwargs}" if wi_kwargs else "") + ")"
+ work_items.append(
+ (running_for, f"running for {int(running_for.total_seconds()):>3} seconds: {func_with_args}")
+ )
+ work_items.sort(key=lambda x: x[0])
+ return [x[-1] for x in work_items]
+
+
class ThreadPoolWrapper:
"""
Layers more granular control overtop of a shared thread pool
Allows setting lower thread limits for modules, etc.
"""
- def __init__(self, executor, max_workers=None):
+ def __init__(self, executor, max_workers=None, qsize=None):
self.executor = executor
self.max_workers = max_workers
+ self.max_qsize = qsize
self.futures = set()
- self._future_lock = threading.Lock()
- self._submit_task_lock = threading.Lock()
+ try:
+ self.executor._thread_pool_wrappers.append(self)
+ except AttributeError:
+ self.executor._thread_pool_wrappers = [self]
+
+ self._num_tasks = 0
+ self._task_count_lock = threading.Lock()
+
+ self._lock = threading.RLock()
+ self.not_full = threading.Condition(self._lock)
def submit_task(self, callback, *args, **kwargs):
"""
A wrapper around threadpool.submit()
-
- This blocks, which isn't ideal, but it ensures that modules don't hog the shared thread pool
"""
- with self._submit_task_lock:
- if self.max_workers is not None:
- while self.num_tasks > self.max_workers:
- sleep(0.1)
+ block = kwargs.get("_block", True)
+ force = kwargs.get("_force_submit", False)
+ success = False
+ with self.not_full:
+ self.num_tasks_increment()
try:
- future = self.executor.submit(callback, *args, **kwargs)
- except RuntimeError as e:
- raise ScanCancelledError(e)
- with self._future_lock:
- self.futures.add(future)
- return future
+ if not force:
+ if not block:
+ if self.is_full or self.underlying_executor_is_full:
+ raise Full
+ else:
+ # wait until there's room
+ while self.is_full or self.underlying_executor_is_full:
+ self.not_full.wait()
+
+ try:
+ # submit the job
+ future = self.executor.submit(self._execute_callback, callback, *args, **kwargs)
+ future.add_done_callback(self._on_future_done)
+ success = True
+ return future
+ except RuntimeError as e:
+ raise ScanCancelledError(e)
+ finally:
+ if not success:
+ self.num_tasks_decrement()
+
+ def _execute_callback(self, callback, *args, **kwargs):
+ try:
+ return callback(*args, **kwargs)
+ finally:
+ self.num_tasks_decrement()
+
+ def _on_future_done(self, future):
+ if future.cancelled():
+ self.num_tasks_decrement()
@property
def num_tasks(self):
- with self._future_lock:
- for f in list(self.futures):
- if f.done():
- self.futures.remove(f)
- return len(self.futures) + (1 if self._submit_task_lock.locked() else 0)
+ with self._task_count_lock:
+ return self._num_tasks
+
+ def num_tasks_increment(self):
+ with self._task_count_lock:
+ self._num_tasks += 1
+
+ def num_tasks_decrement(self):
+ with self._task_count_lock:
+ self._num_tasks = max(0, self._num_tasks - 1)
+ for wrapper in self.executor._thread_pool_wrappers:
+ try:
+ with wrapper.not_full:
+ wrapper.not_full.notify()
+ except RuntimeError:
+ continue
+ except Exception as e:
+ log.warning(f"Unknown error in num_tasks_decrement(): {e}")
+ log.trace(traceback.format_exc())
+
+ @property
+ def is_full(self):
+ if self.max_workers is None:
+ return False
+ return self.num_tasks > self.max_workers
+
+ @property
+ def underlying_executor_is_full(self):
+ return self.max_qsize is not None and self.qsize >= self.max_qsize
@property
def qsize(self):
@@ -54,21 +166,63 @@ def qsize(self):
def shutdown(self, *args, **kwargs):
self.executor.shutdown(*args, **kwargs)
+ @property
+ def threads_status(self):
+ return self.executor.threads_status
+
+
+import time
+from concurrent.futures._base import (
+ FINISHED,
+ _AS_COMPLETED,
+ _AcquireFutures,
+ _create_and_install_waiters,
+ _yield_finished_futures,
+)
+
+
+def as_completed(fs, timeout=None):
+ """
+ Copied from https://github.com/python/cpython/blob/main/Lib/concurrent/futures/_base.py
+ Modified to only yield FINISHED futures (not CANCELLED_AND_NOTIFIED)
+ """
+ if timeout is not None:
+ end_time = timeout + time.monotonic()
+
+ fs = set(fs)
+ total_futures = len(fs)
+ with _AcquireFutures(fs):
+ finished = set(f for f in fs if f._state == FINISHED)
+ pending = fs - finished
+ waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
+ finished = list(finished)
+ try:
+ yield from _yield_finished_futures(finished, waiter, ref_collect=(fs,))
+
+ while pending:
+ if timeout is None:
+ wait_timeout = None
+ else:
+ wait_timeout = end_time - time.monotonic()
+ if wait_timeout < 0:
+ raise TimeoutError("%d (of %d) futures unfinished" % (len(pending), total_futures))
+
+ waiter.event.wait(wait_timeout)
+
+ with waiter.lock:
+ finished = waiter.finished_futures
+ waiter.finished_futures = []
+ waiter.event.clear()
+
+ # reverse to keep finishing order
+ finished.reverse()
+ yield from _yield_finished_futures(finished, waiter, ref_collect=(fs, pending))
-def as_completed(fs):
- fs = list(fs)
- while fs:
- result = False
- for i, f in enumerate(fs):
- if f.done():
- result = True
- future = fs.pop(i)
- if future._state in ("CANCELLED", "CANCELLED_AND_NOTIFIED"):
- continue
- yield future
- break
- if not result:
- sleep(0.05)
+ finally:
+ # Remove waiter from unfinished futures
+ for f in fs:
+ with f._condition:
+ f._waiters.remove(waiter)
class _Lock:
diff --git a/bbot/core/helpers/url.py b/bbot/core/helpers/url.py
index 8fc0b3c40f..a6c5a7aa7e 100644
--- a/bbot/core/helpers/url.py
+++ b/bbot/core/helpers/url.py
@@ -4,6 +4,8 @@
from contextlib import suppress
from urllib.parse import urlparse, parse_qs, urlencode, ParseResult
+from .punycode import smart_decode_punycode
+
log = logging.getLogger("bbot.core.helpers.url")
@@ -76,6 +78,8 @@ def clean_url(url):
# special case for IPv6 URLs
if parsed.netloc.startswith("["):
hostname = f"[{hostname}]"
+ # punycode
+ hostname = smart_decode_punycode(hostname)
parsed = parsed._replace(netloc=hostname)
# normalize double slashes
parsed = parsed._replace(path=double_slash_regex.sub("/", parsed.path))
diff --git a/bbot/core/helpers/validators.py b/bbot/core/helpers/validators.py
index 04eaa653c8..587178cd66 100644
--- a/bbot/core/helpers/validators.py
+++ b/bbot/core/helpers/validators.py
@@ -3,6 +3,7 @@
from bbot.core.helpers import regexes
from bbot.core.helpers.url import clean_url
+from bbot.core.helpers.punycode import smart_decode_punycode
from bbot.core.helpers.misc import split_host_port, make_netloc
log = logging.getLogger("bbot.core.helpers.")
@@ -55,7 +56,7 @@ def validate_host(host):
return str(ip)
except Exception:
# finally, try DNS_NAME
- host = host.lstrip("*.")
+ host = smart_decode_punycode(host.lstrip("*."))
for r in regexes.event_type_regexes["DNS_NAME"]:
if r.match(host):
return host
@@ -87,7 +88,7 @@ def validate_severity(severity):
@validator
def validate_email(email):
- email = str(email).strip().lower()
+ email = smart_decode_punycode(str(email).strip().lower())
if any(r.match(email) for r in regexes.event_type_regexes["EMAIL_ADDRESS"]):
return email
assert False, f'Invalid email: "{email}"'
diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py
index a4a4400c6c..32e8565535 100644
--- a/bbot/core/helpers/web.py
+++ b/bbot/core/helpers/web.py
@@ -3,10 +3,11 @@
from time import sleep
from pathlib import Path
from requests_cache import CachedSession
+from requests.adapters import HTTPAdapter
from requests_cache.backends import SQLiteCache
from requests.exceptions import RequestException
-from bbot.core.errors import WordlistError
+from bbot.core.errors import WordlistError, CurlError
log = logging.getLogger("bbot.core.helpers.web")
@@ -87,21 +88,29 @@ def request(self, *args, **kwargs):
raise_error (bool): Whether to raise exceptions (default: False)
"""
+ # we handle our own retries
+ retries = kwargs.pop("retries", self.config.get("http_retries", 1))
+ if getattr(self, "retry_adapter", None) is None:
+ self.retry_adapter = HTTPAdapter(max_retries=0)
+
raise_error = kwargs.pop("raise_error", False)
cache_for = kwargs.pop("cache_for", None)
if cache_for is not None:
log.debug(f"Caching HTTP session with expire_after={cache_for}")
- try:
- session = self.cache_sessions[cache_for]
- except KeyError:
- db_path = str(self.cache_dir / "requests-cache.sqlite")
- backend = SQLiteCache(db_path=db_path)
- session = CachedSession(expire_after=cache_for, backend=backend)
- self.cache_sessions[cache_for] = session
-
- if kwargs.pop("session", None) or not cache_for:
+ db_path = str(self.cache_dir / "requests-cache.sqlite")
+ backend = SQLiteCache(db_path=db_path)
+ session = CachedSession(expire_after=cache_for, backend=backend)
+ session.mount("https://", self.retry_adapter)
+ session.mount("https://", self.retry_adapter)
+ elif kwargs.get("session", None) is not None:
session = kwargs.pop("session", None)
+ session.mount("https://", self.retry_adapter)
+ session.mount("https://", self.retry_adapter)
+ else:
+ session = requests.Session()
+ session.mount("https://", self.retry_adapter)
+ session.mount("https://", self.retry_adapter)
http_timeout = self.config.get("http_timeout", 20)
user_agent = self.config.get("user_agent", "BBOT")
@@ -112,7 +121,6 @@ def request(self, *args, **kwargs):
args = []
url = kwargs.get("url", "")
- retries = kwargs.pop("retries", 0)
if not args and "method" not in kwargs:
kwargs["method"] = "GET"
@@ -126,6 +134,12 @@ def request(self, *args, **kwargs):
headers = {}
if "User-Agent" not in headers:
headers.update({"User-Agent": user_agent})
+ # only add custom headers if the URL is in-scope
+ if self.scan.in_scope(url):
+ for hk, hv in self.scan.config.get("http_headers", {}).items():
+ # don't clobber headers
+ if hk not in headers:
+ headers[hk] = hv
kwargs["headers"] = headers
http_debug = self.config.get("http_debug", False)
@@ -146,7 +160,7 @@ def request(self, *args, **kwargs):
if retries != "infinite":
retries -= 1
if retries == "infinite" or retries >= 0:
- log.warning(f'Error requesting "{url}" ({e}), retrying...')
+ log.verbose(f'Error requesting "{url}" ({e}), retrying...')
sleep(1)
else:
if raise_error:
@@ -167,7 +181,7 @@ def api_page_iter(self, url, page_size=100, json=True, **requests_kwargs):
import traceback
log.warning(f'Error in api_page_iter() for url: "{new_url}"')
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
break
finally:
offset += page_size
@@ -175,12 +189,10 @@ def api_page_iter(self, url, page_size=100, json=True, **requests_kwargs):
def curl(self, *args, **kwargs):
-
url = kwargs.get("url", "")
if not url:
- log.debug("No URL supplied to CURL helper")
- return
+ raise CurlError("No URL supplied to CURL helper")
curl_command = ["curl", url, "-s"]
@@ -206,6 +218,11 @@ def curl(self, *args, **kwargs):
if "User-Agent" not in headers:
headers["User-Agent"] = user_agent
+ # only add custom headers if the URL is in-scope
+ if self.scan.in_scope(url):
+ for hk, hv in self.scan.config.get("http_headers", {}).items():
+ headers[hk] = hv
+
# add the timeout
if not "timeout" in kwargs:
timeout = http_timeout
@@ -214,7 +231,7 @@ def curl(self, *args, **kwargs):
curl_command.append(str(timeout))
for k, v in headers.items():
- if type(v) == list:
+ if isinstance(v, list):
for x in v:
curl_command.append("-H")
curl_command.append(f"{k}: {x}")
@@ -238,7 +255,6 @@ def curl(self, *args, **kwargs):
cookies = kwargs.get("cookies", "")
if cookies:
-
curl_command.append("-b")
cookies_str = ""
for k, v in cookies.items():
diff --git a/bbot/core/helpers/wordcloud.py b/bbot/core/helpers/wordcloud.py
index c79f91cb61..881ba872ad 100644
--- a/bbot/core/helpers/wordcloud.py
+++ b/bbot/core/helpers/wordcloud.py
@@ -162,7 +162,7 @@ def save(self, filename=None, limit=None):
import traceback
log.warning(f"Failed to save word cloud to {filename}: {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
return False, filename
def load(self, filename=None):
@@ -192,4 +192,4 @@ def load(self, filename=None):
log_fn = log.warning
log_fn(f"Failed to load word cloud from {wordcloud_path}: {e}")
if filename is not None:
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
diff --git a/bbot/core/logger/__init__.py b/bbot/core/logger/__init__.py
index 9d82a31331..d55c5a2b0a 100644
--- a/bbot/core/logger/__init__.py
+++ b/bbot/core/logger/__init__.py
@@ -1 +1 @@
-from .logger import init_logging, get_log_level, ColoredFormatter
+from .logger import init_logging, get_log_level, ColoredFormatter, toggle_log_level
diff --git a/bbot/core/logger/logger.py b/bbot/core/logger/logger.py
index b1e86fea0e..c5668b1d89 100644
--- a/bbot/core/logger/logger.py
+++ b/bbot/core/logger/logger.py
@@ -13,6 +13,9 @@
from ..helpers.logger import colorize, loglevel_mapping
+_log_level_override = None
+
+
class ColoredFormatter(logging.Formatter):
"""
Pretty colors for terminal
@@ -81,6 +84,7 @@ def logToRoot(message, *args, **kwargs):
# custom logging levels
addLoggingLevel("STDOUT", 100)
+addLoggingLevel("TRACE", 49)
addLoggingLevel("HUGEWARNING", 31)
addLoggingLevel("HUGESUCCESS", 26)
addLoggingLevel("SUCCESS", 25)
@@ -89,6 +93,9 @@ def logToRoot(message, *args, **kwargs):
addLoggingLevel("VERBOSE", 15)
+verbosity_levels_toggle = [logging.INFO, logging.VERBOSE, logging.DEBUG]
+
+
def stop_listener(listener):
with suppress(Exception):
listener.stop()
@@ -109,7 +116,6 @@ def log_worker_setup(logging_queue):
def log_listener_setup(logging_queue):
-
log_dir = Path(config["home"]) / "logs"
if not mkdir(log_dir, raise_error=False):
error_and_exit(f"Failure creating or error writing to BBOT logs directory ({log_dir})")
@@ -130,13 +136,13 @@ def log_listener_setup(logging_queue):
f"{log_dir}/bbot.debug.log", when="d", interval=1, backupCount=14
)
- log_level = get_log_level()
-
- config_debug = config.get("debug", False)
- config_silent = config.get("silent", False)
-
def stderr_filter(record):
- if record.levelno == logging.STDOUT:
+ config_silent = config.get("silent", False)
+ log_level = get_log_level()
+ excluded_levels = [logging.STDOUT]
+ if log_level > logging.DEBUG:
+ excluded_levels.append(logging.TRACE)
+ if record.levelno in excluded_levels:
return False
if record.levelno >= logging.ERROR:
return True
@@ -149,7 +155,7 @@ def stderr_filter(record):
stderr_handler.addFilter(stderr_filter)
stdout_handler.addFilter(lambda x: x.levelno == logging.STDOUT)
debug_handler.addFilter(lambda x: x.levelno != logging.STDOUT and x.levelno >= logging.DEBUG)
- main_handler.addFilter(lambda x: x.levelno != logging.STDOUT and x.levelno >= logging.VERBOSE)
+ main_handler.addFilter(lambda x: x.levelno not in (logging.STDOUT, logging.TRACE) and x.levelno >= logging.VERBOSE)
# Set log format
debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
@@ -158,9 +164,7 @@ def stderr_filter(record):
stderr_handler.setFormatter(ColoredFormatter("%(levelname)s %(name)s: %(message)s"))
stdout_handler.setFormatter(logging.Formatter("%(message)s"))
- handlers = [stdout_handler, stderr_handler, main_handler]
- if config_debug:
- handlers.append(debug_handler)
+ handlers = [stdout_handler, stderr_handler, main_handler, debug_handler]
log_listener = QueueListener(logging_queue, *handlers)
log_listener.start()
@@ -192,6 +196,9 @@ def init_logging():
def get_log_level():
+ if _log_level_override is not None:
+ return _log_level_override
+
from bbot.core.configurator.args import cli_options
if config.get("debug", False) or os.environ.get("BBOT_DEBUG", "").lower() in ("true", "yes"):
@@ -204,3 +211,23 @@ def get_log_level():
if cli_options.debug:
loglevel = logging.DEBUG
return loglevel
+
+
+def set_log_level(level, logger=None):
+ global _log_level_override
+ if logger is not None:
+ logger.hugeinfo(f"Setting log level to {logging.getLevelName(level)}")
+ config["silent"] = False
+ _log_level_override = level
+ log = logging.getLogger("bbot")
+ log.setLevel(level)
+
+
+def toggle_log_level(logger=None):
+ log_level = get_log_level()
+ if log_level in verbosity_levels_toggle:
+ for i, level in enumerate(verbosity_levels_toggle):
+ if log_level == level:
+ set_log_level(verbosity_levels_toggle[(i + 1) % len(verbosity_levels_toggle)], logger=logger)
+ else:
+ set_log_level(verbosity_levels_toggle[0], logger=logger)
diff --git a/bbot/defaults.yml b/bbot/defaults.yml
index 350e2667b2..82f9c61b83 100644
--- a/bbot/defaults.yml
+++ b/bbot/defaults.yml
@@ -3,13 +3,19 @@
# BBOT working directory
home: ~/.bbot
# Don't output events that are further than this from the main scope
-scope_report_distance: 1
+# 1 == 1 hope away from main scope
+# 0 == in scope only
+scope_report_distance: 0
# Generate new DNS_NAME and IP_ADDRESS events through DNS resolution
dns_resolution: true
# Limit the number of BBOT threads
max_threads: 25
-# Limit the number of DNS threads
+# Limit the number of DNS threads (this should be approximately 4x max_threads)
max_dns_threads: 100
+# HTTP proxy
+http_proxy:
+# Web user-agent
+user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36
### ADVANCED OPTIONS ###
@@ -28,12 +34,17 @@ excavate: True
# Summarize activity at the end of a scan
aggregate: True
-# HTTP proxy
-http_proxy:
# HTTP timeout (for Python requests; API calls, etc.)
-http_timeout: 30
+http_timeout: 10
# HTTP timeout (for httpx)
httpx_timeout: 5
+# Custom HTTP headers (e.g. cookies, etc.)
+# in the format { "Header-Key": "header_value" }
+# These are attached to all in-scope HTTP requests
+# Note that some modules (e.g. github) may end up sending these to out-of-scope resources
+http_headers: {}
+# HTTP retries (for Python requests; API calls, etc.)
+http_retries: 1
# HTTP retries (for httpx)
httpx_retries: 1
# Enable/disable debug messages for web requests/responses
@@ -64,8 +75,6 @@ dns_debug: false
ssl_verify: false
# How many scan results to keep before cleaning up the older ones
keep_scans: 20
-# Web user-agent
-user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36
# Completely ignore URLs with these extensions
url_extension_blacklist:
# images
diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py
index 43819eaf35..55bf966941 100644
--- a/bbot/modules/anubisdb.py
+++ b/bbot/modules/anubisdb.py
@@ -2,21 +2,31 @@
class anubisdb(crobat):
-
flags = ["subdomain-enum", "passive", "safe"]
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
meta = {"description": "Query jldc.me's database for subdomains"}
base_url = "https://jldc.me/anubis/subdomains"
+ dns_abort_depth = 5
def request_url(self, query):
url = f"{self.base_url}/{self.helpers.quote(query)}"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
+
+ def abort_if_pre(self, hostname):
+ """
+ Discards results that are longer than 5 segments, e.g. a.b.c.d.evilcorp.com
+ This exists because of the _disgusting_ amount of garbage data in this API
+ """
+ dns_depth = hostname.count(".") + 1
+ if dns_depth > self.dns_abort_depth:
+ return True
+ return False
def abort_if(self, event):
# abort if dns name is unresolved
- return not "resolved" in event.tags or super().abort_if(event)
+ return (not "resolved" in event.tags) or super().abort_if(event)
def parse_results(self, r, query):
results = set()
@@ -24,6 +34,6 @@ def parse_results(self, r, query):
if json:
for hostname in json:
hostname = str(hostname).lower()
- if hostname.endswith(f".{query}"):
+ if hostname.endswith(f".{query}") and not self.abort_if_pre(hostname):
results.add(hostname)
return results
diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py
index a075679f65..fafa391a76 100644
--- a/bbot/modules/azure_tenant.py
+++ b/bbot/modules/azure_tenant.py
@@ -23,7 +23,8 @@ def handle_event(self, event):
if domains:
self.success(f'Found {len(domains):,} domains under tenant for "{query}"')
for domain in domains:
- self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"])
+ if domain != query:
+ self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"])
# todo: tenants?
def query(self, domain):
@@ -55,7 +56,7 @@ def query(self, domain):
self.debug(f"Retrieving tenant domains at {url}")
- r = self.helpers.request(url, method="POST", headers=headers, data=data)
+ r = self.request_with_fail_count(url, method="POST", headers=headers, data=data)
status_code = getattr(r, "status_code", 0)
if status_code not in (200, 421):
self.warning(f'Error retrieving azure_tenant domains for "{domain}" (status code: {status_code})')
diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py
index 3393e4c0db..eea9ffaee9 100644
--- a/bbot/modules/badsecrets.py
+++ b/bbot/modules/badsecrets.py
@@ -4,27 +4,28 @@
class badsecrets(BaseModule):
-
watched_events = ["HTTP_RESPONSE"]
produced_events = ["FINDING", "VULNERABILITY"]
- flags = ["active", "safe", "web-basic"]
- meta = {"description": "Library for detecting known or weak secrets on across many platforms"}
-
- deps_pip = ["badsecrets"]
+ flags = ["active", "safe", "web-basic", "web-thorough"]
+ meta = {"description": "Library for detecting known or weak secrets across many web frameworks"}
+ max_event_handlers = 2
+ deps_pip = ["badsecrets>=0.1.220"]
def handle_event(self, event):
resp_body = event.data.get("body", None)
resp_headers = event.data.get("header", None)
resp_cookies = {}
- resp_cookies_raw = resp_headers.get("set_cookie", None)
- if resp_cookies_raw:
- if "," in resp_cookies_raw:
- resp_cookies_list = resp_cookies_raw.split(",")
- else:
- resp_cookies_list = [resp_cookies_raw]
- for c in resp_cookies_list:
- c2 = c.strip().split(";")[0].split("=")
- resp_cookies[c2[0]] = c2[1]
+ if resp_headers:
+ resp_cookies_raw = resp_headers.get("set_cookie", None)
+ if resp_cookies_raw:
+ if "," in resp_cookies_raw:
+ resp_cookies_list = resp_cookies_raw.split(",")
+ else:
+ resp_cookies_list = [resp_cookies_raw]
+ for c in resp_cookies_list:
+ c2 = c.lstrip(";").strip().split(";")[0].split("=")
+ if len(c2) == 2:
+ resp_cookies[c2[0]] = c2[1]
if resp_body or resp_cookies:
r_list = carve_all_modules(body=resp_body, cookies=resp_cookies)
if r_list:
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index a2627cdf43..2ca858a4ae 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -2,18 +2,14 @@
import logging
import threading
import traceback
-from time import sleep
from sys import exc_info
from contextlib import suppress
from ..core.helpers.threadpool import ThreadPoolWrapper
from ..core.errors import ScanCancelledError, ValidationError, WordlistError
-from bbot.core.event.base import is_event
-
class BaseModule:
-
# Event types to watch
watched_events = []
# Event types to produce
@@ -67,13 +63,15 @@ class BaseModule:
batch_size = 1
# Seconds to wait before force-submitting batch
batch_wait = 10
+ # Use in conjunction with .request_with_fail_count() to set_error_state() after this many failed HTTP requests
+ failed_request_abort_threshold = 5
# When set to false, prevents events generated by this module from being automatically marked as in-scope
# Useful for low-confidence modules like speculate and ipneighbor
_scope_shepherding = True
# Exclude from scan statistics
_stats_exclude = False
- # outgoing queue size
- _qsize = 100
+ # outgoing queue size (None == infinite)
+ _qsize = None
# Priority of events raised by this module, 1-5, lower numbers == higher priority
_priority = 3
# Name, overridden automatically
@@ -86,13 +84,11 @@ def __init__(self, scan):
self.errored = False
self._log = None
self._incoming_event_queue = None
- self._outgoing_event_queue = None
- # how many seconds we've gone without processing a batch
- self._batch_idle = 0
+ # seconds since we've submitted a batch
+ self._last_submitted_batch = None
# wrapper around shared thread pool to ensure that a single module doesn't hog more than its share
- self.thread_pool = ThreadPoolWrapper(
- self.scan._thread_pool.executor, max_workers=self.config.get("max_threads", self.max_threads)
- )
+ max_workers = self.config.get("max_threads", self.max_threads)
+ self.thread_pool = ThreadPoolWrapper(self.scan._thread_pool, max_workers=max_workers)
self._internal_thread_pool = ThreadPoolWrapper(
self.scan._internal_thread_pool.executor, max_workers=self.max_event_handlers
)
@@ -101,6 +97,15 @@ def __init__(self, scan):
self._cleanedup = False
self._watched_events = None
+ self._lock = threading.RLock()
+ self.event_received = threading.Condition(self._lock)
+
+ # string constant
+ self._custom_filter_criteria_msg = "it did not meet custom filter criteria"
+
+ # track number of failures (for .request_with_fail_count())
+ self._request_failures = 0
+
def setup(self):
"""
Perform setup functions at the beginning of the scan.
@@ -159,6 +164,42 @@ def cleanup(self):
"""
return
+ def require_api_key(self):
+ """
+ Use in setup() to ensure the module is configured with an API key
+ """
+ self.api_key = self.config.get("api_key", "")
+ if self.auth_secret:
+ try:
+ self.ping()
+ self.hugesuccess(f"API is ready")
+ return True
+ except Exception as e:
+ return None, f"Error with API ({str(e).strip()})"
+ else:
+ return None, "No API key set"
+
+ def ping(self):
+ """
+ Used in conjuction with require_api_key to ensure an API is up and responding
+
+ Requires the use of an assert statement.
+
+ E.g. if your API has a "/ping" endpoint, you can use it like this:
+ def ping(self):
+ r = self.request_with_fail_count(f"{self.base_url}/ping")
+ resp_content = getattr(r, "text", "")
+ assert getattr(r, "status_code", 0) == 200, resp_content
+ """
+ return
+
+ @property
+ def auth_secret(self):
+ """
+ Use this to indicate whether the module has everything it needs for authentication
+ """
+ return getattr(self, "api_key", "")
+
def get_watched_events(self):
"""
Override if you need your watched_events to be dynamic
@@ -173,25 +214,41 @@ def submit_task(self, *args, **kwargs):
def catch(self, *args, **kwargs):
return self.scan.manager.catch(*args, **kwargs)
+ def _postcheck_and_run(self, callback, event):
+ acceptable, reason = self._event_postcheck(event)
+ if not acceptable:
+ if reason:
+ self.debug(f"Not accepting {event} because {reason}")
+ return
+ return callback(event)
+
def _handle_batch(self, force=False):
if self.batch_size <= 1:
return
if self.num_queued_events > 0 and (force or self.num_queued_events >= self.batch_size):
- self._batch_idle = 0
on_finish_callback = None
events, finish, report = self.events_waiting
if finish:
on_finish_callback = self.finish
elif report:
on_finish_callback = self.report
- if events:
+ checked_events = []
+ for e in events:
+ acceptable, reason = self._event_postcheck(e)
+ if not acceptable:
+ if reason:
+ self.debug(f"Not accepting {e} because {reason}")
+ continue
+ checked_events.append(e)
+ if checked_events:
self.debug(f"Handling batch of {len(events):,} events")
- self._internal_thread_pool.submit_task(
- self.catch,
- self.handle_batch,
- *events,
- _on_finish_callback=on_finish_callback,
- )
+ if not self.errored:
+ self._internal_thread_pool.submit_task(
+ self.catch,
+ self.handle_batch,
+ *checked_events,
+ _on_finish_callback=on_finish_callback,
+ )
return True
return False
@@ -209,14 +266,28 @@ def make_event(self, *args, **kwargs):
return event
def emit_event(self, *args, **kwargs):
- if self.scan.stopping:
- return
event_kwargs = dict(kwargs)
for o in ("on_success_callback", "abort_if", "quick"):
event_kwargs.pop(o, None)
event = self.make_event(*args, **event_kwargs)
+ if event is None:
+ return
+ # nerf event's priority if it's likely not to be in scope
+ if event.scope_distance > 0:
+ event_in_scope = self.scan.whitelisted(event) and not self.scan.blacklisted(event)
+ if not event_in_scope:
+ event.module_priority += event.scope_distance
if event:
- self.outgoing_event_queue.put((event, kwargs))
+ # Wait for parent event to resolve (in case its scope distance changes)
+ while 1:
+ if self.scan.stopping:
+ return
+ resolved = event.source._resolved.wait(timeout=0.1)
+ if resolved:
+ # update event's scope distance based on its parent
+ event.scope_distance = event.source.scope_distance + 1
+ break
+ self.scan.manager.incoming_event_queue.put((event, kwargs))
@property
def events_waiting(self):
@@ -231,11 +302,8 @@ def events_waiting(self):
break
try:
event = self.incoming_event_queue.get_nowait()
- if type(event) == str:
- if event == "FINISHED":
- finish = True
- elif event == "REPORT":
- report = True
+ if event.type == "FINISHED":
+ finish = True
else:
events.append(event)
except queue.Empty:
@@ -254,7 +322,6 @@ def start(self):
self.thread.start()
def _setup(self):
-
status_codes = {False: "hard-fail", None: "soft-fail", True: "success"}
status = False
@@ -272,7 +339,7 @@ def _setup(self):
if isinstance(e, WordlistError):
status = None
msg = f"{e}"
- self.debug(traceback.format_exc())
+ self.trace()
return status, str(msg)
@property
@@ -280,40 +347,23 @@ def _force_batch(self):
"""
Determine whether a batch should be forcefully submitted
"""
- # if we've been idle long enough
- if self._batch_idle >= self.batch_wait:
- return True
- # if scan is finishing
- if self.scan.status == "FINISHING":
- return True
- # if there's a batch stalemate
- batch_modules = [m for m in self.scan.modules.values() if m.batch_size > 1]
- if all([(not m.running) for m in batch_modules]):
- return True
- return False
+ # if we're below our maximum threading potential
+ return self._internal_thread_pool.num_tasks < self.max_event_handlers
def _worker(self):
- # keep track of how long we've been running
- iterations = 0
try:
while not self.scan.stopping:
- iterations += 1
-
# hold the reigns if our outgoing queue is full
- if self.outgoing_event_queue.qsize() >= self._qsize:
- self._batch_idle += 1
- sleep(0.1)
+ if self._qsize and self.outgoing_event_queue_qsize >= self._qsize:
+ with self.event_received:
+ self.event_received.wait(timeout=0.1)
continue
if self.batch_size > 1:
- if iterations % 10 == 0:
- self._batch_idle += 1
- force = self._force_batch
- if force:
- self._batch_idle = 0
- submitted = self._handle_batch(force=force)
+ submitted = self._handle_batch(force=self._force_batch)
if not submitted:
- sleep(0.1)
+ with self.event_received:
+ self.event_received.wait(timeout=0.1)
else:
try:
@@ -326,16 +376,15 @@ def _worker(self):
continue
self.debug(f"Got {e} from {getattr(e, 'module', e)}")
# if we receive the special "FINISHED" event
- if type(e) == str:
- if e == "FINISHED":
- self._internal_thread_pool.submit_task(self.catch, self.finish)
- elif e == "REPORT":
- self._internal_thread_pool.submit_task(self.catch, self.report)
+ if e.type == "FINISHED":
+ self._internal_thread_pool.submit_task(self.catch, self.finish)
else:
if self._type == "output":
- self.catch(self.handle_event, e)
+ self.catch(self._postcheck_and_run, self.handle_event, e)
else:
- self._internal_thread_pool.submit_task(self.catch, self.handle_event, e)
+ self._internal_thread_pool.submit_task(
+ self.catch, self._postcheck_and_run, self.handle_event, e
+ )
except KeyboardInterrupt:
self.debug(f"Interrupted")
@@ -344,13 +393,7 @@ def _worker(self):
self.verbose(f"Scan cancelled, {e}")
except Exception as e:
self.set_error_state(f"Exception ({e.__class__.__name__}) in module {self.name}:\n{e}")
- self.debug(traceback.format_exc())
-
- def _filter_event(self, event, precheck_only=False):
- acceptable, reason = self._event_precheck(event)
- if acceptable and not precheck_only:
- acceptable, reason = self._event_postcheck(event)
- return acceptable, reason
+ self.trace()
@property
def max_scope_distance(self):
@@ -361,14 +404,13 @@ def max_scope_distance(self):
def _event_precheck(self, event):
"""
Check if an event should be accepted by the module
- These checks are safe to run before an event has been DNS-resolved
+ Used when putting an event INTO the modules' queue
"""
- # special "FINISHED" event
- if type(event) == str:
- if event in ("FINISHED", "REPORT"):
- return True, ""
- else:
- return False, f'string value "{event}" is invalid'
+ # special signal event types
+ if event.type in ("FINISHED",):
+ return True, ""
+ if self.errored:
+ return False, f"module is in error state"
# exclude non-watched types
if not any(t in self.get_watched_events() for t in ("*", event.type)):
return False, "its type is not in watched_events"
@@ -393,11 +435,14 @@ def _event_precheck(self, event):
def _event_postcheck(self, event):
"""
Check if an event should be accepted by the module
- These checks must be run after an event has been DNS-resolved
+ Used when taking an event FROM the module's queue (immediately before it's handled)
"""
- if type(event) == str:
+ if event.type in ("FINISHED",):
return True, ""
+ if "active" in self.flags and "target" in event.tags and event not in self.scan.whitelist:
+ return False, "it is not in whitelist and module has active flag"
+
if self.in_scope_only:
if event.scope_distance > 0:
return False, "it did not meet in_scope_only filter criteria"
@@ -412,13 +457,18 @@ def _event_postcheck(self, event):
# custom filtering
try:
- if not self.filter_event(event):
- return False, f"{event} did not meet custom filter criteria"
+ filter_result = self.filter_event(event)
+ msg = str(self._custom_filter_criteria_msg)
+ with suppress(ValueError, TypeError):
+ filter_result, reason = filter_result
+ msg += f": {reason}"
+ if not filter_result:
+ return False, msg
+ except ScanCancelledError:
+ return False, "Scan cancelled"
except Exception as e:
- import traceback
-
self.error(f"Error in filter_event({event}): {e}")
- self.debug(traceback.format_exc())
+ self.trace()
return True, ""
@@ -430,21 +480,26 @@ def _cleanup(self):
self.catch(callback, _force=True)
def queue_event(self, event):
- if self.incoming_event_queue is not None and not self.errored:
- acceptable, reason = self._filter_event(event)
- if not acceptable and reason:
+ if self.incoming_event_queue in (None, False):
+ self.debug(f"Not in an acceptable state to queue event")
+ return
+ acceptable, reason = self._event_precheck(event)
+ if not acceptable:
+ if reason and reason != "its type is not in watched_events":
self.debug(f"Not accepting {event} because {reason}")
- return
- if is_event(event):
- self.scan.stats.event_consumed(event, self)
+ return
+ self.scan.stats.event_consumed(event, self)
+ try:
self.incoming_event_queue.put(event)
- else:
+ except AttributeError:
self.debug(f"Not in an acceptable state to queue event")
+ with self.event_received:
+ self.event_received.notify()
def set_error_state(self, message=None):
- if message is not None:
- self.error(str(message))
if not self.errored:
+ if message is not None:
+ self.warning(str(message))
self.debug(f"Setting error state for module {self.name}")
self.errored = True
# clear incoming queue
@@ -471,24 +526,34 @@ def status(self):
internal_pool = self._internal_thread_pool.num_tasks
pool_total = main_pool + internal_pool
incoming_qsize = 0
- outgoing_qsize = 0
if self.incoming_event_queue:
incoming_qsize = self.incoming_event_queue.qsize()
- if self.outgoing_event_queue:
- outgoing_qsize = self.outgoing_event_queue.qsize()
status = {
- "events": {"incoming": incoming_qsize, "outgoing": outgoing_qsize},
+ "events": {"incoming": incoming_qsize, "outgoing": self.outgoing_event_queue_qsize},
"tasks": {"main_pool": main_pool, "internal_pool": internal_pool, "total": pool_total},
"errored": self.errored,
}
status["running"] = self._is_running(status)
return status
+ def request_with_fail_count(self, *args, **kwargs):
+ r = self.helpers.request(*args, **kwargs)
+ if r is None:
+ self._request_failures += 1
+ else:
+ self._request_failures = 0
+ if self._request_failures >= self.failed_request_abort_threshold:
+ self.set_error_state(f"Setting error state due to {self._request_failures:,} failed HTTP requests")
+ return r
+
@staticmethod
def _is_running(module_status):
for pool, count in module_status["tasks"].items():
if count > 0:
return True
+ for direction, qsize in module_status["events"].items():
+ if qsize > 0:
+ return True
return False
@property
@@ -508,14 +573,12 @@ def config(self):
@property
def incoming_event_queue(self):
if self._incoming_event_queue is None:
- self._incoming_event_queue = queue.SimpleQueue()
+ self._incoming_event_queue = queue.PriorityQueue()
return self._incoming_event_queue
@property
- def outgoing_event_queue(self):
- if self._outgoing_event_queue is None:
- self._outgoing_event_queue = queue.SimpleQueue()
- return self._outgoing_event_queue
+ def outgoing_event_queue_qsize(self):
+ return self.scan.manager.incoming_event_queue.modules.get(str(self), 0)
@property
def priority(self):
@@ -527,7 +590,7 @@ def auth_required(self):
@property
def log(self):
- if self._log is None:
+ if getattr(self, "_log", None) is None:
self._log = logging.getLogger(f"bbot.modules.{self.name}")
return self._log
@@ -560,20 +623,21 @@ def hugesuccess(self, *args, **kwargs):
def warning(self, *args, **kwargs):
self.log.warning(*args, extra={"scan_id": self.scan.id}, **kwargs)
- self._log_traceback()
+ self.trace()
def hugewarning(self, *args, **kwargs):
self.log.hugewarning(*args, extra={"scan_id": self.scan.id}, **kwargs)
- self._log_traceback()
+ self.trace()
def error(self, *args, **kwargs):
self.log.error(*args, extra={"scan_id": self.scan.id}, **kwargs)
- self._log_traceback()
+ self.trace()
- def critical(self, *args, **kwargs):
- self.log.critical(*args, extra={"scan_id": self.scan.id}, **kwargs)
-
- def _log_traceback(self):
+ def trace(self):
e_type, e_val, e_traceback = exc_info()
if e_type is not None:
- self.debug(traceback.format_exc())
+ self.log.trace(traceback.format_exc())
+
+ def critical(self, *args, **kwargs):
+ self.log.critical(*args, extra={"scan_id": self.scan.id}, **kwargs)
+ self.trace()
diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py
index 809fe74767..d81a081a91 100644
--- a/bbot/modules/bevigil.py
+++ b/bbot/modules/bevigil.py
@@ -39,11 +39,11 @@ def handle_event(self, event):
def request_subdomains(self, query):
url = f"{self.base_url}/{self.helpers.quote(query)}/subdomains/"
- return self.helpers.request(url, headers=self.headers)
+ return self.request_with_fail_count(url, headers=self.headers)
def request_urls(self, query):
url = f"{self.base_url}/{self.helpers.quote(query)}/urls/"
- return self.helpers.request(url, headers=self.headers)
+ return self.request_with_fail_count(url, headers=self.headers)
def parse_subdomains(self, r, query=None):
results = set()
diff --git a/bbot/modules/binaryedge.py b/bbot/modules/binaryedge.py
index 1ddc16721c..7bcb266806 100644
--- a/bbot/modules/binaryedge.py
+++ b/bbot/modules/binaryedge.py
@@ -21,13 +21,13 @@ def setup(self):
def ping(self):
url = f"{self.base_url}/user/subscription"
- j = self.helpers.request(url, headers=self.headers).json()
+ j = self.request_with_fail_count(url, headers=self.headers).json()
assert j.get("requests_left", 0) > 0
def request_url(self, query):
# todo: host query (certs + services)
url = f"{self.base_url}/query/domains/subdomain/{self.helpers.quote(query)}"
- return self.helpers.request(url, headers=self.headers)
+ return self.request_with_fail_count(url, headers=self.headers)
def parse_results(self, r, query):
j = r.json()
diff --git a/bbot/modules/bucket_aws.py b/bbot/modules/bucket_aws.py
index aff54e1b52..a831e20df0 100644
--- a/bbot/modules/bucket_aws.py
+++ b/bbot/modules/bucket_aws.py
@@ -4,7 +4,7 @@
class bucket_aws(BaseModule):
watched_events = ["DNS_NAME", "STORAGE_BUCKET"]
produced_events = ["STORAGE_BUCKET", "FINDING"]
- flags = ["active", "safe", "cloud-enum"]
+ flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"]
meta = {"description": "Check for S3 buckets related to target"}
options = {"max_threads": 10, "permutations": False}
options_desc = {
diff --git a/bbot/modules/bucket_azure.py b/bbot/modules/bucket_azure.py
index 093a45adcc..d138cdece9 100644
--- a/bbot/modules/bucket_azure.py
+++ b/bbot/modules/bucket_azure.py
@@ -4,7 +4,7 @@
class bucket_azure(bucket_aws):
watched_events = ["DNS_NAME", "STORAGE_BUCKET"]
produced_events = ["STORAGE_BUCKET", "FINDING"]
- flags = ["active", "safe", "cloud-enum"]
+ flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"]
meta = {"description": "Check for Azure storage blobs related to target"}
options = {"max_threads": 10, "permutations": False}
options_desc = {
diff --git a/bbot/modules/bucket_digitalocean.py b/bbot/modules/bucket_digitalocean.py
index 8fd55a258d..7f59d7ee56 100644
--- a/bbot/modules/bucket_digitalocean.py
+++ b/bbot/modules/bucket_digitalocean.py
@@ -4,7 +4,7 @@
class bucket_digitalocean(bucket_aws):
watched_events = ["DNS_NAME", "STORAGE_BUCKET"]
produced_events = ["STORAGE_BUCKET", "FINDING"]
- flags = ["active", "safe", "cloud-enum"]
+ flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"]
meta = {"description": "Check for DigitalOcean spaces related to target"}
options = {"max_threads": 10, "permutations": False}
options_desc = {
diff --git a/bbot/modules/bucket_gcp.py b/bbot/modules/bucket_gcp.py
index 7bcfa799ec..5a2c2d7bc0 100644
--- a/bbot/modules/bucket_gcp.py
+++ b/bbot/modules/bucket_gcp.py
@@ -8,7 +8,7 @@ class bucket_gcp(bucket_aws):
watched_events = ["DNS_NAME", "STORAGE_BUCKET"]
produced_events = ["STORAGE_BUCKET", "FINDING"]
- flags = ["active", "safe", "cloud-enum"]
+ flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"]
meta = {"description": "Check for Google object storage related to target"}
options = {"max_threads": 10, "permutations": False}
options_desc = {
diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py
index 4d989ddf2c..8a920f6a24 100644
--- a/bbot/modules/builtwith.py
+++ b/bbot/modules/builtwith.py
@@ -14,7 +14,6 @@
class builtwith(shodan_dns):
-
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
flags = ["affiliates", "subdomain-enum", "passive", "safe"]
@@ -45,11 +44,11 @@ def handle_event(self, event):
def request_domains(self, query):
url = f"{self.base_url}/v20/api.json?KEY={self.api_key}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def request_redirects(self, query):
url = f"{self.base_url}/redirect1/api.json?KEY={self.api_key}&LOOKUP={query}"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def parse_domains(self, r, query):
"""
diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py
index cf017f69c6..e459abe26e 100644
--- a/bbot/modules/bypass403.py
+++ b/bbot/modules/bypass403.py
@@ -70,15 +70,13 @@
class bypass403(BaseModule):
-
watched_events = ["URL"]
produced_events = ["FINDING"]
- flags = ["active", "aggressive", "web-advanced"]
+ flags = ["active", "aggressive", "web-thorough"]
meta = {"description": "Check 403 pages for common bypasses"}
in_scope_only = True
def handle_event(self, event):
-
try:
compare_helper = self.helpers.http_compare(event.data, allow_redirects=True)
except HttpCompareError as e:
@@ -86,7 +84,6 @@ def handle_event(self, event):
return
for sig in signatures:
-
sig = self.format_signature(sig, event)
if sig[2] != None:
headers = dict(sig[2])
@@ -98,7 +95,6 @@ def handle_event(self, event):
if match == False:
if str(subject_response.status_code)[0] != "4":
-
if sig[2]:
added_header_tuple = next(iter(sig[2].items()))
reported_signature = f"Added Header: {added_header_tuple[0]}: {added_header_tuple[1]}"
diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py
index 66f66e6950..7fde17dcd9 100644
--- a/bbot/modules/c99.py
+++ b/bbot/modules/c99.py
@@ -13,12 +13,12 @@ class c99(shodan_dns):
def ping(self):
url = f"{self.base_url}/randomnumber?key={self.api_key}&between=1,100&json"
- response = self.helpers.request(url)
+ response = self.request_with_fail_count(url)
assert response.json()["success"] == True
def request_url(self, query):
url = f"{self.base_url}/subdomainfinder?key={self.api_key}&domain={self.helpers.quote(query)}&json"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def parse_results(self, r, query):
j = r.json()
diff --git a/bbot/modules/certspotter.py b/bbot/modules/certspotter.py
index 1606b54dc4..5e928fc4be 100644
--- a/bbot/modules/certspotter.py
+++ b/bbot/modules/certspotter.py
@@ -11,7 +11,7 @@ class certspotter(crobat):
def request_url(self, query):
url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def parse_results(self, r, query):
json = r.json()
diff --git a/bbot/modules/crobat.py b/bbot/modules/crobat.py
index 58e133d82c..4be69b1ffe 100644
--- a/bbot/modules/crobat.py
+++ b/bbot/modules/crobat.py
@@ -7,19 +7,25 @@ class crobat(BaseModule):
Inherited by several other modules including sublist3r, dnsdumpster, etc.
"""
- flags = ["subdomain-enum", "passive", "safe"]
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
+ # tag "subdomain-enum" removed 2023-02-24 because API is offline
+ flags = ["passive", "safe"]
meta = {"description": "Query Project Crobat for subdomains"}
base_url = "https://sonar.omnisint.io"
-
+ # set module error state after this many failed requests in a row
+ abort_after_failures = 5
+ # whether to reject wildcard DNS_NAMEs
+ reject_wildcards = True
# this helps combat rate limiting by ensuring that a query doesn't execute
# until the queue is ready to receive its results
_qsize = 1
def setup(self):
self.processed = set()
+ self.http_timeout = self.scan.config.get("http_timeout", 10)
+ self._failures = 0
return True
def filter_event(self, event):
@@ -31,18 +37,19 @@ def filter_event(self, event):
This filter_event is used across many modules
"""
- if "unresolved" in event.tags:
- return False
query = self.make_query(event)
if self.already_processed(query):
- return False
- # discard dns-names with errors
- if any([t in event.tags for t in ("a-error", "aaaa-error")]):
- return False
- # discard wildcards
- wildcard_rdtypes = self.helpers.is_wildcard_domain(query)
- if any([t in wildcard_rdtypes for t in ("A", "AAAA")]):
- return False
+ return False, "Event was already processed"
+ if not "target" in event.tags:
+ if "unresolved" in event.tags:
+ return False, "Event is unresolved"
+ if any(t.startswith("cloud-") for t in event.tags):
+ return False, "Event is a cloud resource and not a direct target"
+ if self.reject_wildcards:
+ if any(t in event.tags for t in ("a-wildcard-domain", "aaaa-wildcard-domain", "cname-wildcard-domain")):
+ return False, "Event is a wildcard domain"
+ if any(t in event.tags for t in ("a-error", "aaaa-error")):
+ return False, "Event has a DNS resolution error"
self.processed.add(hash(query))
return True
@@ -54,19 +61,25 @@ def already_processed(self, hostname):
def abort_if(self, event):
# this helps weed out unwanted results when scanning IP_RANGES and wildcard domains
- return "in-scope" not in event.tags or "wildcard" in event.tags
+ if "in-scope" not in event.tags:
+ return True
+ if any(t in event.tags for t in ("wildcard", "wildcard-domain")):
+ return True
+ return False
def handle_event(self, event):
query = self.make_query(event)
results = self.query(query)
if results:
- for hostname in results:
- if not hostname == event:
- self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if)
+ for hostname in set(results):
+ if hostname:
+ hostname = hostname.lower()
+ if hostname.endswith(f".{query}") and not hostname == event.data:
+ self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if)
def request_url(self, query):
url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def make_query(self, event):
if "target" in event.tags:
@@ -92,7 +105,5 @@ def query(self, query, parse_fn=None, request_fn=None):
return results
self.debug(f'No results for "{query}"')
except Exception:
- import traceback
-
self.verbose(f"Error retrieving results for {query}")
- self.debug(traceback.format_exc())
+ self.trace()
diff --git a/bbot/modules/crt.py b/bbot/modules/crt.py
index dc18951c3a..d474241f77 100644
--- a/bbot/modules/crt.py
+++ b/bbot/modules/crt.py
@@ -2,13 +2,13 @@
class crt(crobat):
-
flags = ["subdomain-enum", "passive", "safe"]
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
meta = {"description": "Query crt.sh (certificate transparency) for subdomains"}
base_url = "https://crt.sh"
+ reject_wildcards = False
def setup(self):
self.cert_ids = set()
@@ -17,7 +17,7 @@ def setup(self):
def request_url(self, query):
params = {"q": query, "output": "json"}
url = self.helpers.add_get_params(self.base_url, params).geturl()
- return self.helpers.request(url)
+ return self.request_with_fail_count(url, timeout=self.http_timeout + 10)
def parse_results(self, r, query):
j = r.json()
diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py
index dfe7b33428..3bf30f8f81 100644
--- a/bbot/modules/deadly/ffuf.py
+++ b/bbot/modules/deadly/ffuf.py
@@ -7,10 +7,9 @@
class ffuf(BaseModule):
-
watched_events = ["URL"]
- produced_events = ["URL"]
- flags = ["brute-force", "aggressive", "active", "web-advanced"]
+ produced_events = ["URL_UNVERIFIED"]
+ flags = ["aggressive", "active"]
meta = {"description": "A fast web fuzzer written in Go"}
options = {
@@ -19,6 +18,7 @@ class ffuf(BaseModule):
"max_depth": 0,
"version": "1.5.0",
"extensions": "",
+ "ignore_redirects": True,
}
options_desc = {
@@ -27,6 +27,7 @@ class ffuf(BaseModule):
"max_depth": "the maxium directory depth to attempt to solve",
"version": "ffuf version",
"extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)",
+ "ignore_redirects": "Explicitly ignore redirects (301,302)",
}
blacklist = ["images", "css", "image"]
@@ -35,7 +36,7 @@ class ffuf(BaseModule):
{
"name": "Download ffuf",
"unarchive": {
- "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_linux_amd64.tar.gz",
+ "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz",
"include": "ffuf",
"dest": "#{BBOT_TOOLS}",
"remote_src": True,
@@ -46,12 +47,17 @@ class ffuf(BaseModule):
in_scope_only = True
def setup(self):
-
self.sanity_canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
wordlist_url = self.config.get("wordlist", "")
+ self.debug(f"Using wordlist [{wordlist_url}]")
self.wordlist = self.helpers.wordlist(wordlist_url)
- self.tempfile = self.generate_templist(self.wordlist)
+ f = open(self.wordlist, "r")
+ self.wordlist_lines = f.readlines()
+ f.close()
+ self.tempfile, tempfile_len = self.generate_templist()
+ self.verbose(f"Generated dynamic wordlist with length [{str(tempfile_len)}]")
self.extensions = self.config.get("extensions")
+ self.ignore_redirects = self.config.get("ignore_redirects")
return True
def handle_event(self, event):
@@ -67,19 +73,18 @@ def handle_event(self, event):
# if we think its a directory, normalize it.
fixed_url = event.data.rstrip("/") + "/"
- for r in self.execute_ffuf(self.tempfile, event, fixed_url):
- self.emit_event(r["url"], "URL", source=event, tags=[f"status-{r['status']}"])
-
- def execute_ffuf(self, tempfile, event, url, suffix=""):
+ for r in self.execute_ffuf(self.tempfile, fixed_url):
+ self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"])
- ffuf_exts = [""]
+ def execute_ffuf(self, tempfile, url, prefix="", suffix=""):
+ ffuf_exts = ["", "/"]
if self.extensions:
for ext in self.extensions.split(","):
ffuf_exts.append(f".{ext}")
for x in ffuf_exts:
- fuzz_url = f"{url}FUZZ{suffix}"
+ fuzz_url = f"{url}{prefix}FUZZ{suffix}"
command = [
"ffuf",
"-H",
@@ -93,6 +98,10 @@ def execute_ffuf(self, tempfile, event, url, suffix=""):
f"{fuzz_url}{x}",
]
+ if self.ignore_redirects:
+ command.append("-fc")
+ command.append("301,302")
+
for found in self.helpers.run_live(command):
try:
found_json = json.loads(found)
@@ -114,22 +123,19 @@ def execute_ffuf(self, tempfile, event, url, suffix=""):
except json.decoder.JSONDecodeError:
self.debug("Received invalid JSON from FFUF")
- def generate_templist(self, wordlist, prefix=None):
-
- f = open(wordlist, "r")
- fl = f.readlines()
- f.close()
+ def generate_templist(self, prefix=None):
+ line_count = 0
virtual_file = []
virtual_file.append(self.sanity_canary)
- for idx, val in enumerate(fl):
+ for idx, val in enumerate(self.wordlist_lines):
if idx > self.config.get("lines"):
break
if len(val) > 0:
-
if val.strip().lower() in self.blacklist:
self.debug(f"Skipping adding [{val.strip()}] to wordlist because it was in the blacklist")
else:
- if not prefix or val.startswith(prefix):
- virtual_file.append(f"{val.strip()}")
- return self.helpers.tempfile(virtual_file, pipe=False)
+ if not prefix or val.strip().lower().startswith(prefix.strip().lower()):
+ line_count += 1
+ virtual_file.append(f"{val.strip().lower()}")
+ return self.helpers.tempfile(virtual_file, pipe=False), line_count
diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py
index aed9a5adbb..9fdbc73875 100644
--- a/bbot/modules/deadly/nuclei.py
+++ b/bbot/modules/deadly/nuclei.py
@@ -5,15 +5,14 @@
class nuclei(BaseModule):
-
- watched_events = ["URL", "TECHNOLOGY"]
+ watched_events = ["URL"]
produced_events = ["FINDING", "VULNERABILITY"]
- flags = ["active", "aggressive", "web-advanced"]
+ flags = ["active", "aggressive"]
meta = {"description": "Fast and customisable vulnerability scanner"}
batch_size = 100
options = {
- "version": "2.7.9",
+ "version": "2.8.9",
"tags": "",
"templates": "",
"severity": "",
@@ -38,7 +37,7 @@ class nuclei(BaseModule):
{
"name": "Download nuclei",
"unarchive": {
- "src": "https://github.com/projectdiscovery/nuclei/releases/download/v#{BBOT_MODULES_NUCLEI_VERSION}/nuclei_#{BBOT_MODULES_NUCLEI_VERSION}_linux_amd64.zip",
+ "src": "https://github.com/projectdiscovery/nuclei/releases/download/v#{BBOT_MODULES_NUCLEI_VERSION}/nuclei_#{BBOT_MODULES_NUCLEI_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.zip",
"include": "nuclei",
"dest": "#{BBOT_TOOLS}",
"remote_src": True,
@@ -49,12 +48,11 @@ class nuclei(BaseModule):
in_scope_only = True
def setup(self):
-
# attempt to update nuclei templates
self.nuclei_templates_dir = self.helpers.tools_dir / "nuclei-templates"
self.info("Updating Nuclei templates")
update_results = self.helpers.run(
- ["nuclei", "-update-directory", self.nuclei_templates_dir, "-update-templates"]
+ ["nuclei", "-update-template-dir", self.nuclei_templates_dir, "-update-templates"]
)
if update_results.stderr:
if "Successfully downloaded nuclei-templates" in update_results.stderr:
@@ -116,12 +114,9 @@ def setup(self):
f"Template Severity: Critical [{self.nucleibudget.severity_stats['critical']}] High [{self.nucleibudget.severity_stats['high']}] Medium [{self.nucleibudget.severity_stats['medium']}] Low [{self.nucleibudget.severity_stats['low']}] Info [{self.nucleibudget.severity_stats['info']}] Unknown [{self.nucleibudget.severity_stats['unknown']}]"
)
- self.stats_file = self.helpers.tempfile_tail(callback=self.log_nuclei_status)
-
return True
def handle_batch(self, *events):
-
nuclei_input = [str(e.data) for e in events]
for severity, template, host, name, extracted_results in self.execute_nuclei(nuclei_input):
source_event = self.correlate_event(events, host)
@@ -161,11 +156,10 @@ def correlate_event(self, events, host):
self.warning("Failed to correlate nuclei result with event")
def execute_nuclei(self, nuclei_input):
-
command = [
"nuclei",
"-json",
- "-update-directory",
+ "-update-template-dir",
self.nuclei_templates_dir,
"-rate-limit",
self.ratelimit,
@@ -195,35 +189,39 @@ def execute_nuclei(self, nuclei_input):
command.append("-t")
command.append(self.budget_templates_file)
- with open(self.stats_file, "w") as stats_file:
- for line in self.helpers.run_live(command, input=nuclei_input, stderr=stats_file):
- try:
- j = json.loads(line)
- except json.decoder.JSONDecodeError:
- self.debug(f"Failed to decode line: {line}")
- continue
-
- template = j.get("template-id", "")
-
- # try to get the specific matcher name
- name = j.get("matcher-name", "")
-
- # fall back to regular name
- if not name:
- self.debug(
- f"Couldn't get matcher-name from nuclei json, falling back to regular name. Template: [{template}]"
- )
- name = j.get("info", {}).get("name", "")
-
- severity = j.get("info", {}).get("severity", "").upper()
- host = j.get("host", "")
-
- extracted_results = j.get("extracted-results", [])
-
- if template and name and severity and host:
- yield (severity, template, host, name, extracted_results)
- else:
- self.debug("Nuclei result missing one or more required elements, not reporting. JSON: ({j})")
+ stats_file = self.helpers.tempfile_tail(callback=self.log_nuclei_status)
+ try:
+ with open(stats_file, "w") as stats_fh:
+ for line in self.helpers.run_live(command, input=nuclei_input, stderr=stats_fh):
+ try:
+ j = json.loads(line)
+ except json.decoder.JSONDecodeError:
+ self.debug(f"Failed to decode line: {line}")
+ continue
+
+ template = j.get("template-id", "")
+
+ # try to get the specific matcher name
+ name = j.get("matcher-name", "")
+
+ # fall back to regular name
+ if not name:
+ self.debug(
+ f"Couldn't get matcher-name from nuclei json, falling back to regular name. Template: [{template}]"
+ )
+ name = j.get("info", {}).get("name", "")
+
+ severity = j.get("info", {}).get("severity", "").upper()
+ host = j.get("host", "")
+
+ extracted_results = j.get("extracted-results", [])
+
+ if template and name and severity and host:
+ yield (severity, template, host, name, extracted_results)
+ else:
+ self.debug("Nuclei result missing one or more required elements, not reporting. JSON: ({j})")
+ finally:
+ stats_file.unlink()
def log_nuclei_status(self, line):
try:
@@ -297,7 +295,6 @@ def find_collapsable_templates(self):
if yf:
for paths in self.get_yaml_request_attr(yf, "path"):
if set(paths).issubset(self.budget_paths):
-
headers = self.get_yaml_request_attr(yf, "headers")
for header in headers:
if header:
diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py
index aa301d1c9f..1b7c25c140 100644
--- a/bbot/modules/deadly/vhost.py
+++ b/bbot/modules/deadly/vhost.py
@@ -2,10 +2,9 @@
class vhost(BaseModule):
-
watched_events = ["URL"]
produced_events = ["VHOST", "DNS_NAME"]
- flags = ["active", "brute-force", "aggressive", "slow", "web-advanced"]
+ flags = ["active", "aggressive", "slow"]
meta = {"description": "Fuzz for virtual hosts"}
special_vhost_list = ["127.0.0.1", "localhost", "host.docker.internal"]
@@ -21,7 +20,7 @@ class vhost(BaseModule):
{
"name": "Download ffuf",
"unarchive": {
- "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_linux_amd64.tar.gz",
+ "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz",
"include": "ffuf",
"dest": "#{BBOT_TOOLS}",
"remote_src": True,
diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py
index b2aa44a409..8333e8293b 100644
--- a/bbot/modules/dnscommonsrv.py
+++ b/bbot/modules/dnscommonsrv.py
@@ -106,4 +106,4 @@ def handle_event(self, event):
queries = [event.data] + [f"{srv}.{event.data}" for srv in common_srvs]
for query, results in self.helpers.resolve_batch(queries, type="srv"):
if results:
- self.emit_event(query, "DNS_NAME", tags=["srv_record"], source=event)
+ self.emit_event(query, "DNS_NAME", tags=["srv-record"], source=event)
diff --git a/bbot/modules/dnsdumpster.py b/bbot/modules/dnsdumpster.py
index 45eb4591ed..82ce316bb2 100644
--- a/bbot/modules/dnsdumpster.py
+++ b/bbot/modules/dnsdumpster.py
@@ -10,14 +10,14 @@ class dnsdumpster(crobat):
flags = ["subdomain-enum", "passive", "safe"]
meta = {"description": "Query dnsdumpster for subdomains"}
- deps_pip = ["beautifulsoup4", "lxml"]
+ deps_pip = ["bs4", "lxml"]
base_url = "https://dnsdumpster.com"
def query(self, domain):
ret = []
# first, get the CSRF tokens
- res1 = self.helpers.request(self.base_url)
+ res1 = self.request_with_fail_count(self.base_url)
status_code = getattr(res1, "status_code", 0)
if status_code in [429]:
self.verbose(f'Too many requests "{status_code}"')
@@ -56,7 +56,7 @@ def query(self, domain):
# Otherwise, do the needful
subdomains = set()
- res2 = self.helpers.request(
+ res2 = self.request_with_fail_count(
f"{self.base_url}/",
method="POST",
cookies={"csrftoken": csrftoken},
diff --git a/bbot/modules/dnszonetransfer.py b/bbot/modules/dnszonetransfer.py
index cfda3b24e0..6e56284f92 100644
--- a/bbot/modules/dnszonetransfer.py
+++ b/bbot/modules/dnszonetransfer.py
@@ -5,7 +5,6 @@
class dnszonetransfer(BaseModule):
-
flags = ["subdomain-enum", "active", "safe"]
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py
index a54325ad79..fa47f23cf5 100644
--- a/bbot/modules/emailformat.py
+++ b/bbot/modules/emailformat.py
@@ -16,7 +16,7 @@ def extract_emails(self, content):
def handle_event(self, event):
_, query = self.helpers.split_domain(event.data)
url = f"{self.base_url}/d/{self.helpers.quote(query)}/"
- r = self.helpers.request(url)
+ r = self.request_with_fail_count(url)
if not r:
return
for email in self.extract_emails(r.text):
diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py
index 525adaa97c..d3d469654f 100644
--- a/bbot/modules/ffuf_shortnames.py
+++ b/bbot/modules/ffuf_shortnames.py
@@ -1,30 +1,61 @@
+import re
import random
import string
from bbot.modules.deadly.ffuf import ffuf
-class ffuf_shortnames(ffuf):
+def find_common_prefixes(strings, minimum_set_length=4):
+ prefix_candidates = [s[:i] for s in strings if len(s) == 6 for i in range(3, 6)]
+ frequency_dict = {item: prefix_candidates.count(item) for item in prefix_candidates}
+ frequency_dict = {k: v for k, v in frequency_dict.items() if v >= minimum_set_length}
+ prefix_list = list(set(frequency_dict.keys()))
+
+ found_prefixes = set()
+ for prefix in prefix_list:
+ prefix_frequency = frequency_dict[prefix]
+ is_substring = False
+
+ for k, v in frequency_dict.items():
+ if prefix != k:
+ if prefix in k:
+ is_substring = True
+ if not is_substring:
+ found_prefixes.add(prefix)
+ else:
+ if prefix_frequency > v and (len(k) - len(prefix) == 1):
+ found_prefixes.add(prefix)
+ return list(found_prefixes)
+
+class ffuf_shortnames(ffuf):
watched_events = ["URL_HINT"]
- produced_events = ["URL"]
- flags = ["brute-force", "aggressive", "active", "web-advanced", "iis-shortnames"]
+ produced_events = ["URL_UNVERIFIED"]
+ flags = ["aggressive", "active", "iis-shortnames", "web-thorough"]
meta = {"description": "Use ffuf in combination IIS shortnames"}
options = {
- "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-large-words.txt",
- "lines": 20000,
+ "wordlist": "", # default is defined within setup function
+ "wordlist_extensions": "", # default is defined within setup function
+ "lines": 1000000,
"max_depth": 1,
"version": "1.5.0",
"extensions": "",
+ "ignore_redirects": True,
+ "find_common_prefixes": False,
+ "find_delimeters": True,
}
options_desc = {
"wordlist": "Specify wordlist to use when finding directories",
+ "wordlist_extensions": "Specify wordlist to use when making extension lists",
"lines": "take only the first N lines from the wordlist when finding directories",
"max_depth": "the maxium directory depth to attempt to solve",
"version": "ffuf version",
"extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)",
+ "ignore_redirects": "Explicitly ignore redirects (301,302)",
+ "find_common_prefixes": "Attempt to automatically detect common prefixes and make additional ffuf runs against them",
+ "find_delimeters": "Attempt to detect common delimeters and make additional ffuf runs against them",
}
in_scope_only = True
@@ -33,7 +64,7 @@ class ffuf_shortnames(ffuf):
{
"name": "Download ffuf",
"unarchive": {
- "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_linux_amd64.tar.gz",
+ "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_#{BBOT_OS_PLATFORM}_#{BBOT_CPU_ARCH}.tar.gz",
"include": "ffuf",
"dest": "#{BBOT_TOOLS}",
"remote_src": True,
@@ -41,46 +72,154 @@ class ffuf_shortnames(ffuf):
}
]
- extension_helper = {
- "asp": ["aspx"],
- "asm": ["asmx"],
- "ash": ["ashx"],
- "jsp": ["jspx"],
- "htm": ["html"],
- "sht": ["shtml"],
- "php": ["php2", "php3", "php4", "ph5"],
- }
-
def setup(self):
self.sanity_canary = "".join(random.choice(string.ascii_lowercase) for i in range(10))
wordlist = self.config.get("wordlist", "")
+ if not wordlist:
+ wordlist = f"{self.helpers.wordlist_dir}/ffuf_shortname_candidates.txt"
+ self.debug(f"Using [{wordlist}] for shortname candidate list")
self.wordlist = self.helpers.wordlist(wordlist)
+ f = open(self.wordlist, "r")
+ self.wordlist_lines = f.readlines()
+ f.close()
+
+ wordlist_extensions = self.config.get("wordlist_extensions", "")
+ if not wordlist_extensions:
+ wordlist_extensions = f"{self.helpers.wordlist_dir}/raft-small-extensions-lowercase_CLEANED.txt"
+ self.debug(f"Using [{wordlist_extensions}] for shortname candidate extension list")
+ self.wordlist_extensions = self.helpers.wordlist(wordlist_extensions)
self.extensions = self.config.get("extensions")
- return True
+ self.ignore_redirects = self.config.get("ignore_redirects")
- def handle_event(self, event):
-
- filename_hint = event.parsed.path.rsplit(".", 1)[0].split("/")[-1]
-
- tempfile = self.generate_templist(self.wordlist, prefix=filename_hint)
-
- root_stub = "/".join(event.parsed.path.split("/")[:-1])
- root_url = f"{event.parsed.scheme}://{event.parsed.netloc}{root_stub}/"
-
- if "file" in event.tags:
- extension_hint = event.parsed.path.rsplit(".", 1)[1]
- used_extensions = []
- used_extensions.append(extension_hint)
- for ex in self.extension_helper.keys():
- if extension_hint == ex:
- for ex2 in self.extension_helper[ex]:
- used_extensions.append(ex2)
-
- for ext in used_extensions:
- for r in self.execute_ffuf(tempfile, event, root_url, suffix=f".{ext}"):
- self.emit_event(r["url"], "URL", source=event, tags=[f"status-{r['status']}"])
+ self.per_host_collection = {}
+ self.shortname_to_event = {}
+ return True
- elif "dir" in event.tags:
+ def build_extension_list(self, event):
+ used_extensions = []
+ extension_hint = event.parsed.path.rsplit(".", 1)[1].lower().strip()
+ with open(self.wordlist_extensions) as f:
+ for l in f:
+ l = l.lower().lstrip(".")
+ if l.lower().startswith(extension_hint):
+ used_extensions.append(l.strip())
+
+ return used_extensions
+
+ def find_delimeter(self, hint):
+ delimeters = ["_", "-"]
+ for d in delimeters:
+ if d in hint:
+ if not hint.startswith(d) and not hint.endswith(d):
+ return d, hint.split(d)[0], hint.split(d)[1]
+ return None
- for r in self.execute_ffuf(tempfile, event, root_url):
- self.emit_event(r["url"], "URL", source=event, tags=[f"status-{r['status']}"])
+ def handle_event(self, event):
+ if event.source.type == "URL":
+ filename_hint = re.sub(r"~\d", "", event.parsed.path.rsplit(".", 1)[0].split("/")[-1]).lower()
+
+ host = f"{event.source.parsed.scheme}://{event.source.parsed.netloc}/"
+ if host not in self.per_host_collection.keys():
+ self.per_host_collection[host] = [(filename_hint, event.source.data)]
+
+ else:
+ self.per_host_collection[host].append((filename_hint, event.source.data))
+
+ self.shortname_to_event[filename_hint] = event
+
+ root_stub = "/".join(event.parsed.path.split("/")[:-1])
+ root_url = f"{event.parsed.scheme}://{event.parsed.netloc}{root_stub}/"
+
+ if "shortname-file" in event.tags:
+ used_extensions = self.build_extension_list(event)
+
+ if len(filename_hint) == 6:
+ tempfile, tempfile_len = self.generate_templist(prefix=filename_hint)
+ self.verbose(
+ f"generated temp word list of size [{str(tempfile_len)}] for filename hint: [{filename_hint}]"
+ )
+
+ else:
+ tempfile = self.helpers.tempfile([filename_hint], pipe=False)
+ tempfile_len = 1
+
+ if tempfile_len > 0:
+ if "shortname-file" in event.tags:
+ for ext in used_extensions:
+ for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"):
+ self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"])
+
+ elif "shortname-directory" in event.tags:
+ for r in self.execute_ffuf(tempfile, root_url):
+ self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"])
+
+ if self.config.get("find_delimeters"):
+ if "shortname-directory" in event.tags:
+ delimeter_r = self.find_delimeter(filename_hint)
+ if delimeter_r:
+ delimeter, prefix, partial_hint = delimeter_r
+ self.verbose(f"Detected delimeter [{delimeter}] in hint [{filename_hint}]")
+ tempfile, tempfile_len = self.generate_templist(prefix=partial_hint)
+ for r in self.execute_ffuf(tempfile, root_url, prefix=f"{prefix}{delimeter}"):
+ self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"])
+
+ elif "shortname-file" in event.tags:
+ for ext in used_extensions:
+ delimeter_r = self.find_delimeter(filename_hint)
+ if delimeter_r:
+ delimeter, prefix, partial_hint = delimeter_r
+ self.verbose(f"Detected delimeter [{delimeter}] in hint [{filename_hint}]")
+ tempfile, tempfile_len = self.generate_templist(prefix=partial_hint)
+ for r in self.execute_ffuf(
+ tempfile, root_url, prefix=f"{prefix}{delimeter}", suffix=f".{ext}"
+ ):
+ self.emit_event(
+ r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]
+ )
+
+ def finish(self):
+ if self.config.get("find_common_prefixes"):
+ per_host_collection = dict(self.per_host_collection)
+ self.per_host_collection.clear()
+
+ for host, hint_tuple_list in per_host_collection.items():
+ hint_list = [x[0] for x in hint_tuple_list]
+
+ common_prefixes = find_common_prefixes(hint_list)
+ for prefix in common_prefixes:
+ self.verbose(f"Found common prefix: [{prefix}] for host [{host}]")
+ for hint_tuple in hint_tuple_list:
+ hint, url = hint_tuple
+ if hint.startswith(prefix):
+ partial_hint = hint[len(prefix) :]
+
+ # safeguard to prevent loading the entire wordlist
+ if len(partial_hint) > 0:
+ tempfile, tempfile_len = self.generate_templist(prefix=partial_hint)
+
+ if "shortname-directory" in self.shortname_to_event[hint].tags:
+ self.verbose(
+ f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix} and partial_hint: {partial_hint}"
+ )
+
+ for r in self.execute_ffuf(tempfile, url, prefix=prefix):
+ self.emit_event(
+ r["url"],
+ "URL_UNVERIFIED",
+ source=self.shortname_to_event[hint],
+ tags=[f"status-{r['status']}"],
+ )
+ elif "shortname-file" in self.shortname_to_event[hint].tags:
+ used_extensions = self.build_extension_list(self.shortname_to_event[hint])
+
+ for ext in used_extensions:
+ self.verbose(
+ f"Running common prefix check for URL_HINT: {hint} with prefix: {prefix}, extension: .{ext}, and partial_hint: {partial_hint}"
+ )
+ for r in self.execute_ffuf(tempfile, url, prefix=prefix, suffix=f".{ext}"):
+ self.emit_event(
+ r["url"],
+ "URL_UNVERIFIED",
+ source=self.shortname_to_event[hint],
+ tags=[f"status-{r['status']}"],
+ )
diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py
new file mode 100644
index 0000000000..e6b76227e0
--- /dev/null
+++ b/bbot/modules/fingerprintx.py
@@ -0,0 +1,55 @@
+import json
+import subprocess
+from bbot.modules.base import BaseModule
+
+
+class fingerprintx(BaseModule):
+ watched_events = ["OPEN_TCP_PORT"]
+ produced_events = ["PROTOCOL"]
+ flags = ["active", "safe", "service-enum", "slow"]
+ meta = {"description": "Fingerprint exposed services like RDP, SSH, MySQL, etc."}
+ options = {"version": "1.1.4"}
+ options_desc = {"version": "fingerprintx version"}
+ batch_size = 10
+ max_event_handlers = 2
+ _priority = 2
+
+ deps_ansible = [
+ {
+ "name": "Download fingerprintx",
+ "unarchive": {
+ "src": "https://github.com/praetorian-inc/fingerprintx/releases/download/v#{BBOT_MODULES_FINGERPRINTX_VERSION}/fingerprintx_#{BBOT_MODULES_FINGERPRINTX_VERSION}_#{BBOT_OS_PLATFORM}_#{BBOT_CPU_ARCH}.tar.gz",
+ "include": "fingerprintx",
+ "dest": "#{BBOT_TOOLS}",
+ "remote_src": True,
+ },
+ },
+ ]
+
+ def handle_batch(self, *events):
+ _input = {e.data: e for e in events}
+ command = ["fingerprintx", "--json"]
+ for line in self.helpers.run_live(command, input=list(_input), stderr=subprocess.DEVNULL):
+ try:
+ j = json.loads(line)
+ except Exception as e:
+ self.debug(f'Error parsing line "{line}" as JSON: {e}')
+ break
+ ip = j.get("ip", "")
+ host = j.get("host", ip)
+ port = str(j.get("port", ""))
+ banner = j.get("metadata", {}).get("banner", "").strip()
+ if port:
+ port_data = f"{host}:{port}"
+ protocol = j.get("protocol", "")
+ tags = set()
+ if host and ip:
+ tags.add(f"ip-{ip}")
+ if host and port and protocol:
+ source_event = _input.get(port_data)
+ protocol_data = {"host": host, "protocol": protocol.upper()}
+ if port:
+ protocol_data["port"] = port
+ if banner:
+ protocol_data["banner"] = banner
+ self.emit_event(protocol_data, "PROTOCOL", source=source_event, tags=tags)
diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py
index 8e3e44c956..e0c051c561 100644
--- a/bbot/modules/fullhunt.py
+++ b/bbot/modules/fullhunt.py
@@ -18,13 +18,13 @@ def setup(self):
def ping(self):
url = f"{self.base_url}/auth/status"
- j = self.helpers.request(url, headers=self.headers).json()
+ j = self.request_with_fail_count(url, headers=self.headers).json()
remaining = j["user_credits"]["remaining_credits"]
assert remaining > 0, "No credits remaining"
def request_url(self, query):
url = f"{self.base_url}/domain/{self.helpers.quote(query)}/subdomains"
- return self.helpers.request(url, headers=self.headers)
+ return self.request_with_fail_count(url, headers=self.headers)
def parse_results(self, r, query):
return r.json().get("hosts", [])
diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py
index ce7c2811b0..db0aa2a3c6 100644
--- a/bbot/modules/generic_ssrf.py
+++ b/bbot/modules/generic_ssrf.py
@@ -35,7 +35,6 @@
class BaseSubmodule:
-
technique_description = "base technique description"
severity = "INFO"
paths = None
@@ -80,7 +79,6 @@ def process(self, event, r, subdomain_tag):
class Generic_SSRF(BaseSubmodule):
-
technique_description = "Generic SSRF (GET)"
severity = "HIGH"
@@ -88,7 +86,6 @@ def set_base_url(self, event):
return event.data
def create_paths(self):
-
query_string = ""
for param in ssrf_params:
query_string += f"{param}=http://SSRF_CANARY&"
@@ -101,7 +98,6 @@ def create_paths(self):
class Generic_SSRF_POST(BaseSubmodule):
-
technique_description = "Generic SSRF (POST)"
severity = "HIGH"
@@ -109,7 +105,6 @@ def set_base_url(self, event):
return event.data
def test(self, event):
-
test_url = f"{event.data}"
subdomain_tag = self.parent_module.helpers.rand_string(4, digits=False)
@@ -131,13 +126,11 @@ def test(self, event):
class Generic_XXE(BaseSubmodule):
-
technique_description = "Generic XXE"
severity = "HIGH"
paths = None
def test(self, event):
-
rand_entity = self.parent_module.helpers.rand_string(4, digits=False)
subdomain_tag = self.parent_module.helpers.rand_string(4, digits=False)
@@ -156,17 +149,15 @@ def test(self, event):
class generic_ssrf(BaseModule):
-
watched_events = ["URL"]
produced_events = ["VULNERABILITY"]
- flags = ["active", "aggressive", "web-advanced"]
+ flags = ["active", "aggressive", "web-thorough"]
meta = {"description": "Check for generic SSRFs"}
in_scope_only = True
deps_apt = ["curl"]
def setup(self):
-
self.submodules = {}
self.interactsh_subdomain_tags = {}
self.severity = None
diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index 8936229969..8759d94074 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -33,6 +33,7 @@ class gowitness(BaseModule):
"package": {"name": "chromium", "state": "present"},
"become": True,
"when": "ansible_facts['os_family'] != 'Debian'",
+ "ignore_errors": True,
},
{
"name": "Install Chromium dependencies (Debian)",
@@ -42,6 +43,7 @@ class gowitness(BaseModule):
},
"become": True,
"when": "ansible_facts['os_family'] == 'Debian'",
+ "ignore_errors": True,
},
{
"name": "Get latest Chromium version (Debian)",
@@ -51,6 +53,7 @@ class gowitness(BaseModule):
},
"register": "chromium_version",
"when": "ansible_facts['os_family'] == 'Debian'",
+ "ignore_errors": True,
},
{
"name": "Download Chromium (Debian)",
@@ -61,11 +64,12 @@ class gowitness(BaseModule):
"creates": "#{BBOT_TOOLS}/chrome-linux",
},
"when": "ansible_facts['os_family'] == 'Debian'",
+ "ignore_errors": True,
},
{
"name": "Download gowitness",
"get_url": {
- "url": "https://github.com/sensepost/gowitness/releases/download/#{BBOT_MODULES_GOWITNESS_VERSION}/gowitness-#{BBOT_MODULES_GOWITNESS_VERSION}-linux-amd64",
+ "url": "https://github.com/sensepost/gowitness/releases/download/#{BBOT_MODULES_GOWITNESS_VERSION}/gowitness-#{BBOT_MODULES_GOWITNESS_VERSION}-#{BBOT_OS_PLATFORM}-#{BBOT_CPU_ARCH}",
"dest": "#{BBOT_TOOLS}/gowitness",
"mode": "755",
},
diff --git a/bbot/modules/hackertarget.py b/bbot/modules/hackertarget.py
index 05bf6828cf..38ff695818 100644
--- a/bbot/modules/hackertarget.py
+++ b/bbot/modules/hackertarget.py
@@ -10,7 +10,7 @@ class hackertarget(crobat):
base_url = "https://api.hackertarget.com"
def request_url(self, query):
- return self.helpers.request(f"{self.base_url}/hostsearch/?q={self.helpers.quote(query)}")
+ return self.request_with_fail_count(f"{self.base_url}/hostsearch/?q={self.helpers.quote(query)}")
def parse_results(self, r, query):
for line in r.text.splitlines():
diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py
index e349b8149a..f6d28fee69 100644
--- a/bbot/modules/host_header.py
+++ b/bbot/modules/host_header.py
@@ -3,10 +3,9 @@
class host_header(BaseModule):
-
watched_events = ["HTTP_RESPONSE"]
produced_events = ["FINDING"]
- flags = ["active", "aggressive", "web-advanced"]
+ flags = ["active", "aggressive", "web-thorough"]
meta = {"description": "Try common HTTP Host header spoofing techniques"}
in_scope_only = True
@@ -14,7 +13,6 @@ class host_header(BaseModule):
deps_apt = ["curl"]
def setup(self):
-
self.interactsh_subdomain_tags = {}
if self.scan.config.get("interactsh_disable", False) == False:
try:
@@ -65,15 +63,12 @@ def cleanup(self):
self.warning(f"Interactsh failure: {e}")
def handle_event(self, event):
-
# get any set-cookie responses from the response and add them to the request
added_cookies = {}
for k, v in event.data["header-dict"].items():
-
if k.lower() == "set-cookie":
-
cookie_string = v
cookie_split = cookie_string.split("=")
added_cookies = {cookie_split[0]: cookie_split[1]}
@@ -164,7 +159,6 @@ def handle_event(self, event):
# emit all the domain reflections we found
for dr in domain_reflections:
-
self.emit_event(
{
"host": str(event.host),
diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py
index 9ee5d18eb3..8a941ac6d1 100644
--- a/bbot/modules/httpx.py
+++ b/bbot/modules/httpx.py
@@ -4,10 +4,9 @@
class httpx(BaseModule):
-
watched_events = ["OPEN_TCP_PORT", "URL_UNVERIFIED", "URL"]
produced_events = ["URL", "HTTP_RESPONSE"]
- flags = ["active", "safe", "web-basic"]
+ flags = ["active", "safe", "web-basic", "web-thorough", "subdomain-enum"]
meta = {"description": "Visit webpages. Many other modules rely on httpx"}
batch_size = 500
@@ -21,7 +20,7 @@ class httpx(BaseModule):
{
"name": "Download httpx",
"unarchive": {
- "src": "https://github.com/projectdiscovery/httpx/releases/download/v#{BBOT_MODULES_HTTPX_VERSION}/httpx_#{BBOT_MODULES_HTTPX_VERSION}_linux_amd64.zip",
+ "src": "https://github.com/projectdiscovery/httpx/releases/download/v#{BBOT_MODULES_HTTPX_VERSION}/httpx_#{BBOT_MODULES_HTTPX_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.zip",
"include": "httpx",
"dest": "#{BBOT_TOOLS}",
"remote_src": True,
@@ -30,6 +29,7 @@ class httpx(BaseModule):
]
scope_distance_modifier = 0
+ _priority = 2
def setup(self):
self.timeout = self.scan.config.get("httpx_timeout", 5)
@@ -39,7 +39,6 @@ def setup(self):
return True
def filter_event(self, event):
-
if "_wildcard" in str(event.host).split("."):
return False
@@ -60,7 +59,6 @@ def filter_event(self, event):
return True
def handle_batch(self, *events):
-
stdin = {}
for e in events:
url_hash = None
@@ -98,6 +96,8 @@ def handle_batch(self, *events):
# "-r",
# self.helpers.resolver_file,
]
+ for hk, hv in self.scan.config.get("http_headers", {}).items():
+ command += ["-header", f"{hk}: {hv}"]
proxy = self.scan.config.get("http_proxy", "")
if proxy:
command += ["-http-proxy", proxy]
@@ -127,10 +127,16 @@ def handle_batch(self, *events):
# main URL
httpx_ip = j.get("host", "unknown")
- url_event = self.make_event(url, "URL", source_event, tags=[f"status-{status_code}", f"ip-{httpx_ip}"])
+ tags = [f"status-{status_code}", f"ip-{httpx_ip}"]
+ title = self.helpers.tagify(j.get("title", ""))
+ if title:
+ tags.append(f"http-title-{title}")
+ url_event = self.make_event(url, "URL", source_event, tags=tags)
if url_event and not "httpx-only" in url_event.tags:
if url_event != source_event:
self.emit_event(url_event)
+ else:
+ url_event._resolved.set()
# HTTP response
self.emit_event(j, "HTTP_RESPONSE", url_event, internal=True)
diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py
index 8790f6477b..88448c8e55 100644
--- a/bbot/modules/hunt.py
+++ b/bbot/modules/hunt.py
@@ -124,13 +124,12 @@ class hunt(BaseModule):
watched_events = ["HTTP_RESPONSE"]
produced_events = ["FINDING"]
- flags = ["active", "safe", "web-advanced"]
+ flags = ["active", "safe", "web-basic", "web-thorough"]
meta = {"description": "Watch for commonly-exploitable HTTP parameters"}
# accept all events regardless of scope distance
scope_distance_modifier = None
def extract_params(self, body):
-
# check for input tags
input_tag = self.input_tag_regex.findall(body)
diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py
index 46aeadd19e..845488844c 100644
--- a/bbot/modules/hunterio.py
+++ b/bbot/modules/hunterio.py
@@ -2,7 +2,6 @@
class hunterio(shodan_dns):
-
watched_events = ["DNS_NAME"]
produced_events = ["EMAIL_ADDRESS", "DNS_NAME", "URL_UNVERIFIED"]
flags = ["passive", "email-enum", "subdomain-enum", "safe"]
diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py
index 42b746932f..54d5582498 100644
--- a/bbot/modules/iis_shortnames.py
+++ b/bbot/modules/iis_shortnames.py
@@ -1,130 +1,217 @@
+import re
+from threading import Lock
+
from bbot.modules.base import BaseModule
+valid_chars = "ETAONRISHDLFCMUGYPWBVKJXQZ0123456789_-$~()&!#%'@^`{}]]"
-class iis_shortnames(BaseModule):
+def encode_all(string):
+ return "".join("%{0:0>2}".format(format(ord(char), "x")) for char in string)
+
+
+class iis_shortnames(BaseModule):
watched_events = ["URL"]
produced_events = ["URL_HINT"]
- flags = ["active", "safe", "web-basic", "iis-shortnames"]
+ flags = ["active", "safe", "web-basic", "web-thorough", "iis-shortnames"]
meta = {"description": "Check for IIS shortname vulnerability"}
- options = {"detect_only": True, "threads": 8}
+ options = {"detect_only": True, "max_node_count": 30}
options_desc = {
"detect_only": "Only detect the vulnerability and do not run the shortname scanner",
- "threads": "the number of threads to run concurrently when executing the IIS shortname scanner",
+ "max_node_count": "Limit how many nodes to attempt to resolve on any given recursion branch",
}
in_scope_only = True
- deps_ansible = [
- {
- "name": "Install Java JRE (Debian)",
- "become": True,
- "package": {"name": "default-jre", "state": "latest"},
- "when": """ansible_facts['os_family'] == 'Debian'""",
- },
- {
- "name": "Install Java JRE (RedHat)",
- "become": True,
- "package": {"name": "java-latest-openjdk", "state": "latest"},
- "when": """ansible_facts['os_family'] == 'RedHat'""",
- },
- {
- "name": "Install Java JRE (Archlinux)",
- "package": {"name": "jre-openjdk", "state": "present"},
- "become": True,
- "when": """ansible_facts['os_family'] == 'Archlinux'""",
- },
- ]
+ max_event_handlers = 8
+
+ def detect(self, target):
+ technique = None
+ detections = []
+ random_string = self.helpers.rand_string(8)
+ control_url = f"{target}{random_string}*~1*/a.aspx"
+ test_url = f"{target}*~1*/a.aspx"
+
+ for method in ["GET", "POST", "OPTIONS", "DEBUG", "HEAD", "TRACE"]:
+ control = self.helpers.request(method=method, url=control_url, allow_redirects=False, retries=2)
+ test = self.helpers.request(method=method, url=test_url, allow_redirects=False, retries=2)
+ if (control != None) and (test != None):
+ if control.status_code != test.status_code:
+ technique = f"{str(control.status_code)}/{str(test.status_code)} HTTP Code"
+ detections.append((method, test.status_code, technique))
+
+ elif ("Error Code| 0x80070002" in control.text) and (
+ "Error Code | 0x00000000" in test.text
+ ):
+ detections.append((method, 0, technique))
+ technique = "HTTP Body Error Message"
+ return detections
def setup(self):
- iis_shortname_jar = (
- "https://github.com/irsdl/IIS-ShortName-Scanner/raw/master/release/iis_shortname_scanner.jar"
- )
-
- iis_shortname_config = (
- "https://raw.githubusercontent.com/irsdl/IIS-ShortName-Scanner/master/release/config.xml"
- )
- self.iis_scanner_jar = self.helpers.download(iis_shortname_jar, cache_hrs=720)
- self.iis_scanner_config = self.helpers.download(iis_shortname_config, cache_hrs=720)
- if self.iis_scanner_jar and self.iis_scanner_config:
+ self.scanned_tracker_lock = Lock()
+ self.scanned_tracker = set()
+ return True
+
+ @staticmethod
+ def normalize_url(url):
+ return str(url.rstrip("/") + "/").lower()
+
+ def directory_confirm(self, target, method, url_hint, affirmative_status_code):
+ payload = encode_all(f"{url_hint}")
+ url = f"{target}{payload}"
+ directory_confirm_result = self.helpers.request(method=method, url=url, allow_redirects=False, retries=2)
+
+ if directory_confirm_result.status_code == affirmative_status_code:
return True
- return False
+ else:
+ return False
+
+ def duplicate_check(self, target, method, url_hint, affirmative_status_code):
+ duplicates = []
+ count = 2
+ base_hint = re.sub(r"~\d", "", url_hint)
+ suffix = "\\a.aspx"
+
+ while 1:
+ payload = encode_all(f"{base_hint}~{str(count)}*")
+ url = f"{target}{payload}{suffix}"
+
+ duplicate_check_results = self.helpers.request(method=method, url=url, allow_redirects=False, retries=2)
+ if duplicate_check_results.status_code != affirmative_status_code:
+ break
+ else:
+ duplicates.append(f"{base_hint}~{str(count)}")
+ count += 1
+
+ if count > 5:
+ self.warning("Found more than 5 files with the same shortname. Will stop further duplicate checking.")
+ break
+
+ return duplicates
+
+ def threaded_request(self, method, url, affirmative_status_code):
+ r = self.helpers.request(method=method, url=url, allow_redirects=False, retries=2)
+ if r is not None:
+ if r.status_code == affirmative_status_code:
+ return True
+
+ def solve_shortname_recursive(
+ self, method, target, prefix, affirmative_status_code, extension_mode=False, node_count=0
+ ):
+ url_hint_list = []
+ found_results = False
+
+ futures = {}
+ for c in valid_chars:
+ suffix = "\\a.aspx"
+ wildcard = "*" if extension_mode else "*~1*"
+ payload = encode_all(f"{prefix}{c}{wildcard}")
+ url = f"{target}{payload}{suffix}"
+ future = self.submit_task(self.threaded_request, method, url, affirmative_status_code)
+ futures[future] = c
+
+ for future in self.helpers.as_completed(futures):
+ c = futures[future]
+ result = future.result()
+ if result:
+ found_results = True
+ node_count += 1
+ self.verbose(f"node_count: {str(node_count)} for node: {target}")
+ if node_count > self.config.get("max_node_count"):
+ self.warning(
+ f"iis_shortnames: max_node_count ({str(self.config.get('max_node_count'))}) exceeded for node: {target}. Affected branch will be terminated."
+ )
+ return url_hint_list
+
+ # check to make sure the file isn't shorter than 6 characters
+ wildcard = "~1*"
+ payload = encode_all(f"{prefix}{c}{wildcard}")
+ url = f"{target}{payload}{suffix}"
+ r = self.helpers.request(method=method, url=url, allow_redirects=False, retries=2)
+ if r is not None:
+ if r.status_code == affirmative_status_code:
+ url_hint_list.append(f"{prefix}{c}")
+
+ url_hint_list += self.solve_shortname_recursive(
+ method, target, f"{prefix}{c}", affirmative_status_code, extension_mode, node_count=node_count
+ )
+ if len(prefix) > 0 and found_results == False:
+ url_hint_list.append(f"{prefix}")
+ self.verbose(f"Found new (possibly partial) URL_HINT: {prefix} from node {target}")
+ return url_hint_list
def handle_event(self, event):
+ normalized_url = self.normalize_url(event.data)
+ with self.scanned_tracker_lock:
+ self.scanned_tracker.add(normalized_url)
- normalized_url = event.data.rstrip("/") + "/"
- result = self.detect(normalized_url)
+ detections = self.detect(normalized_url)
- if result:
- description = f"IIS Shortname Vulnerability"
+ technique_strings = []
+ if detections:
+ for detection in detections:
+ method, affirmative_status_code, technique = detection
+ technique_strings.append(f"{method} ({technique})")
+
+ description = f"IIS Shortname Vulnerability Detected. Potentially Vulnerable Method/Techniques: [{','.join(technique_strings)}]"
self.emit_event(
{"severity": "LOW", "host": str(event.host), "url": normalized_url, "description": description},
"VULNERABILITY",
event,
)
if not self.config.get("detect_only"):
- command = [
- "java",
- "-jar",
- self.iis_scanner_jar,
- "0",
- str(self.config.get("threads", 8)),
- normalized_url,
- self.iis_scanner_config,
- ]
- output = self.helpers.run(command).stdout
- self.debug(output)
- discovered_directories, discovered_files = self.shortname_parse(output)
- for d in discovered_directories:
- if d[-2] == "~":
- d = d.split("~")[:-1][0]
- self.emit_event(normalized_url + d, "URL_HINT", event, tags=["directory"])
- for f in discovered_files:
- if f[-2] == "~":
- f = f.split("~")[:-1][0]
- self.emit_event(normalized_url + f, "URL_HINT", event, tags=["file"])
-
- def detect(self, url):
-
- detected = False
- http_methods = ["GET", "OPTIONS", "DEBUG"]
- for http_method in http_methods:
- dir_name = self.helpers.rand_string(8)
- file_name = self.helpers.rand_string(1)
- control_url = url.rstrip("/") + "/" + f"{dir_name}*~1*/{file_name}.aspx"
- control = self.helpers.request(control_url, method=http_method)
- test_url = url.rstrip("/") + "/" + f"*~1*/{file_name}.aspx"
- test = self.helpers.request(test_url, method=http_method)
- if (control != None) and (test != None):
- if (control.status_code != 404) and (test.status_code == 404):
- detected = True
- return detected
-
- def shortname_parse(self, output):
- discovered_directories = []
- discovered_files = []
- parseLines = output.split("\n")
- inDirectories = False
- inFiles = False
- for idx, line in enumerate(parseLines):
- if "Identified directories" in line:
- inDirectories = True
- elif "Indentified files" in line:
- inFiles = True
- inDirectories = False
- elif ":" in line:
- pass
- elif "Actual" in line:
- pass
- else:
- if inFiles == True:
- if len(line) > 0:
- shortname = line.split(" ")[-1].split(".")[0].split("~")[0]
- extension = line.split(" ")[-1].split(".")[1]
- if "?" not in extension:
- discovered_files.append(f"{shortname}.{extension}".lower())
-
- elif inDirectories == True:
- if len(line) > 0:
- shortname = line.split(" ")[-1]
- discovered_directories.append(shortname.lower())
- return discovered_directories, discovered_files
+ for detection in detections:
+ method, affirmative_status_code, technique = detection
+ valid_method_confirmed = False
+
+ if valid_method_confirmed:
+ break
+
+ file_name_hints = list(
+ set(self.solve_shortname_recursive(method, normalized_url, "", affirmative_status_code))
+ )
+ if len(file_name_hints) == 0:
+ continue
+ else:
+ valid_method_confirmed = True
+
+ file_name_hints = [f"{x}~1" for x in file_name_hints]
+ url_hint_list = []
+
+ file_name_hints_dedupe = file_name_hints[:]
+
+ for x in file_name_hints_dedupe:
+ duplicates = self.duplicate_check(normalized_url, method, x, affirmative_status_code)
+ if duplicates:
+ file_name_hints += duplicates
+
+ # check for the case of a folder and file with the same filename
+
+ for d in file_name_hints:
+ if self.directory_confirm(normalized_url, method, d, affirmative_status_code):
+ self.verbose(f"Confirmed Directory URL_HINT: {d} from node {normalized_url}")
+ url_hint_list.append(d)
+
+ for y in file_name_hints:
+ file_name_extension_hints = self.solve_shortname_recursive(
+ method, normalized_url, f"{y}.", affirmative_status_code, extension_mode=True
+ )
+ for z in file_name_extension_hints:
+ if z.endswith("."):
+ z = z.rstrip(".")
+ self.verbose(f"Found new file URL_HINT: {z} from node {normalized_url}")
+ url_hint_list.append(z)
+
+ for url_hint in url_hint_list:
+ if "." in url_hint:
+ hint_type = "shortname-file"
+ else:
+ hint_type = "shortname-directory"
+ self.emit_event(f"{normalized_url}/{url_hint}", "URL_HINT", event, tags=[hint_type])
+
+ def filter_event(self, event):
+ if "dir" in event.tags:
+ with self.scanned_tracker_lock:
+ if self.normalize_url(event.data) not in self.scanned_tracker:
+ return True
+ return False
+ return False
diff --git a/bbot/modules/internal/aggregate.py b/bbot/modules/internal/aggregate.py
index a8d3ba10d3..5e38347abb 100644
--- a/bbot/modules/internal/aggregate.py
+++ b/bbot/modules/internal/aggregate.py
@@ -1,11 +1,9 @@
-from bbot.modules.base import BaseModule
+from bbot.modules.report.base import BaseReportModule
-class aggregate(BaseModule):
- watched_events = ["SUMMARY"]
- produced_events = ["SUMMARY"]
+class aggregate(BaseReportModule):
flags = ["passive", "safe"]
- meta = {"description": "Report on scan statistics"}
+ meta = {"description": "Summarize statistics at the end of a scan"}
def report(self):
for table_row in str(self.scan.stats).splitlines():
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 292d7cc361..f2b11e8a58 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -3,8 +3,8 @@
import base64
import jwt as j
-from bbot.core.helpers.regexes import _email_regex
from bbot.modules.internal.base import BaseInternalModule
+from bbot.core.helpers.regexes import _email_regex, junk_remover
class BaseExtractor:
@@ -39,16 +39,18 @@ def __init__(self, excavate):
for i, t in enumerate(dns_targets):
if not any(x in dns_targets_set for x in excavate.helpers.domain_parents(t, include_self=True)):
dns_targets_set.add(t)
- self.regexes[f"dns_name_{i+1}"] = r"(%[a-fA-F0-9]{2})?((?:(?:[\w-]+)\.)+" + re.escape(t) + ")"
+ self.regexes[f"dns_name_{i+1}"] = junk_remover + r"((?:(?:[\w-]+)\.)+" + re.escape(t) + ")"
super().__init__(excavate)
def report(self, result, name, event, **kwargs):
- self.excavate.emit_event(result[1], "DNS_NAME", source=event)
+ self.excavate.emit_event(result, "DNS_NAME", source=event)
class URLExtractor(BaseExtractor):
regexes = {
- "fullurl": r"https?://(?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]+)*/?",
+ "fullurl": r"(?i)"
+ + junk_remover
+ + r"(\w{2,15})://((?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]+)*/?)",
"a-tag": r"]*?\s+)?href=([\"'])(.*?)\1",
"script-tag": r"]*?\s+)?src=([\"'])(.*?)\1",
}
@@ -56,15 +58,18 @@ class URLExtractor(BaseExtractor):
prefix_blacklist = ["javascript:", "mailto:", "tel:"]
def report(self, result, name, event, **kwargs):
-
spider_danger = kwargs.get("spider_danger", True)
tags = []
parsed = getattr(event, "parsed", None)
- if (name == "a-tag" or name == "script-tag") and parsed:
+ if name == "fullurl":
+ protocol, other = result
+ result = f"{protocol}://{other}"
+
+ elif name in ("a-tag", "script-tag") and parsed:
path = html.unescape(result[1]).lstrip("/")
- if not path.startswith("https://") and not path.startswith("https://"):
+ if not self.compiled_regexes["fullurl"].match(path):
result = f"{event.parsed.scheme}://{event.parsed.netloc}/{path}"
else:
result = path
@@ -74,6 +79,26 @@ def report(self, result, name, event, **kwargs):
self.excavate.debug(f"omitted result from a-tag parser because of blacklisted prefix [{p}]")
return
+ parsed_uri = self.excavate.helpers.urlparse(result)
+ host, port = self.excavate.helpers.split_host_port(parsed_uri.netloc)
+ # Handle non-HTTP URIs (ftp, s3, etc.)
+ if parsed_uri.scheme.lower() not in ("http", "https"):
+ event_data = {"host": str(host), "description": f"Non-HTTP URI: {result}"}
+ parsed_url = getattr(event, "parsed", None)
+ if parsed_url:
+ event_data["url"] = parsed_url.geturl()
+ self.excavate.emit_event(
+ event_data,
+ "FINDING",
+ source=event,
+ )
+ self.excavate.emit_event(
+ {"protocol": parsed_uri.scheme, "host": str(host)},
+ "PROTOCOL",
+ source=event,
+ )
+ return
+
url_depth = self.excavate.helpers.url_depth(result)
web_spider_depth = self.excavate.scan.config.get("web_spider_depth", 1)
spider_distance = getattr(event, "web_spider_distance", 0)
@@ -86,7 +111,6 @@ def report(self, result, name, event, **kwargs):
class EmailExtractor(BaseExtractor):
-
regexes = {"email": _email_regex}
tld_blacklist = ["png", "jpg", "jpeg", "bmp", "ico", "gif", "svg", "css", "ttf", "woff", "woff2"]
@@ -99,7 +123,6 @@ def report(self, result, name, event, **kwargs):
class ErrorExtractor(BaseExtractor):
-
regexes = {
"PHP:1": r"\.php on line [0-9]+",
"PHP:2": r"\.php on line [0-9]+",
@@ -128,7 +151,6 @@ def report(self, result, name, event, **kwargs):
class JWTExtractor(BaseExtractor):
-
regexes = {"JWT": r"eyJ(?:[\w-]*\.)(?:[\w-]*\.)[\w-]*"}
def report(self, result, name, event, **kwargs):
@@ -161,6 +183,19 @@ def report(self, result, name, event, **kwargs):
)
+class FunctionalityExtractor(BaseExtractor):
+ regexes = {
+ "File Upload Functionality": r"(]+type=[\"']?file[\"']?[^>]+>)",
+ "Web Service WSDL": r"(?i)((?:http|https)://[^\s]*?.(?:wsdl))",
+ }
+
+ def report(self, result, name, event, **kwargs):
+ description = f"{name} found"
+ self.excavate.emit_event(
+ {"host": str(event.host), "url": event.data.get("url"), "description": description}, "FINDING", event
+ )
+
+
class JavascriptExtractor(BaseExtractor):
# based on on https://github.com/m4ll0k/SecretFinder/blob/master/SecretFinder.py
@@ -197,7 +232,6 @@ class JavascriptExtractor(BaseExtractor):
}
def report(self, result, name, event, **kwargs):
-
# ensure that basic auth matches aren't false positives
if name == "authorization_basic":
try:
@@ -215,7 +249,6 @@ def report(self, result, name, event, **kwargs):
class excavate(BaseInternalModule):
-
watched_events = ["HTTP_RESPONSE"]
produced_events = ["URL_UNVERIFIED"]
flags = ["passive"]
@@ -226,7 +259,6 @@ class excavate(BaseInternalModule):
deps_pip = ["pyjwt"]
def setup(self):
-
self.hostname = HostnameExtractor(self)
self.url = URLExtractor(self)
self.email = EmailExtractor(self)
@@ -234,6 +266,7 @@ def setup(self):
self.jwt = JWTExtractor(self)
self.javascript = JavascriptExtractor(self)
self.serialization = SerializationExtractor(self)
+ self.functionality = FunctionalityExtractor(self)
self.max_redirects = self.scan.config.get("http_max_redirects", 5)
return True
@@ -243,12 +276,10 @@ def search(self, source, extractors, event, **kwargs):
e.search(source, event, **kwargs)
def handle_event(self, event):
-
data = event.data
# HTTP_RESPONSE is a special case
if event.type == "HTTP_RESPONSE":
-
# handle redirects
num_redirects = getattr(event, "num_redirects", 0)
location = event.data.get("location", "")
@@ -281,6 +312,7 @@ def handle_event(self, event):
self.jwt,
self.javascript,
self.serialization,
+ self.functionality,
],
event,
spider_danger=True,
@@ -295,7 +327,6 @@ def handle_event(self, event):
)
else:
-
self.search(
str(data),
[self.hostname, self.url, self.email, self.error_extractor, self.jwt, self.serialization],
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 2b4dea89d4..2505f39ea6 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -1,3 +1,4 @@
+import random
import ipaddress
from bbot.modules.internal.base import BaseInternalModule
@@ -14,32 +15,46 @@ class speculate(BaseInternalModule):
flags = ["passive"]
meta = {"description": "Derive certain event types from others by common sense"}
- options = {"max_hosts": 65536}
- options_desc = {"max_hosts": "Max number of IP_RANGE hosts to convert into IP_ADDRESS events"}
+ options = {"max_hosts": 65536, "ports": [80, 443]}
+ options_desc = {
+ "max_hosts": "Max number of IP_RANGE hosts to convert into IP_ADDRESS events",
+ "ports": "The set of ports to speculate on",
+ }
max_event_handlers = 5
scope_distance_modifier = 0
_scope_shepherding = False
+ _priority = 4
def setup(self):
self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in self.scan.modules.values()])
self.portscanner_enabled = any(["portscan" in m.flags for m in self.scan.modules.values()])
self.range_to_ip = True
+
+ self.ports = self.config.get("ports", [80, 443])
+ if isinstance(self.ports, int):
+ self.ports = [self.ports]
+ if not self.portscanner_enabled:
+ self.info(f"No portscanner enabled. Assuming open ports: {', '.join(str(x) for x in self.ports)}")
+
target_len = len(self.scan.target)
if target_len > self.config.get("max_hosts", 65536):
if not self.portscanner_enabled:
self.hugewarning(
f"Selected target ({target_len:,} hosts) is too large, skipping IP_RANGE --> IP_ADDRESS speculation"
)
- self.hugewarning(f"Enabling a port scanner module is highly recommended")
+ self.hugewarning(f"Enabling a port scanner (naabu or masscan) module is highly recommended")
self.range_to_ip = False
+
return True
def handle_event(self, event):
# generate individual IP addresses from IP range
if event.type == "IP_RANGE" and self.range_to_ip:
net = ipaddress.ip_network(event.data)
- for x in net:
- self.emit_event(x, "IP_ADDRESS", source=event, internal=True)
+ ips = list(net)
+ random.shuffle(ips)
+ for ip in ips:
+ self.emit_event(ip, "IP_ADDRESS", source=event, internal=True)
# parent domains
if event.type == "DNS_NAME":
@@ -51,7 +66,7 @@ def handle_event(self, event):
emit_open_ports = self.open_port_consumers and not self.portscanner_enabled
# from URLs
if event.type == "URL" or (event.type == "URL_UNVERIFIED" and emit_open_ports):
- if event.host and event.port not in (80, 443):
+ if event.host and event.port not in self.ports:
self.emit_event(
self.helpers.make_netloc(event.host, event.port), "OPEN_TCP_PORT", source=event, internal=True
)
@@ -61,15 +76,14 @@ def handle_event(self, event):
usable_dns = False
if event.type == "DNS_NAME":
-
if "a-record" in event.tags or "aaaa-record" in event.tags:
usable_dns = True
if event.type == "IP_ADDRESS" or usable_dns:
- self.emit_event(self.helpers.make_netloc(event.data, 80), "OPEN_TCP_PORT", source=event, internal=True)
- self.emit_event(
- self.helpers.make_netloc(event.data, 443), "OPEN_TCP_PORT", source=event, internal=True
- )
+ for port in self.ports:
+ self.emit_event(
+ self.helpers.make_netloc(event.data, port), "OPEN_TCP_PORT", source=event, internal=True
+ )
def filter_event(self, event):
# don't accept IP_RANGE --> IP_ADDRESS events from self
diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py
index 0f139227ac..c2ef99a44d 100644
--- a/bbot/modules/ipneighbor.py
+++ b/bbot/modules/ipneighbor.py
@@ -4,7 +4,6 @@
class ipneighbor(BaseModule):
-
watched_events = ["IP_ADDRESS"]
produced_events = ["IP_ADDRESS"]
flags = ["passive", "subdomain-enum", "aggressive"]
diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py
index d818dc161b..18513fa2f7 100644
--- a/bbot/modules/ipstack.py
+++ b/bbot/modules/ipstack.py
@@ -14,37 +14,44 @@ class Ipstack(shodan_dns):
options = {"api_key": ""}
options_desc = {"api_key": "IPStack GeoIP API Key"}
scope_distance_modifier = 0
+ _priority = 2
suppress_dupes = False
base_url = "http://api.ipstack.com/"
def ping(self):
- r = self.helpers.request(f"{self.base_url}/check?access_key={self.api_key}")
+ r = self.request_with_fail_count(f"{self.base_url}/check?access_key={self.api_key}")
resp_content = getattr(r, "text", "")
assert getattr(r, "status_code", 0) == 200, resp_content
def handle_event(self, event):
try:
url = f"{self.base_url}/{event.data}?access_key={self.api_key}"
- result = self.helpers.request(url)
+ result = self.request_with_fail_count(url)
if result:
- json = result.json()
- if json:
- location = json.get("country_name")
- city = json.get("city")
- zip_code = json.get("zip")
- region = json.get("region_name")
- latitude = json.get("latitude")
- longitude = json.get("longitude")
- self.emit_event(
- f"{location}, {city}, {zip_code}, {region}, {latitude}, {longitude}", "GEOLOCATION", event
- )
- else:
+ j = result.json()
+ if not j:
self.verbose(f"No JSON response from {url}")
else:
self.verbose(f"No response from {url}")
except Exception:
- import traceback
-
self.verbose(f"Error retrieving results for {event.data}")
- self.debug(traceback.format_exc())
+ self.trace()
+ return
+ geo_data = {
+ "ip": j.get("ip"),
+ "country": j.get("country_name"),
+ "city": j.get("city"),
+ "zip_code": j.get("zip"),
+ "region": j.get("region_name"),
+ "latitude": j.get("latitude"),
+ "longitude": j.get("longitude"),
+ }
+ geo_data = {k: v for k, v in geo_data.items() if v is not None}
+ if geo_data:
+ event_data = ", ".join(f"{k.capitalize()}: {v}" for k, v in geo_data.items())
+ self.emit_event(event_data, "GEOLOCATION", event)
+ elif "error" in j:
+ error_msg = j.get("error").get("info", "")
+ if error_msg:
+ self.warning(error_msg)
diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py
index cba106c27c..a18a186b7e 100644
--- a/bbot/modules/leakix.py
+++ b/bbot/modules/leakix.py
@@ -12,7 +12,7 @@ class leakix(crobat):
def handle_event(self, event):
query = self.make_query(event)
headers = {"Accept": "application/json"}
- r = self.helpers.request(f"{self.base_url}/domain/{self.helpers.quote(query)}", headers=headers)
+ r = self.request_with_fail_count(f"{self.base_url}/domain/{self.helpers.quote(query)}", headers=headers)
if not r:
return
try:
diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py
new file mode 100644
index 0000000000..42947c0065
--- /dev/null
+++ b/bbot/modules/masscan.py
@@ -0,0 +1,176 @@
+import json
+import functools
+import subprocess
+from contextlib import suppress
+
+from bbot.modules.base import BaseModule
+
+
+class masscan(BaseModule):
+ flags = ["active", "portscan", "aggressive"]
+ watched_events = ["SCAN"]
+ produced_events = ["OPEN_TCP_PORT"]
+ meta = {"description": "Port scan IP subnets with masscan"}
+ # 600 packets/s ~= entire private IP space in 8 hours
+ options = {"ports": "80,443", "rate": 600, "wait": 10, "ping_first": False}
+ options_desc = {
+ "ports": "Ports to scan",
+ "rate": "Rate in packets per second",
+ "wait": "Seconds to wait for replies after scan is complete",
+ "ping_first": "Only portscan hosts that reply to pings",
+ }
+ deps_ansible = [
+ {
+ "name": "install dev tools",
+ "package": {"name": ["gcc", "git", "make"], "state": "present"},
+ "become": True,
+ "ignore_errors": True,
+ },
+ {
+ "name": "Download masscan source code",
+ "git": {
+ "repo": "https://github.com/robertdavidgraham/masscan.git",
+ "dest": "#{BBOT_TEMP}/masscan",
+ "single_branch": True,
+ "version": "master",
+ },
+ },
+ {
+ "name": "Build masscan",
+ "command": {
+ "chdir": "#{BBOT_TEMP}/masscan",
+ "cmd": "make -j",
+ "creates": "#{BBOT_TEMP}/masscan/bin/masscan",
+ },
+ },
+ {
+ "name": "Install masscan",
+ "copy": {"src": "#{BBOT_TEMP}/masscan/bin/masscan", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"},
+ },
+ ]
+ _qsize = 100
+
+ def setup(self):
+ self.ports = self.config.get("ports", "80,443")
+ self.rate = self.config.get("rate", 600)
+ self.wait = self.config.get("wait", 10)
+ self.ping_first = self.config.get("ping_first", False)
+ self.alive_hosts = dict()
+ # make a quick dry run to validate ports etc.
+ self._target_findkey = "9.8.7.6"
+ try:
+ dry_run_command = self._build_masscan_command(self._target_findkey, dry_run=True)
+ dry_run_result = self.helpers.run(dry_run_command)
+ self.masscan_config = dry_run_result.stdout
+ self.masscan_config = "\n".join(l for l in self.masscan_config.splitlines() if "nocapture" not in l)
+ except subprocess.CalledProcessError as e:
+ self.warning(f"Error in masscan: {e.stderr}")
+ return False
+ self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges")
+ return True
+
+ def handle_event(self, event):
+ exclude, invalid_exclude = self._build_targets(self.scan.blacklist)
+ targets, invalid_targets = self._build_targets(self.scan.whitelist)
+ if invalid_exclude > 0:
+ self.warning(
+ f"Masscan can only accept IP addresses or IP ranges for blacklist ({invalid_exclude:,} blacklisted were hostnames)"
+ )
+ if invalid_targets > 0:
+ self.warning(
+ f"Masscan can only accept IP addresses or IP ranges as target ({invalid_targets:,} targets were hostnames)"
+ )
+
+ if not targets:
+ self.warning("No targets specified")
+ return
+
+ # ping scan
+ if self.ping_first:
+ self.verbose("Starting masscan (ping scan)")
+
+ def append_alive_host(host, source):
+ host_event = self.make_event(host, "IP_ADDRESS", source=self.scan.whitelist.get(host))
+ self.alive_hosts[host] = host_event
+ self.emit_event(host_event)
+
+ self.masscan(targets, result_callback=append_alive_host, exclude=exclude, ping=True)
+ targets = ",".join(str(h) for h in self.alive_hosts)
+ if not targets:
+ self.warning("No hosts responded to pings")
+ return
+
+ # TCP SYN scan
+ self.verbose("Starting masscan (TCP SYN scan)")
+ self.masscan(targets, result_callback=self.emit_open_tcp_port, exclude=exclude, event=event)
+ # save memory
+ self.alive_hosts.clear()
+
+ def masscan(self, targets, result_callback, exclude=None, event=None, ping=False):
+ # config file
+ masscan_config = self.masscan_config.replace(self._target_findkey, targets)
+ self.debug("Masscan config:")
+ for line in masscan_config.splitlines():
+ self.debug(line)
+ config_file = self.helpers.tempfile(masscan_config)
+ # output file
+ process_output = functools.partial(self.process_output, source=event, result_callback=result_callback)
+ json_output_file = self.helpers.tempfile_tail(process_output)
+ # command
+ command = self._build_masscan_command(config=config_file, exclude=exclude, ping=ping)
+ command += ("-oJ", json_output_file)
+ # execute
+ self.helpers.run(command, sudo=True)
+
+ def _build_masscan_command(self, targets=None, config=None, exclude=None, dry_run=False, ping=False):
+ command = ("masscan", "--rate", self.rate, "--wait", self.wait, "--open-only")
+ if targets is not None:
+ command += (targets,)
+ if config is not None:
+ command += ("-c", config)
+ if ping:
+ command += ("--ping",)
+ elif not dry_run:
+ command += ("-p", self.ports)
+ if exclude is not None:
+ command += ("--exclude", exclude)
+ if dry_run:
+ command += ("--echo",)
+ return command
+
+ def process_output(self, line, source, result_callback):
+ try:
+ j = json.loads(line)
+ except Exception:
+ return
+ ip = j.get("ip", "")
+ if not ip:
+ return
+ ports = j.get("ports", [])
+ if not ports:
+ return
+ for p in ports:
+ proto = p.get("proto", "")
+ port_number = p.get("port", "")
+ if proto == "" or port_number == "":
+ continue
+ result = str(ip)
+ if proto != "icmp":
+ result += f":{port_number}"
+ with suppress(KeyError):
+ source = self.alive_hosts[ip]
+ result_callback(result, source=source)
+
+ def emit_open_tcp_port(self, data, source):
+ self.emit_event(data, "OPEN_TCP_PORT", source=source)
+
+ def _build_targets(self, target):
+ invalid_targets = 0
+ targets = []
+ for t in target:
+ t = self.helpers.make_ip_type(t.data)
+ if isinstance(t, str):
+ invalid_targets += 1
+ else:
+ targets.append(t)
+ return ",".join(str(t) for t in targets), invalid_targets
diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py
index b3fd5fa52a..92a7a222c8 100644
--- a/bbot/modules/massdns.py
+++ b/bbot/modules/massdns.py
@@ -5,8 +5,7 @@
class massdns(crobat):
-
- flags = ["brute-force", "subdomain-enum", "passive", "slow", "aggressive"]
+ flags = ["subdomain-enum", "passive", "slow", "aggressive"]
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
meta = {"description": "Brute-force subdomains with massdns (highly effective)"}
@@ -17,7 +16,12 @@ class massdns(crobat):
options_desc = {"wordlist": "Subdomain wordlist URL", "max_resolvers": "Number of concurrent massdns resolvers"}
subdomain_file = None
deps_ansible = [
- {"name": "install dev tools", "package": {"name": ["gcc", "git", "make"], "state": "present"}, "become": True},
+ {
+ "name": "install dev tools",
+ "package": {"name": ["gcc", "git", "make"], "state": "present"},
+ "become": True,
+ "ignore_errors": True,
+ },
{
"name": "Download massdns source code",
"git": {
@@ -28,8 +32,18 @@ class massdns(crobat):
},
},
{
- "name": "Build massdns",
+ "name": "Build massdns (Linux)",
"command": {"chdir": "#{BBOT_TEMP}/massdns", "cmd": "make", "creates": "#{BBOT_TEMP}/massdns/bin/massdns"},
+ "when": "ansible_facts['system'] == 'Linux'",
+ },
+ {
+ "name": "Build massdns (non-Linux)",
+ "command": {
+ "chdir": "#{BBOT_TEMP}/massdns",
+ "cmd": "make nolinux",
+ "creates": "#{BBOT_TEMP}/massdns/bin/massdns",
+ },
+ "when": "ansible_facts['system'] != 'Linux'",
},
{
"name": "Install massdns",
@@ -43,17 +57,34 @@ def setup(self):
self.mutations_tried = set()
self.source_events = dict()
self.subdomain_file = self.helpers.wordlist(self.config.get("wordlist"))
- ret = super().setup()
- if not len(self.helpers.resolvers) >= 100 and not self.helpers.in_tests:
- return None, "Not enough nameservers available for DNS brute-forcing"
- return ret
+ self.max_resolvers = self.config.get("max_resolvers", 500)
+ nameservers_url = (
+ "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
+ )
+ self.resolver_file = self.helpers.wordlist(
+ nameservers_url,
+ cache_hrs=24 * 7,
+ )
+ return super().setup()
def filter_event(self, event):
- if "unresolved" in event.tags and not "target" in event.tags:
- return False
query = self.make_query(event)
if self.already_processed(query):
- return False
+ return False, "Event was already processed"
+ is_cloud = False
+ if any(t.startswith("cloud-") for t in event.tags):
+ is_cloud = True
+ is_wildcard = False
+ for domain, wildcard_rdtypes in self.helpers.is_wildcard_domain(query).items():
+ if any(t in wildcard_rdtypes for t in ("A", "AAAA", "CNAME")):
+ is_wildcard = True
+ if not "target" in event.tags:
+ if "unresolved" in event.tags:
+ return False, "Event is unresolved"
+ if is_cloud:
+ return False, "Event is a cloud resource and not a direct target"
+ if is_wildcard and is_cloud:
+ return False, "Event is both a cloud resource and a wildcard domain"
self.processed.add(hash(query))
return True
@@ -68,12 +99,14 @@ def handle_event(self, event):
self.emit_result(hostname, event, query)
def abort_if(self, event):
- # abort if the event is a wildcard
+ if not event.scope_distance == 0:
+ return True, "event is not in scope"
+ if "unresolved" in event.tags:
+ return True, "event is unresolved"
if "wildcard" in event.tags:
- return True
- # abort if the event is not a valid record type
+ return True, "event is a wildcard"
if not any(x in event.tags for x in ("a-record", "aaaa-record", "cname-record")):
- return True
+ return True, "event is not a valid record type"
def emit_result(self, result, source_event, query):
if not result == source_event:
@@ -88,6 +121,18 @@ def already_processed(self, hostname):
return False
def massdns(self, domain, subdomains):
+ canary_checks = 50
+ canary_subdomains = [self.helpers.rand_string(10) for i in range(canary_checks)]
+ self.verbose(f"Testing {canary_checks:,} canaries against {domain}")
+ canary_results = list(self._massdns(domain, canary_subdomains))
+ if len(canary_results) > 10:
+ self.info(
+ f"Aborting massdns run on {domain} due to {len(canary_results):,}/{canary_checks:,} false positives"
+ )
+ else:
+ yield from self._massdns(domain, subdomains)
+
+ def _massdns(self, domain, subdomains):
"""
{
"name": "www.blacklanternsecurity.com.",
@@ -118,12 +163,18 @@ def massdns(self, domain, subdomains):
if self.scan.stopping:
return
+ domain_wildcard_rdtypes = set()
+ for domain, rdtypes in self.helpers.is_wildcard_domain(domain).items():
+ for rdtype, results in rdtypes.items():
+ if results:
+ domain_wildcard_rdtypes.add(rdtype)
+
command = (
"massdns",
"-r",
- self.helpers.dns.mass_resolver_file,
+ self.resolver_file,
"-s",
- self.config.get("max_resolvers", 1000),
+ self.max_resolvers,
"-t",
"A",
"-t",
@@ -151,10 +202,18 @@ def massdns(self, domain, subdomains):
# 8AAAA queries have been locally blocked by dnscrypt-proxy/Set block_ipv6 to false to disable this feature
if data and rdtype and not " " in data:
# skip wildcards
- wildcard_rdtypes = self.helpers.is_wildcard(hostname, ips=(data,))
- if rdtype in wildcard_rdtypes:
- self.debug(f"Skipping {hostname}:{rdtype} because it's a wildcard")
- continue
+ if rdtype in domain_wildcard_rdtypes:
+ # skip wildcard checking on multi-level subdomains for performance reasons
+ stem = hostname.split(domain)[0].strip(".")
+ if "." in stem:
+ self.debug(
+ f"Skipping {hostname}:{rdtype} because it may be a wildcard (reason: performance)"
+ )
+ continue
+ wildcard_rdtypes = self.helpers.is_wildcard(hostname, ips=(data,))
+ if rdtype in wildcard_rdtypes:
+ self.debug(f"Skipping {hostname}:{rdtype} because it's a wildcard")
+ continue
hostname = hostname.rstrip(".").lower()
hostname_hash = hash(hostname)
if hostname_hash not in hosts_yielded:
@@ -205,7 +264,8 @@ def add_found(self, event):
def gen_subdomains(self, prefixes, domain):
for p in prefixes:
- yield f"{p}.{domain}"
+ d = f"{p}.{domain}"
+ yield d
def get_source_event(self, hostname):
for p in self.helpers.domain_parents(hostname):
diff --git a/bbot/modules/naabu.py b/bbot/modules/naabu.py
index 02921bc17b..7327dc02b8 100644
--- a/bbot/modules/naabu.py
+++ b/bbot/modules/naabu.py
@@ -4,10 +4,9 @@
class naabu(BaseModule):
-
watched_events = ["IP_ADDRESS", "DNS_NAME", "IP_RANGE"]
produced_events = ["OPEN_TCP_PORT"]
- flags = ["active", "portscan", "aggressive"]
+ flags = ["active", "portscan", "aggressive", "web-thorough"]
meta = {"description": "Execute port scans with naabu"}
options = {
"ports": "",
@@ -21,6 +20,7 @@ class naabu(BaseModule):
}
max_event_handlers = 2
batch_size = 100
+ _priority = 2
deps_ansible = [
{
@@ -28,23 +28,25 @@ class naabu(BaseModule):
"package": {"name": "libpcap0.8", "state": "present"},
"become": True,
"when": """ansible_facts['os_family'] == 'Debian'""",
+ "ignore_errors": True,
},
{
"name": "install libpcap (others)",
"package": {"name": "libpcap", "state": "present"},
"become": True,
"when": """ansible_facts['os_family'] != 'Debian'""",
+ "ignore_errors": True,
},
{
"name": "symlink libpcap",
"file": {"src": "/usr/lib/libpcap.so", "dest": "#{BBOT_LIB}/libpcap.so.0.8", "state": "link"},
- "ignore_errors": "yes",
"when": """ansible_facts['os_family'] != 'Debian'""",
+ "ignore_errors": True,
},
{
"name": "Download naabu",
"unarchive": {
- "src": "https://github.com/projectdiscovery/naabu/releases/download/v#{BBOT_MODULES_NAABU_VERSION}/naabu_#{BBOT_MODULES_NAABU_VERSION}_linux_amd64.zip",
+ "src": "https://github.com/projectdiscovery/naabu/releases/download/v#{BBOT_MODULES_NAABU_VERSION}/naabu_#{BBOT_MODULES_NAABU_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.zip",
"include": "naabu",
"dest": "#{BBOT_TOOLS}",
"remote_src": True,
@@ -52,11 +54,14 @@ class naabu(BaseModule):
},
]
- def handle_batch(self, *events):
+ def setup(self):
+ self.helpers.depsinstaller.ensure_root(message="Naabu requires root privileges")
+ return True
+ def handle_batch(self, *events):
_input = [str(e.data) for e in events]
command = self.construct_command()
- for line in self.helpers.run_live(command, input=_input, stderr=subprocess.DEVNULL):
+ for line in self.helpers.run_live(command, input=_input, stderr=subprocess.DEVNULL, sudo=True):
try:
j = json.loads(line)
except Exception as e:
diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py
index 240895b86b..684267806c 100644
--- a/bbot/modules/ntlm.py
+++ b/bbot/modules/ntlm.py
@@ -61,10 +61,9 @@
class ntlm(BaseModule):
-
watched_events = ["URL", "HTTP_RESPONSE"]
produced_events = ["FINDING", "DNS_NAME"]
- flags = ["active", "safe", "web-basic"]
+ flags = ["active", "safe", "web-basic", "web-thorough"]
meta = {"description": "Watch for HTTP endpoints that support NTLM authentication"}
options = {"max_threads": 10, "try_all": False}
options_desc = {"max_threads": "Maximum concurrent requests", "try_all": "Try every NTLM endpoint"}
@@ -137,7 +136,6 @@ def handle_url(self, event):
return None, None
def check_ntlm(self, test_url):
-
url_hash = hash(test_url)
with self.processed_lock:
diff --git a/bbot/modules/otx.py b/bbot/modules/otx.py
index 75c8b9318a..abe856e47e 100644
--- a/bbot/modules/otx.py
+++ b/bbot/modules/otx.py
@@ -2,7 +2,6 @@
class otx(crobat):
-
flags = ["subdomain-enum", "passive", "safe"]
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
@@ -12,7 +11,7 @@ class otx(crobat):
def request_url(self, query):
url = f"{self.base_url}/api/v1/indicators/domain/{self.helpers.quote(query)}/passive_dns"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def parse_results(self, r, query):
j = r.json()
diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py
index 8f9bdf2be3..b36d4ec189 100644
--- a/bbot/modules/output/asset_inventory.py
+++ b/bbot/modules/output/asset_inventory.py
@@ -26,10 +26,12 @@ class asset_inventory(CSV):
def setup(self):
self.assets = {}
+ self.open_port_producers = "httpx" in self.scan.modules or any(
+ ["portscan" in m.flags for m in self.scan.modules.values()]
+ )
return super().setup()
def handle_event(self, event):
-
if (
(not event._internal)
and str(event.module) != "speculate"
@@ -37,22 +39,24 @@ def handle_event(self, event):
and self.scan.in_scope(event)
and not "unresolved" in event.tags
):
-
if event.host not in self.assets:
self.assets[event.host] = Asset(event.host)
for rh in event.resolved_hosts:
- self.assets[event.host].ip_addresses.add(str(rh))
+ if self.helpers.is_ip(rh):
+ self.assets[event.host].ip_addresses.add(str(rh))
if event.port:
self.assets[event.host].ports.add(str(event.port))
if event.type == "FINDING":
- self.assets[event.host].findings.add(f"{event.data['url']}:{event.data['description']}")
+ location = event.data.get("url", event.data.get("host"))
+ self.assets[event.host].findings.add(f"{location}:{event.data['description']}")
if event.type == "VULNERABILITY":
+ location = event.data.get("url", event.data.get("host"))
self.assets[event.host].findings.add(
- f"{event.data['url']}:{event.data['description']}:{event.data['severity']}"
+ f"{location}:{event.data['description']}:{event.data['severity']}"
)
severity_int = severity_map.get(event.data.get("severity", "N/A"), 0)
if severity_int > self.assets[event.host].risk_rating:
@@ -62,13 +66,13 @@ def handle_event(self, event):
self.assets[event.host].technologies.add(event.data["technology"])
def report(self):
- for asset in self.assets.values():
+ for asset in sorted(self.assets.values(), key=lambda a: str(a.host)):
findings_and_vulns = asset.findings.union(asset.vulnerabilities)
self.writerow(
[
getattr(asset, "host", ""),
",".join(str(x) for x in getattr(asset, "ip_addresses", set())),
- "Active" if (asset.ports) else "Timeout",
+ "Active" if (asset.ports) else ("Inactive" if self.open_port_producers else "N/A"),
",".join(str(x) for x in getattr(asset, "ports", set())),
severity_map[getattr(asset, "risk_rating", "")],
",".join(findings_and_vulns),
diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py
index 684ee750a0..5a01b0307f 100644
--- a/bbot/modules/output/base.py
+++ b/bbot/modules/output/base.py
@@ -1,29 +1,37 @@
import logging
-
+from pathlib import Path
from bbot.modules.base import BaseModule
class BaseOutputModule(BaseModule):
accept_dupes = True
_type = "output"
- emit_graph_trail = True
scope_distance_modifier = None
_stats_exclude = True
- def _filter_event(self, event, precheck_only=False):
- if type(event) == str:
- if event in ("FINISHED", "REPORT"):
- return True, ""
- else:
- return False, f'string value "{event}" is invalid'
+ def _event_precheck(self, event):
if event._omit:
return False, "_omit is True"
- if not precheck_only:
- if event._force_output:
- return True, "_force_output is True"
- if event._internal:
- return False, "_internal is True"
- return True, ""
+ if event._force_output:
+ return True, "_force_output is True"
+ if event._internal:
+ return False, "_internal is True"
+ return super()._event_precheck(event)
+
+ def _prep_output_dir(self, filename):
+ self.output_file = self.config.get("output_file", "")
+ if self.output_file:
+ self.output_file = Path(self.output_file)
+ else:
+ self.output_file = self.scan.home / str(filename)
+ self.helpers.mkdir(self.output_file.parent)
+ self._file = None
+
+ @property
+ def file(self):
+ if self._file is None:
+ self._file = open(self.output_file, mode="a")
+ return self._file
@property
def config(self):
diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py
index 929ef02d5b..64f7cd54e6 100644
--- a/bbot/modules/output/csv.py
+++ b/bbot/modules/output/csv.py
@@ -1,5 +1,4 @@
import csv
-from pathlib import Path
from contextlib import suppress
from bbot.modules.output.base import BaseOutputModule
@@ -10,20 +9,13 @@ class CSV(BaseOutputModule):
meta = {"description": "Output to CSV"}
options = {"output_file": ""}
options_desc = {"output_file": "Output to CSV file"}
- emit_graph_trail = False
header_row = ["Event type", "Event data", "IP Address", "Source Module", "Scope Distance", "Event Tags"]
filename = "output.csv"
def setup(self):
- self.output_file = self.config.get("output_file", "")
- if self.output_file:
- self.output_file = Path(self.output_file)
- else:
- self.output_file = self.scan.home / self.filename
- self.helpers.mkdir(self.output_file.parent)
- self._file = None
self._writer = None
+ self._prep_output_dir(self.filename)
return True
@property
@@ -48,7 +40,7 @@ def handle_event(self, event):
[
getattr(event, "type", ""),
getattr(event, "data", ""),
- ",".join(str(x) for x in getattr(event, "resolved_hosts", set())),
+ ",".join(str(x) for x in getattr(event, "resolved_hosts", set()) if self.helpers.is_ip(x)),
str(getattr(event, "module", "")),
str(getattr(event, "scope_distance", "")),
",".join(sorted(list(getattr(event, "tags", [])))),
@@ -56,7 +48,7 @@ def handle_event(self, event):
)
def cleanup(self):
- if self._file is not None:
+ if getattr(self, "_file", None) is not None:
with suppress(Exception):
self.file.close()
diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py
index 8661284a58..04c8e30101 100644
--- a/bbot/modules/output/http.py
+++ b/bbot/modules/output/http.py
@@ -7,7 +7,7 @@
class HTTP(BaseOutputModule):
watched_events = ["*"]
- meta = {"description": "Output to HTTP"}
+ meta = {"description": "Send every event to a custom URL via a web request"}
options = {
"url": "",
"method": "POST",
diff --git a/bbot/modules/output/human.py b/bbot/modules/output/human.py
index 9f633bfe71..b9eef9402e 100644
--- a/bbot/modules/output/human.py
+++ b/bbot/modules/output/human.py
@@ -1,4 +1,3 @@
-from pathlib import Path
from contextlib import suppress
from bbot.core.helpers.logger import log_to_stderr
@@ -10,25 +9,12 @@ class Human(BaseOutputModule):
meta = {"description": "Output to text"}
options = {"output_file": "", "console": True}
options_desc = {"output_file": "Output to file", "console": "Output to console"}
- emit_graph_trail = False
vuln_severity_map = {"LOW": "HUGEWARNING", "MEDIUM": "HUGEWARNING", "HIGH": "CRITICAL", "CRITICAL": "CRITICAL"}
def setup(self):
- self.output_file = self.config.get("output_file", "")
- if self.output_file:
- self.output_file = Path(self.output_file)
- else:
- self.output_file = self.scan.home / "output.txt"
- self.helpers.mkdir(self.output_file.parent)
- self._file = None
+ self._prep_output_dir("output.txt")
return True
- @property
- def file(self):
- if self._file is None:
- self._file = open(self.output_file, mode="a")
- return self._file
-
def handle_event(self, event):
event_type = f"[{event.type}]"
event_tags = ""
@@ -51,7 +37,7 @@ def handle_event(self, event):
self.stdout(event_str)
def cleanup(self):
- if self._file is not None:
+ if getattr(self, "_file", None) is not None:
with suppress(Exception):
self.file.close()
diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py
index 961494bef7..8d2b743f5a 100644
--- a/bbot/modules/output/json.py
+++ b/bbot/modules/output/json.py
@@ -1,5 +1,4 @@
import json
-from pathlib import Path
from contextlib import suppress
from bbot.modules.output.base import BaseOutputModule
@@ -12,21 +11,9 @@ class JSON(BaseOutputModule):
options_desc = {"output_file": "Output to file", "console": "Output to console"}
def setup(self):
- self.output_file = self.config.get("output_file", "")
- if self.output_file:
- self.output_file = Path(self.output_file)
- else:
- self.output_file = self.scan.home / "output.json"
- self.helpers.mkdir(self.output_file.parent)
- self._file = None
+ self._prep_output_dir("output.json")
return True
- @property
- def file(self):
- if self._file is None:
- self._file = open(self.output_file, mode="a")
- return self._file
-
def handle_event(self, event):
event_str = json.dumps(dict(event))
if self.file is not None:
@@ -36,7 +23,7 @@ def handle_event(self, event):
self.stdout(event_str)
def cleanup(self):
- if self._file is not None:
+ if getattr(self, "_file", None) is not None:
with suppress(Exception):
self.file.close()
diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py
new file mode 100644
index 0000000000..51af7323e1
--- /dev/null
+++ b/bbot/modules/output/web_report.py
@@ -0,0 +1,96 @@
+from bbot.modules.output.base import BaseOutputModule
+import markdown
+import html
+
+
+class web_report(BaseOutputModule):
+ watched_events = ["URL", "TECHNOLOGY", "FINDING", "VULNERABILITY", "VHOST"]
+ meta = {"description": "Create a markdown report with web assets"}
+ options = {
+ "output_file": "",
+ "css_theme_file": "https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css",
+ }
+ options_desc = {"output_file": "Output to file", "css_theme_file": "CSS theme URL for HTML output"}
+ deps_pip = ["markdown"]
+
+ def setup(self):
+ html_css_file = self.config.get("css_theme_file", "")
+
+ self.html_header = f"""
+
+
+
+
+
+
+ """
+
+ self.html_footer = ""
+ self.web_assets = {}
+ self.markdown = ""
+
+ self._prep_output_dir("web_report.html")
+ return True
+
+ def handle_event(self, event):
+ if event.type == "URL":
+ parsed = event.parsed
+ host = f"{parsed.scheme}://{parsed.netloc}/"
+ if host not in self.web_assets.keys():
+ self.web_assets[host] = {"URL": []}
+ source_chain = []
+
+ current_parent = event.source
+ while not current_parent.type == "SCAN":
+ source_chain.append(
+ f" ({current_parent.module})---> [{current_parent.type}]:{html.escape(current_parent.pretty_string)}"
+ )
+ current_parent = current_parent.source
+
+ source_chain.reverse()
+ source_chain_text = (
+ "".join(source_chain)
+ + f" ({event.module})---> "
+ + f"[{event.type}]:{html.escape(event.pretty_string)}"
+ )
+ self.web_assets[host]["URL"].append(f"**{html.escape(event.data)}**: {source_chain_text}")
+
+ else:
+ current_parent = event.source
+ parsed = None
+ while 1:
+ if current_parent.type == "URL":
+ parsed = current_parent.parsed
+ break
+ current_parent = current_parent.source
+ if current_parent.source.type == "SCAN":
+ break
+ if parsed:
+ host = f"{parsed.scheme}://{parsed.netloc}/"
+ if host not in self.web_assets.keys():
+ self.web_assets[host] = {"URL": []}
+ if event.type not in self.web_assets[host].keys():
+ self.web_assets[host][event.type] = [html.escape(event.pretty_string)]
+ else:
+ self.web_assets[host][event.type].append(html.escape(event.pretty_string))
+
+ def report(self):
+ for host in self.web_assets.keys():
+ self.markdown += f"# {host}\n\n"
+
+ for event_type in self.web_assets[host].keys():
+ self.markdown += f"### {event_type}\n"
+ dedupe = []
+ for e in self.web_assets[host][event_type]:
+ if e in dedupe:
+ continue
+ dedupe.append(e)
+ self.markdown += f"\n* {e}\n"
+ self.markdown += "\n"
+
+ if self.file is not None:
+ self.file.write(self.html_header)
+ self.file.write(markdown.markdown(self.markdown))
+ self.file.write(self.html_footer)
+ self.file.flush()
+ self.info(f"Web Report saved to {self.output_file}")
diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py
index 827c711ae5..b920b0e7a7 100644
--- a/bbot/modules/output/websocket.py
+++ b/bbot/modules/output/websocket.py
@@ -22,11 +22,16 @@ def setup(self):
if self.token:
kwargs.update({"header": {"Authorization": f"Bearer {self.token}"}})
self.ws = websocket.WebSocketApp(self.url, **kwargs)
- self.thread = threading.Thread(target=self.start_websocket, daemon=True)
- self.thread.start()
+ self.started = False
return True
def start_websocket(self):
+ if not self.started:
+ self.thread = threading.Thread(target=self._start_websocket, daemon=True)
+ self.thread.start()
+ self.started = True
+
+ def _start_websocket(self):
not_keyboardinterrupt = False
while not self.scan.stopping:
not_keyboardinterrupt = self.ws.run_forever()
@@ -35,6 +40,7 @@ def start_websocket(self):
sleep(1)
def handle_event(self, event):
+ self.start_websocket()
event_json = event.json()
self.send(event_json)
diff --git a/bbot/modules/cookie_brute.py b/bbot/modules/paramminer_cookies.py
similarity index 89%
rename from bbot/modules/cookie_brute.py
rename to bbot/modules/paramminer_cookies.py
index f6acb6e516..fafaa70c96 100644
--- a/bbot/modules/cookie_brute.py
+++ b/bbot/modules/paramminer_cookies.py
@@ -1,15 +1,15 @@
-from .header_brute import header_brute
from bbot.core.errors import ScanCancelledError
+from .paramminer_headers import paramminer_headers
-class cookie_brute(header_brute):
+class paramminer_cookies(paramminer_headers):
"""
Inspired by https://github.com/PortSwigger/param-miner
"""
watched_events = ["URL"]
produced_events = ["FINDING"]
- flags = ["brute-force", "active", "aggressive", "slow", "web-paramminer"]
+ flags = ["active", "aggressive", "slow", "web-paramminer"]
meta = {
"description": "Check for common HTTP cookie parameters",
}
@@ -22,14 +22,12 @@ class cookie_brute(header_brute):
compare_mode = "cookie"
def check_batch(self, compare_helper, url, cookie_list):
-
if self.scan.stopping:
raise ScanCancelledError()
cookies = {p: self.rand_string(14) for p in cookie_list}
return compare_helper.compare(url, cookies=cookies)
def gen_count_args(self, url):
-
cookie_count = 40
while 1:
if cookie_count < 0:
diff --git a/bbot/modules/getparam_brute.py b/bbot/modules/paramminer_getparams.py
similarity index 84%
rename from bbot/modules/getparam_brute.py
rename to bbot/modules/paramminer_getparams.py
index 103029351f..688f394628 100644
--- a/bbot/modules/getparam_brute.py
+++ b/bbot/modules/paramminer_getparams.py
@@ -1,16 +1,16 @@
-from .header_brute import header_brute
from bbot.core.errors import ScanCancelledError
+from .paramminer_headers import paramminer_headers
-class getparam_brute(header_brute):
+class paramminer_getparams(paramminer_headers):
"""
Inspired by https://github.com/PortSwigger/param-miner
"""
watched_events = ["URL"]
produced_events = ["FINDING"]
- flags = ["brute-force", "active", "aggressive", "slow", "web-paramminer"]
- meta = {"description": "Check for common HTTP GET parameters"}
+ flags = ["active", "aggressive", "slow", "web-paramminer"]
+ meta = {"description": "Use smart brute-force to check for common HTTP GET parameters"}
options = {"wordlist": "https://raw.githubusercontent.com/PortSwigger/param-miner/master/resources/params"}
options_desc = {"wordlist": "Define the wordlist to be used to derive GET params"}
@@ -27,7 +27,6 @@ def check_batch(self, compare_helper, url, getparam_list):
return compare_helper.compare(self.helpers.add_get_params(url, test_getparams).geturl())
def gen_count_args(self, url):
-
getparam_count = 40
while 1:
if getparam_count < 0:
diff --git a/bbot/modules/header_brute.py b/bbot/modules/paramminer_headers.py
similarity index 91%
rename from bbot/modules/header_brute.py
rename to bbot/modules/paramminer_headers.py
index f847381449..0b0678825b 100644
--- a/bbot/modules/header_brute.py
+++ b/bbot/modules/paramminer_headers.py
@@ -2,15 +2,15 @@
from bbot.core.errors import HttpCompareError, ScanCancelledError
-class header_brute(BaseModule):
+class paramminer_headers(BaseModule):
"""
Inspired by https://github.com/PortSwigger/param-miner
"""
watched_events = ["URL"]
produced_events = ["FINDING"]
- flags = ["brute-force", "active", "aggressive", "slow", "web-paramminer"]
- meta = {"description": "Check for common HTTP header parameters"}
+ flags = ["active", "aggressive", "slow", "web-paramminer"]
+ meta = {"description": "Use smart brute-force to check for common HTTP header parameters"}
options = {"wordlist": "https://raw.githubusercontent.com/PortSwigger/param-miner/master/resources/headers"}
options_desc = {"wordlist": "Define the wordlist to be used to derive headers"}
scanned_hosts = []
@@ -30,17 +30,14 @@ class header_brute(BaseModule):
compare_mode = "header"
def setup(self):
-
wordlist_url = self.config.get("wordlist", "")
self.wordlist = self.helpers.wordlist(wordlist_url)
return True
def rand_string(self, *args, **kwargs):
-
return self.helpers.rand_string(*args, **kwargs)
def handle_event(self, event):
-
url = event.data
try:
compare_helper = self.helpers.http_compare(url)
@@ -54,7 +51,7 @@ def handle_event(self, event):
self.debug(f"Resolved batch_size at {str(batch_size)}")
if compare_helper.canary_check(url, mode=self.compare_mode) == False:
- self.warning(f'Aborting "{url}" due to failed canary check')
+ self.verbose(f'Aborting "{url}" due to failed canary check')
return
fl = [h.strip().lower() for h in self.helpers.read_file(self.wordlist)]
@@ -79,11 +76,10 @@ def handle_event(self, event):
pass
for result, reasons, reflection in results:
-
tags = []
if reflection:
tags = ["http_reflection"]
- description = f"[{self.compare_mode.upper()}_BRUTE] {self.compare_mode.capitalize()}: [{result}] Reasons: [{reasons}]"
+ description = f"[Paramminer] {self.compare_mode.capitalize()}: [{result}] Reasons: [{reasons}]"
self.emit_event(
{"host": str(event.host), "url": url, "description": description},
"FINDING",
@@ -92,7 +88,6 @@ def handle_event(self, event):
)
def count_test(self, url):
-
baseline = self.helpers.request(url)
if baseline is None:
return
@@ -124,7 +119,8 @@ def binary_search(self, compare_helper, url, group, reasons=None, reflection=Fal
reasons = []
self.debug(f"Entering recursive binary_search with {len(group):,} sized group")
if len(group) == 1:
- yield group[0], reasons, reflection
+ if reasons:
+ yield group[0], reasons, reflection
elif len(group) > 1:
for group_slice in self.helpers.split_list(group):
match, reasons, reflection, subject_response = self.check_batch(compare_helper, url, group_slice)
@@ -134,7 +130,6 @@ def binary_search(self, compare_helper, url, group, reasons=None, reflection=Fal
self.warning(f"Submitted group of size 0 to binary_search()")
def check_batch(self, compare_helper, url, header_list):
-
if self.scan.stopping:
raise ScanCancelledError()
rand = self.rand_string()
diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py
index da22babdb2..27220bca35 100644
--- a/bbot/modules/passivetotal.py
+++ b/bbot/modules/passivetotal.py
@@ -19,7 +19,7 @@ def setup(self):
def ping(self):
url = f"{self.base_url}/account/quota"
- j = self.helpers.request(url, auth=self.auth).json()
+ j = self.request_with_fail_count(url, auth=self.auth).json()
limit = j["user"]["limits"]["search_api"]
used = j["user"]["counts"]["search_api"]
assert used < limit, "No quota remaining"
@@ -30,7 +30,7 @@ def abort_if(self, event):
def request_url(self, query):
url = f"{self.base_url}/enrichment/subdomains?query={self.helpers.quote(query)}"
- return self.helpers.request(url, auth=self.auth)
+ return self.request_with_fail_count(url, auth=self.auth)
def parse_results(self, r, query):
for subdomain in r.json().get("subdomains", []):
diff --git a/bbot/modules/rapiddns.py b/bbot/modules/rapiddns.py
index e5a5887ef9..61bfa767dd 100644
--- a/bbot/modules/rapiddns.py
+++ b/bbot/modules/rapiddns.py
@@ -2,7 +2,6 @@
class rapiddns(crobat):
-
flags = ["subdomain-enum", "passive", "safe"]
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
@@ -12,7 +11,7 @@ class rapiddns(crobat):
def request_url(self, query):
url = f"{self.base_url}/subdomain/{self.helpers.quote(query)}?full=1#result"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def parse_results(self, r, query):
results = set()
diff --git a/bbot/modules/report/affiliates.py b/bbot/modules/report/affiliates.py
index aed38fb2b6..3ef1d1315b 100644
--- a/bbot/modules/report/affiliates.py
+++ b/bbot/modules/report/affiliates.py
@@ -1,7 +1,7 @@
-from bbot.modules.report.base import ReportModule
+from bbot.modules.report.base import BaseReportModule
-class affiliates(ReportModule):
+class affiliates(BaseReportModule):
watched_events = ["*"]
produced_events = []
flags = ["passive", "safe"]
diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py
index 1d64cdf415..ebaace3cba 100644
--- a/bbot/modules/report/asn.py
+++ b/bbot/modules/report/asn.py
@@ -1,8 +1,8 @@
from bbot.core.errors import ScanCancelledError
-from bbot.modules.report.base import ReportModule
+from bbot.modules.report.base import BaseReportModule
-class asn(ReportModule):
+class asn(BaseReportModule):
watched_events = ["IP_ADDRESS"]
produced_events = ["ASN"]
flags = ["passive", "subdomain-enum", "safe"]
@@ -123,7 +123,6 @@ def get_asn_ripe(self, ip):
return asns
def get_asn_metadata_ripe(self, asn_number):
-
metadata_keys = {
"name": ["ASName", "OrgId"],
"description": ["OrgName", "OrgTechName", "RTechName"],
diff --git a/bbot/modules/report/base.py b/bbot/modules/report/base.py
index 832c35345a..8a97cddd56 100644
--- a/bbot/modules/report/base.py
+++ b/bbot/modules/report/base.py
@@ -1,5 +1,5 @@
from bbot.modules.base import BaseModule
-class ReportModule(BaseModule):
+class BaseReportModule(BaseModule):
_stats_exclude = True
diff --git a/bbot/modules/riddler.py b/bbot/modules/riddler.py
index ff39a74d3a..a7128943c8 100644
--- a/bbot/modules/riddler.py
+++ b/bbot/modules/riddler.py
@@ -2,7 +2,6 @@
class riddler(crobat):
-
flags = ["subdomain-enum", "passive", "safe"]
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
@@ -12,7 +11,7 @@ class riddler(crobat):
def request_url(self, query):
url = f"{self.base_url}/search/exportcsv?q=pld:{self.helpers.quote(query)}"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def parse_results(self, r, query):
results = set()
diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py
new file mode 100644
index 0000000000..0ddd005f55
--- /dev/null
+++ b/bbot/modules/robots.py
@@ -0,0 +1,56 @@
+from bbot.modules.base import BaseModule
+
+
+class robots(BaseModule):
+ watched_events = ["URL"]
+ produced_events = ["URL_UNVERIFIED"]
+ flags = ["active", "safe", "web-basic", "web-thorough"]
+ meta = {"description": "Look for and parse robots.txt"}
+
+ options = {"include_sitemap": False, "include_allow": True, "include_disallow": True}
+ options_desc = {
+ "include_sitemap": "Include 'sitemap' entries",
+ "include_allow": "Include 'Allow' Entries",
+ "include_disallow": "Include 'Disallow' Entries",
+ }
+
+ in_scope_only = True
+
+ def setup(self):
+ self.scanned_hosts = set()
+ return True
+
+ def handle_event(self, event):
+ parsed_host = event.parsed
+ host = f"{parsed_host.scheme}://{parsed_host.netloc}/"
+ host_hash = hash(host)
+ if host_hash in self.scanned_hosts:
+ self.debug(f"Host {host} was already scanned, exiting")
+ return
+ else:
+ self.scanned_hosts.add(host_hash)
+
+ result = None
+ url = f"{host}robots.txt"
+ result = self.helpers.request(url)
+ if result:
+ body = result.text
+
+ if body:
+ lines = body.split("\n")
+ for l in lines:
+ if len(l) > 0:
+ split_l = l.split(": ")
+ if (split_l[0].lower() == "allow" and self.config.get("include_allow") == True) or (
+ split_l[0].lower() == "disallow" and self.config.get("include_disallow") == True
+ ):
+ unverified_url = f"{host}{split_l[1].lstrip('/')}".replace(
+ "*", self.helpers.rand_string(4)
+ )
+
+ elif split_l[0].lower() == "sitemap" and self.config.get("include_sitemap") == True:
+ unverified_url = split_l[1]
+ else:
+ continue
+
+ self.emit_event(unverified_url, "URL_UNVERIFIED", source=event)
diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py
index c114c4c181..592616ae55 100644
--- a/bbot/modules/securitytrails.py
+++ b/bbot/modules/securitytrails.py
@@ -2,7 +2,6 @@
class securitytrails(shodan_dns):
-
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
flags = ["subdomain-enum", "passive", "safe"]
@@ -17,13 +16,13 @@ def setup(self):
return super().setup()
def ping(self):
- r = self.helpers.request(f"{self.base_url}/ping?apikey={self.api_key}")
+ r = self.request_with_fail_count(f"{self.base_url}/ping?apikey={self.api_key}")
resp_content = getattr(r, "text", "")
assert getattr(r, "status_code", 0) == 200, resp_content
def request_url(self, query):
url = f"{self.base_url}/domain/{query}/subdomains?apikey={self.api_key}"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def parse_results(self, r, query):
j = r.json()
diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py
index 8bade6c4cd..ded5e4ee94 100644
--- a/bbot/modules/shodan_dns.py
+++ b/bbot/modules/shodan_dns.py
@@ -18,32 +18,19 @@ class shodan_dns(crobat):
def setup(self):
super().setup()
- self.api_key = self.config.get("api_key", "")
- if self.auth_secret:
- try:
- self.ping()
- self.hugesuccess(f"API is ready")
- return True
- except Exception as e:
- return None, f"Error with API ({str(e).strip()})"
- else:
- return None, "No API key set"
+ return self.require_api_key()
def ping(self):
- r = self.helpers.request(f"{self.base_url}/api-info?key={self.api_key}")
+ r = self.request_with_fail_count(f"{self.base_url}/api-info?key={self.api_key}")
resp_content = getattr(r, "text", "")
assert getattr(r, "status_code", 0) == 200, resp_content
def request_url(self, query):
url = f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={self.api_key}"
- return self.helpers.request(url)
+ return self.request_with_fail_count(url)
def parse_results(self, r, query):
json = r.json()
if json:
for hostname in json.get("subdomains"):
yield f"{hostname}.{query}"
-
- @property
- def auth_secret(self):
- return self.api_key
diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py
index 5e4f047148..e319cc6dba 100644
--- a/bbot/modules/skymem.py
+++ b/bbot/modules/skymem.py
@@ -15,7 +15,7 @@ def handle_event(self, event):
_, query = self.helpers.split_domain(event.data)
# get first page
url = f"{self.base_url}/srch?q={self.helpers.quote(query)}"
- r = self.helpers.request(url)
+ r = self.request_with_fail_count(url)
if not r:
return
for email in self.extract_emails(r.text):
@@ -27,7 +27,7 @@ def handle_event(self, event):
return
domain_id = domain_ids[0]
for page in range(2, 22):
- r2 = self.helpers.request(f"{self.base_url}/domain/{domain_id}?p={page}")
+ r2 = self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}")
if not r2:
continue
for email in self.extract_emails(r2.text):
diff --git a/bbot/modules/smuggler.py b/bbot/modules/smuggler.py
index 6f3e33748a..7284ccec60 100644
--- a/bbot/modules/smuggler.py
+++ b/bbot/modules/smuggler.py
@@ -9,10 +9,9 @@
class smuggler(BaseModule):
-
watched_events = ["URL"]
produced_events = ["FINDING"]
- flags = ["active", "aggressive", "web-advanced", "slow", "brute-force"]
+ flags = ["active", "aggressive", "slow", "web-thorough"]
meta = {"description": "Check for HTTP smuggling"}
in_scope_only = True
@@ -29,7 +28,6 @@ def setup(self):
return True
def handle_event(self, event):
-
host = f"{event.parsed.scheme}://{event.parsed.netloc}/"
host_hash = hash(host)
if host_hash in self.scanned_hosts:
diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py
index 85843d054c..f424eeecbb 100644
--- a/bbot/modules/sslcert.py
+++ b/bbot/modules/sslcert.py
@@ -3,6 +3,7 @@
import threading
from OpenSSL import SSL
from ssl import PROTOCOL_TLSv1
+from contextlib import suppress
from bbot.modules.base import BaseModule
from bbot.core.errors import ValidationError
@@ -12,26 +13,41 @@
class sslcert(BaseModule):
watched_events = ["OPEN_TCP_PORT"]
produced_events = ["DNS_NAME", "EMAIL_ADDRESS"]
- flags = ["affiliates", "subdomain-enum", "email-enum", "active", "safe"]
+ flags = ["affiliates", "subdomain-enum", "email-enum", "active", "safe", "web-basic", "web-thorough"]
meta = {
"description": "Visit open ports and retrieve SSL certificates",
}
- options = {"timeout": 5.0}
- options_desc = {"timeout": "Socket connect timeout in seconds"}
+ options = {"timeout": 5.0, "skip_non_ssl": True}
+ options_desc = {"timeout": "Socket connect timeout in seconds", "skip_non_ssl": "Don't try common non-SSL ports"}
deps_apt = ["openssl"]
deps_pip = ["pyOpenSSL"]
- max_event_handlers = 50
+ max_threads = 50
+ max_event_handlers = 25
scope_distance_modifier = 0
_priority = 2
def setup(self):
+ self.timeout = self.config.get("timeout", 5.0)
+ self.skip_non_ssl = self.config.get("skip_non_ssl", True)
+ self.non_ssl_ports = (22, 53, 80)
+
+ # sometimes we run into a server with A LOT of SANs
+ # these are usually stupid and useless, so we abort based on a different threshold
+ # depending on whether the source event is in scope
+ self.in_scope_abort_threshold = 50
+ self.out_of_scope_abort_threshold = 10
+
self.hosts_visited = set()
self.hosts_visited_lock = threading.Lock()
self.ip_lock = NamedLock()
return True
- def handle_event(self, event):
+ def filter_event(self, event):
+ if self.skip_non_ssl and event.port in self.non_ssl_ports:
+ return False, f"Port {event.port} doesn't typically use SSL"
+ return True
+ def handle_event(self, event):
_host = event.host
if event.port:
port = event.port
@@ -44,26 +60,46 @@ def handle_event(self, event):
else:
hosts = list(self.helpers.resolve(_host))
+ futures = {}
for host in hosts:
- for event_type, event_data in self.visit_host(host, port):
- if event_data is not None and event_data != event:
- self.debug(f"Discovered new {event_type} via SSL certificate parsing: [{event_data}]")
- try:
- ssl_event = self.make_event(event_data, event_type, source=event, raise_error=True)
- if ssl_event:
- self.emit_event(ssl_event)
- except ValidationError as e:
- self.hugeinfo(f'Malformed {event_type} "{event_data}" at {event.data}')
- self.debug(f"Invalid data at {host}:{port}: {e}")
+ future = self.submit_task(self.visit_host, host, port)
+ futures[future] = host
+
+ if event.scope_distance == 0:
+ abort_threshold = self.in_scope_abort_threshold
+ else:
+ abort_threshold = self.out_of_scope_abort_threshold
+ for future in self.helpers.as_completed(futures):
+ host = futures[future]
+ dns_names, emails = future.result()
+ if len(dns_names) > abort_threshold:
+ netloc = self.helpers.make_netloc(host, port)
+ self.info(
+ f"Skipping Subject Alternate Names (SANs) on {netloc} because number of hostnames ({len(dns_names):,}) exceeds threshold ({abort_threshold})"
+ )
+ dns_names = dns_names[:1]
+ for event_type, results in (("DNS_NAME", dns_names), ("EMAIL_ADDRESS", emails)):
+ for event_data in results:
+ if event_data is not None and event_data != event:
+ self.debug(f"Discovered new {event_type} via SSL certificate parsing: [{event_data}]")
+ try:
+ ssl_event = self.make_event(event_data, event_type, source=event, raise_error=True)
+ if ssl_event:
+ self.emit_event(ssl_event)
+ except ValidationError as e:
+ self.hugeinfo(f'Malformed {event_type} "{event_data}" at {event.data}')
+ self.debug(f"Invalid data at {host}:{port}: {e}")
def visit_host(self, host, port):
host = self.helpers.make_ip_type(host)
host_hash = hash((host, port))
+ dns_names = []
+ emails = set()
with self.ip_lock.get_lock(host_hash):
with self.hosts_visited_lock:
if host_hash in self.hosts_visited:
self.debug(f"Already processed {host} on port {port}, skipping")
- return None, None
+ return [], []
else:
self.hosts_visited.add(host_hash)
@@ -73,15 +109,14 @@ def visit_host(self, host, port):
socket_type = socket.AF_INET6
host = str(host)
sock = socket.socket(socket_type, socket.SOCK_STREAM)
- timeout = self.config.get("timeout", 5.0)
- sock.settimeout(timeout)
+ sock.settimeout(self.timeout)
context = SSL.Context(PROTOCOL_TLSv1)
self.debug(f"Connecting to {host} on port {port}")
try:
sock.connect((host, port))
except Exception as e:
self.debug(f"Error connecting to {host} on port {port}: {e}")
- return None, None
+ return [], []
connection = SSL.Connection(context, sock)
connection.set_tlsext_host_name(self.helpers.smart_encode(host))
connection.set_connect_state()
@@ -92,29 +127,29 @@ def visit_host(self, host, port):
except SSL.WantReadError:
rd, _, _ = select.select([sock], [], [], sock.gettimeout())
if not rd:
- raise timeout("select timed out")
+ raise SSL.Error("select timed out")
continue
break
except Exception as e:
self.debug(f"Error with SSL handshake on {host} port {port}: {e}")
- return None, None
+ return [], []
cert = connection.get_peer_certificate()
sock.close()
issuer = cert.get_issuer()
if issuer.emailAddress and self.helpers.regexes.email_regex.match(issuer.emailAddress):
- yield "EMAIL_ADDRESS", issuer.emailAddress
+ emails.add(issuer.emailAddress)
subject = cert.get_subject()
if subject.emailAddress and self.helpers.regexes.email_regex.match(subject.emailAddress):
- yield "EMAIL_ADDRESS", subject.emailAddress
- common_name = subject.commonName
- cert_results = self.get_cert_sans(cert)
- cert_results.append(str(common_name).lstrip("*.").lower())
- for c in set(cert_results):
- yield "DNS_NAME", c
+ emails.add(subject.emailAddress)
+ common_name = str(subject.commonName).lstrip("*.").lower()
+ dns_names = set(self.get_cert_sans(cert))
+ with suppress(KeyError):
+ dns_names.remove(common_name)
+ dns_names = [common_name] + list(dns_names)
+ return dns_names, list(emails)
@staticmethod
def get_cert_sans(cert):
-
sans = []
raw_sans = None
ext_count = cert.get_extension_count()
diff --git a/bbot/modules/subdomain_hijack.py b/bbot/modules/subdomain_hijack.py
new file mode 100644
index 0000000000..73833fd73f
--- /dev/null
+++ b/bbot/modules/subdomain_hijack.py
@@ -0,0 +1,127 @@
+import re
+import json
+import requests
+
+from bbot.modules.base import BaseModule
+from bbot.core.helpers.misc import tldextract
+
+
+class subdomain_hijack(BaseModule):
+ flags = ["subdomain-hijack", "subdomain-enum", "cloud-enum", "safe", "active", "web-basic", "web-thorough"]
+ watched_events = ["DNS_NAME"]
+ produced_events = ["FINDING"]
+ meta = {"description": "Detect hijackable subdomains"}
+ options = {
+ "fingerprints": "https://raw.githubusercontent.com/EdOverflow/can-i-take-over-xyz/master/fingerprints.json"
+ }
+ options_desc = {"fingerprints": "URL or path to fingerprints.json"}
+ scope_distance_modifier = 2
+ max_event_handlers = 5
+
+ def setup(self):
+ fingerprints_url = self.config.get("fingerprints")
+ fingerprints_file = self.helpers.wordlist(fingerprints_url)
+ with open(fingerprints_file) as f:
+ fingerprints = json.load(f)
+ self.fingerprints = []
+ for f in fingerprints:
+ try:
+ f = Fingerprint(f)
+ except Exception as e:
+ self.warning(f"Error instantiating fingerprint: {e}")
+ continue
+ if not (f.domains and f.vulnerable and f.fingerprint and f.cicd_pass):
+ self.debug(f"Skipping fingerprint: {f}")
+ continue
+ self.debug(f"Processed fingerprint: {f}")
+ self.fingerprints.append(f)
+ if not self.fingerprints:
+ return None, "No valid fingerprints"
+ self.debug(f"Successfully processed {len(self.fingerprints):,} fingerprints")
+ return True
+
+ def handle_event(self, event):
+ hijackable, reason = self.check_subdomain(event)
+ if hijackable:
+ source_hosts = []
+ e = event
+ while 1:
+ host = getattr(e, "host", "")
+ if host:
+ if e not in source_hosts:
+ source_hosts.append(e)
+ e = e.get_source()
+ else:
+ break
+
+ url = f"https://{event.host}"
+ description = f'Hijackable Subdomain "{event.data}": {reason}'
+ source_hosts = source_hosts[::-1]
+ if source_hosts:
+ source_hosts_str = str(source_hosts[0].host)
+ for e in source_hosts[1:]:
+ source_hosts_str += f" -[{e.module.name}]-> {e.host}"
+ description += f" ({source_hosts_str})"
+ self.emit_event({"host": event.host, "url": url, "description": description}, "FINDING", source=event)
+ else:
+ self.debug(reason)
+
+ def check_subdomain(self, event):
+ for f in self.fingerprints:
+ for domain in f.domains:
+ self_matches = self.helpers.host_in_host(event.data, domain)
+ child_matches = any(self.helpers.host_in_host(domain, h) for h in event.resolved_hosts)
+ if self_matches or child_matches:
+ for scheme in ("https", "http"):
+ # first, try base request
+ url = f"{scheme}://{event.data}"
+ match, reason = self._verify_fingerprint(f, url)
+ if match:
+ return match, reason
+ # next, try {random_domain} -[DNS]-> domain
+ url = f"{scheme}://{domain}"
+ headers = {"Host": event.data}
+ match, reason = self._verify_fingerprint(f, url, headers=headers)
+ if match:
+ return match, reason
+ return False, f'Subdomain "{event.data}" not hijackable'
+
+ def _verify_fingerprint(self, fingerprint, *args, **kwargs):
+ kwargs["raise_error"] = True
+ if fingerprint.http_status is not None:
+ kwargs["allow_redirects"] = False
+ try:
+ r = self.helpers.request(*args, **kwargs)
+ if fingerprint.http_status is not None and r.status_code == fingerprint.http_status:
+ return True, f"HTTP status == {fingerprint.http_status}"
+ text = getattr(r, "text", "")
+ if (
+ not fingerprint.nxdomain
+ and not fingerprint.http_status
+ and fingerprint.fingerprint_regex.findall(text)
+ ):
+ return True, "Fingerprint match"
+ except requests.exceptions.RequestException as e:
+ if fingerprint.nxdomain and "Name or service not known" in str(e):
+ return True, f"NXDOMAIN"
+ return False, "No match"
+
+
+class Fingerprint:
+ def __init__(self, fingerprint):
+ assert isinstance(fingerprint, dict), "fingerprint must be a dictionary"
+ self.engine = fingerprint.get("service")
+ self.cnames = fingerprint.get("cname", [])
+ self.domains = list(set([tldextract(c).registered_domain for c in self.cnames]))
+ self.http_status = fingerprint.get("http_status", None)
+ self.nxdomain = fingerprint.get("nxdomain", False)
+ self.vulnerable = fingerprint.get("vulnerable", False)
+ self.fingerprint = fingerprint.get("fingerprint", "")
+ self.cicd_pass = fingerprint.get("cicd_pass", False)
+ try:
+ self.fingerprint_regex = re.compile(self.fingerprint, re.MULTILINE)
+ except re.error:
+ self.fingerprint_regex = re.compile(re.escape(self.fingerprint), re.MULTILINE)
+
+ def __str__(self):
+ return f"{self.engine}: {self.fingerprint} (cnames: {self.cnames}, vulnerable: {self.vulnerable}, cicd_pass: {self.cicd_pass})"
diff --git a/bbot/modules/sublist3r.py b/bbot/modules/sublist3r.py
index a278c11266..ee15a145b0 100644
--- a/bbot/modules/sublist3r.py
+++ b/bbot/modules/sublist3r.py
@@ -4,7 +4,8 @@
class sublist3r(crobat):
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
- flags = ["subdomain-enum", "passive", "safe"]
+ # tag "subdomain-enum" removed 2023-02-24 because API is offline
+ flags = ["passive", "safe"]
meta = {
"description": "Query sublist3r's API for subdomains",
}
@@ -12,10 +13,4 @@ class sublist3r(crobat):
base_url = "https://api.sublist3r.com/search.php"
def request_url(self, query):
- return self.helpers.request(f"{self.base_url}?domain={query}")
-
- def parse_results(self, r, query):
- json = r.json()
- if json:
- for hostname in json:
- yield hostname
+ return self.request_with_fail_count(f"{self.base_url}?domain={query}", timeout=self.http_timeout + 10)
diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py
index 1dd40754ab..dcf0296133 100644
--- a/bbot/modules/telerik.py
+++ b/bbot/modules/telerik.py
@@ -4,10 +4,9 @@
class telerik(BaseModule):
-
watched_events = ["URL"]
produced_events = ["VULNERABILITY", "FINDING"]
- flags = ["active", "aggressive", "slow", "web-basic"]
+ flags = ["active", "aggressive", "slow", "web-thorough"]
meta = {"description": "Scan for critical Telerik vulnerabilities"}
telerikVersions = [
@@ -105,7 +104,7 @@ class telerik(BaseModule):
"AsiCommon/Controls/ContentManagement/ContentDesigner/Telerik.Web.UI.DialogHandler.aspx",
"cms/portlets/telerik.web.ui.dialoghandler.aspx",
"common/admin/Calendar/Telerik.Web.UI.DialogHandler.aspx",
- "common/admin/Jobs2/Telerik.Web.UI.DialogHandler.aspx,"
+ "common/admin/Jobs2/Telerik.Web.UI.DialogHandler.aspx",
"common/admin/PhotoGallery2/Telerik.Web.UI.DialogHandler.aspx",
"dashboard/UserControl/CMS/Page/Telerik.Web.UI.DialogHandler.aspx",
"DesktopModule/UIQuestionControls/UIAskQuestion/Telerik.Web.UI.DialogHandler.aspx",
@@ -159,10 +158,10 @@ class telerik(BaseModule):
def setup(self):
self.scanned_hosts = set()
+ self.timeout = self.scan.config.get("httpx_timeout", 5)
return True
def handle_event(self, event):
-
host = f"{event.parsed.scheme}://{event.parsed.netloc}/"
host_hash = hash(host)
if host_hash in self.scanned_hosts:
@@ -224,6 +223,11 @@ def handle_event(self, event):
for future in self.helpers.as_completed(futures):
dh = futures[future]
result = future.result()
+ # cancel if we run into timeouts etc.
+ if result is None:
+ self.debug(f"Cancelling run against {event.data} due to failed request")
+ for future in futures:
+ future.cancel()
if result:
if "Cannot deserialize dialog parameters" in result.text:
for future in futures:
@@ -237,7 +241,6 @@ def handle_event(self, event):
event,
)
# Once we have a match we need to stop, because the basic handler (Telerik.Web.UI.DialogHandler.aspx) usually works with a path wildcard
-
break
spellcheckhandler = "Telerik.Web.UI.SpellCheckHandler.axd"
@@ -264,17 +267,15 @@ def handle_event(self, event):
pass
def test_detector(self, baseurl, detector):
-
result = None
if "/" != baseurl[-1]:
url = f"{baseurl}/{detector}"
else:
url = f"{baseurl}{detector}"
- result = self.helpers.request(url)
+ result = self.helpers.request(url, timeout=self.timeout)
return result
def filter_event(self, event):
-
if "endpoint" in event.tags:
return False
else:
diff --git a/bbot/modules/threatminer.py b/bbot/modules/threatminer.py
index e152c8c12e..3f6c99f2fe 100644
--- a/bbot/modules/threatminer.py
+++ b/bbot/modules/threatminer.py
@@ -12,7 +12,7 @@ class threatminer(crobat):
base_url = "https://api.threatminer.org/v2"
def request_url(self, query):
- return self.helpers.request(f"{self.base_url}/domain.php?q={self.helpers.quote(query)}&rt=5")
+ return self.request_with_fail_count(f"{self.base_url}/domain.php?q={self.helpers.quote(query)}&rt=5")
def parse_results(self, r, query):
j = r.json()
diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py
index 8d6dbe4aec..91eb4c5c50 100644
--- a/bbot/modules/url_manipulation.py
+++ b/bbot/modules/url_manipulation.py
@@ -3,10 +3,9 @@
class url_manipulation(BaseModule):
-
watched_events = ["URL"]
produced_events = ["FINDING"]
- flags = ["active", "aggressive", "web-advanced"]
+ flags = ["active", "aggressive", "web-thorough"]
meta = {"description": "Attempt to identify URL parsing/routing based vulnerabilities"}
in_scope_only = True
@@ -40,15 +39,19 @@ def setup(self):
return True
def handle_event(self, event):
-
try:
- compare_helper = self.helpers.http_compare(event.data, allow_redirects=self.allow_redirects)
+ compare_helper = self.helpers.http_compare(
+ event.data, allow_redirects=self.allow_redirects, include_cache_buster=False
+ )
except HttpCompareError as e:
self.debug(e)
return
- for sig in self.signatures:
+ if compare_helper.canary_check(event.data, mode="getparam") == False:
+ self.verbose(f'Aborting "{event.data}" due to failed canary check')
+ return
+ for sig in self.signatures:
sig = self.format_signature(sig, event)
match, reasons, reflection, subject_response = compare_helper.compare(
sig[1], method=sig[0], allow_redirects=self.allow_redirects
@@ -62,7 +65,6 @@ def handle_event(self, event):
if self.rand_string not in subject_content:
if match == False:
if str(subject_response.status_code).startswith("2"):
-
if "body" in reasons:
reported_signature = f"Modified URL: {sig[1]}"
description = f"Url Manipulation: [{','.join(reasons)}] Sig: [{reported_signature}]"
@@ -77,7 +79,6 @@ def handle_event(self, event):
self.debug("Ignoring positive result due to presence of parameter name in result")
def filter_event(self, event):
-
accepted_status_codes = ["200", "301", "302"]
for c in accepted_status_codes:
diff --git a/bbot/modules/viewdns.py b/bbot/modules/viewdns.py
index 1176258454..58c76dbe50 100644
--- a/bbot/modules/viewdns.py
+++ b/bbot/modules/viewdns.py
@@ -5,6 +5,9 @@
class viewdns(BaseModule):
+ """
+ Used as a base for modules that only act on root domains and not individual hostnames
+ """
watched_events = ["DNS_NAME"]
produced_events = ["DNS_NAME"]
@@ -12,7 +15,7 @@ class viewdns(BaseModule):
meta = {
"description": "Query viewdns.info's reverse whois for related domains",
}
- deps_pip = ["beautifulsoup4", "lxml"]
+ deps_pip = ["bs4", "lxml"]
base_url = "https://viewdns.info"
in_scope_only = True
_qsize = 1
diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py
index 5eb0136943..1a15985620 100644
--- a/bbot/modules/virustotal.py
+++ b/bbot/modules/virustotal.py
@@ -22,7 +22,7 @@ def ping(self):
def request_url(self, query):
url = f"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains"
- return self.helpers.request(url, headers=self.headers)
+ return self.request_with_fail_count(url, headers=self.headers)
def parse_results(self, r, query):
results = set()
diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py
new file mode 100644
index 0000000000..79e169e4e0
--- /dev/null
+++ b/bbot/modules/wafw00f.py
@@ -0,0 +1,48 @@
+from bbot.modules.base import BaseModule
+from wafw00f import main as wafw00f_main
+
+
+class wafw00f(BaseModule):
+ """
+ https://github.com/EnableSecurity/wafw00f
+ """
+
+ watched_events = ["URL"]
+ produced_events = ["WAF"]
+ flags = ["active", "aggressive"]
+ meta = {"description": "Web Application Firewall Fingerprinting Tool"}
+
+ deps_pip = ["wafw00f"]
+
+ options = {"generic_detect": True}
+ options_desc = {"generic_detect": "When no specific WAF detections are made, try to peform a generic detect"}
+
+ in_scope_only = True
+
+ def setup(self):
+ self.scanned_hosts = set()
+ return True
+
+ def handle_event(self, event):
+ parsed_host = event.parsed
+ host = f"{parsed_host.scheme}://{parsed_host.netloc}/"
+ host_hash = hash(host)
+ if host_hash in self.scanned_hosts:
+ self.debug(f"Host {host} was already scanned, exiting")
+ return
+ else:
+ self.scanned_hosts.add(host_hash)
+
+ WW = wafw00f_main.WAFW00F(host)
+ waf_detections = WW.identwaf()
+ if waf_detections:
+ for waf in WW.identwaf():
+ self.emit_event({"host": host, "WAF": waf}, "WAF", source=event)
+ else:
+ if self.config.get("generic_detect") == True:
+ if WW.genericdetect():
+ self.emit_event(
+ {"host": host, "WAF": "generic detection", "info": WW.knowledge["generic"]["reason"]},
+ "WAF",
+ source=event,
+ )
diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py
index 27f5b1f2e0..579ee8bb9d 100644
--- a/bbot/modules/wappalyzer.py
+++ b/bbot/modules/wappalyzer.py
@@ -11,10 +11,9 @@
class wappalyzer(BaseModule):
-
watched_events = ["HTTP_RESPONSE"]
produced_events = ["TECHNOLOGY"]
- flags = ["active", "safe", "web-basic"]
+ flags = ["active", "safe", "web-basic", "web-thorough"]
meta = {
"description": "Extract technologies from web responses",
}
diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py
index fef2ecd11f..6ce7751526 100644
--- a/bbot/modules/wayback.py
+++ b/bbot/modules/wayback.py
@@ -29,7 +29,7 @@ def handle_event(self, event):
def query(self, query):
waybackurl = f"{self.base_url}/cdx/search/cdx?url={self.helpers.quote(query)}&matchType=domain&output=json&fl=original&collapse=original"
- r = self.helpers.request(waybackurl)
+ r = self.helpers.request(waybackurl, timeout=self.http_timeout + 10)
if not r:
self.warning(f'Error connecting to archive.org for query "{query}"')
return
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index bd9654e386..47a34169be 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -6,6 +6,7 @@
from contextlib import suppress
from datetime import datetime, timedelta
+from ..core.helpers.queueing import EventQueue
from ..core.errors import ScanCancelledError, ValidationError
log = logging.getLogger("bbot.scanner.manager")
@@ -18,7 +19,7 @@ class ScanManager:
def __init__(self, scan):
self.scan = scan
- self.queued_event_types = dict()
+ self.incoming_event_queue = EventQueue()
# tracks duplicate events on a global basis
self.events_distributed = set()
@@ -27,9 +28,14 @@ def __init__(self, scan):
self.events_accepted = set()
self.events_accepted_lock = threading.Lock()
+ self._lock = threading.Lock()
+ self.event_emitted = threading.Condition(self._lock)
+
self.events_resolved = dict()
self.dns_resolution = self.scan.config.get("dns_resolution", False)
+ self.last_log_time = datetime.now()
+
def init_events(self):
"""
seed scanner with target events
@@ -38,12 +44,18 @@ def init_events(self):
sorted_events = sorted(self.scan.target.events, key=lambda e: len(e.data))
for event in sorted_events:
self.scan.verbose(f"Target: {event}")
- self.emit_event(event)
+ self.emit_event(event, _block=False, _force_submit=True)
# force submit batches
for mod in self.scan.modules.values():
mod._handle_batch(force=True)
def emit_event(self, event, *args, **kwargs):
+ """
+ TODO: Register + kill duplicate events immediately?
+ bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com"))
+ bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com"))
+ bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com"))
+ """
# skip event if it fails precheck
if not self._event_precheck(event):
event._resolved.set()
@@ -53,8 +65,8 @@ def emit_event(self, event, *args, **kwargs):
quick = kwargs.pop("quick", False)
if quick:
event._resolved.set()
- kwargs.pop("abort_if", None)
- kwargs.pop("on_success_callback", None)
+ for kwarg in ["abort_if", "on_success_callback", "_block"]:
+ kwargs.pop(kwargs, None)
try:
self.distribute_event(event, *args, **kwargs)
return True
@@ -62,7 +74,7 @@ def emit_event(self, event, *args, **kwargs):
return False
except Exception as e:
log.error(f"Unexpected error in manager.emit_event(): {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
else:
# don't raise an exception if the thread pool has been shutdown
try:
@@ -70,9 +82,12 @@ def emit_event(self, event, *args, **kwargs):
return True
except ScanCancelledError:
return False
+ except queue.Full:
+ raise
except Exception as e:
log.error(f"Unexpected error in manager.emit_event(): {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
+ finally:
event._resolved.set()
return False
@@ -89,16 +104,6 @@ def _event_precheck(self, event, exclude=("DNS_NAME",)):
if self.is_duplicate_event(event):
log.debug(f"Skipping {event} because it is a duplicate")
return False
-
- # this is disabled because it doesn't consider the potential for new dns children / speculate derivatives
- # if event.type not in exclude:
- # any_acceptable = False
- # for mod in self.scan.modules.values():
- # acceptable, reason = mod._filter_event(event, precheck_only=True)
- # any_acceptable |= acceptable
- # if not any_acceptable:
- # log.debug(f"Skipping {event} because no modules would accept it")
- # return any_acceptable
return True
def _emit_event(self, event, *args, **kwargs):
@@ -109,21 +114,15 @@ def _emit_event(self, event, *args, **kwargs):
abort_if = kwargs.pop("abort_if", None)
log.debug(f'Module "{event.module}" raised {event}')
- # Wait for parent event to resolve (in case its scope distance changes)`
- while 1:
- if self.scan.stopping:
- raise ScanCancelledError()
- resolved = event.source._resolved.wait(timeout=0.1)
- if resolved:
- # update event's scope distance based on its parent
- event.scope_distance = event.source.scope_distance + 1
- break
-
# skip DNS resolution if it's disabled in the config and the event is a target and we don't have a blacklist
skip_dns_resolution = (not self.dns_resolution) and "target" in event.tags and not self.scan.blacklist
if skip_dns_resolution:
event._resolved.set()
- event.tags.add("resolved")
+ dns_children = []
+ dns_tags = {"resolved"}
+ event_whitelisted_dns = True
+ event_blacklisted_dns = False
+ resolved_hosts = []
else:
# DNS resolution
(
@@ -132,71 +131,80 @@ def _emit_event(self, event, *args, **kwargs):
event_whitelisted_dns,
event_blacklisted_dns,
resolved_hosts,
- ) = self.scan.helpers.dns.resolve_event(event)
-
- # kill runaway DNS chains
- dns_resolve_distance = getattr(event, "dns_resolve_distance", 0)
- if dns_resolve_distance >= self.scan.helpers.dns.dns_resolve_distance:
- log.debug(
- f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.scan.helpers.dns.dns_resolve_distance})"
- )
- dns_children = []
-
- # We do this again in case event.data changed during resolve_event()
- if event.type == "DNS_NAME" and not self._event_precheck(event, exclude=()):
- log.debug(f"Omitting due to failed precheck: {event}")
- distribute_event = False
-
- event._resolved_hosts = resolved_hosts
-
- event_whitelisted = event_whitelisted_dns | self.scan.whitelisted(event)
- event_blacklisted = event_blacklisted_dns | self.scan.blacklisted(event)
- if event.type in ("DNS_NAME", "IP_ADDRESS"):
- event.tags.update(dns_tags)
- if event_blacklisted:
- event.tags.add("blacklisted")
-
- # Cloud tagging
- for provider in self.scan.helpers.cloud.providers.values():
- provider.tag_event(event)
-
- # Blacklist purging
- if "blacklisted" in event.tags:
- reason = "event host"
- if event_blacklisted_dns:
- reason = "DNS associations"
- log.debug(f"Omitting due to blacklisted {reason}: {event}")
- distribute_event = False
-
- # Scope shepherding
- event_is_duplicate = self.is_duplicate_event(event)
- event_in_report_distance = event.scope_distance <= self.scan.scope_report_distance
- set_scope_distance = event.scope_distance
- if event_whitelisted:
- set_scope_distance = 0
- if event.host:
- if (event_whitelisted or event_in_report_distance) and not event_is_duplicate:
- if set_scope_distance == 0:
- log.debug(f"Making {event} in-scope")
- source_trail = event.make_in_scope(set_scope_distance)
- for s in source_trail:
- self.emit_event(s)
- else:
- if event.scope_distance > self.scan.scope_report_distance:
- log.debug(
- f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})"
- )
- event.make_internal()
- else:
- log.debug(f"Making {event} in-scope because it does not have identifying scope information")
- source_trail = event.make_in_scope(0)
+ ) = self.scan.helpers.dns.resolve_event(event, minimal=not self.dns_resolution)
+
+ # kill runaway DNS chains
+ dns_resolve_distance = getattr(event, "dns_resolve_distance", 0)
+ if dns_resolve_distance >= self.scan.helpers.dns.max_dns_resolve_distance:
+ log.debug(
+ f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.scan.helpers.dns.max_dns_resolve_distance})"
+ )
+ dns_children = []
+
+ # We do this again in case event.data changed during resolve_event()
+ if event.type == "DNS_NAME" and not self._event_precheck(event, exclude=()):
+ log.debug(f"Omitting due to failed precheck: {event}")
+ distribute_event = False
+
+ if event.type in ("DNS_NAME", "IP_ADDRESS"):
+ event.tags.update(dns_tags)
+
+ event._resolved_hosts = resolved_hosts
+
+ event_whitelisted = event_whitelisted_dns | self.scan.whitelisted(event)
+ event_blacklisted = event_blacklisted_dns | self.scan.blacklisted(event)
+ if event_blacklisted:
+ event.add_tag("blacklisted")
+
+ # Blacklist purging
+ if "blacklisted" in event.tags:
+ reason = "event host"
+ if event_blacklisted_dns:
+ reason = "DNS associations"
+ log.debug(f"Omitting due to blacklisted {reason}: {event}")
+ distribute_event = False
+
+ # Cloud tagging
+ for provider in self.scan.helpers.cloud.providers.values():
+ provider.tag_event(event)
+
+ # Scope shepherding
+ event_is_duplicate = self.is_duplicate_event(event)
+ event_in_report_distance = event.scope_distance <= self.scan.scope_report_distance
+ set_scope_distance = event.scope_distance
+ if event_whitelisted:
+ set_scope_distance = 0
+ if event.host:
+ if (event_whitelisted or event_in_report_distance) and not event_is_duplicate:
+ if set_scope_distance == 0:
+ log.debug(f"Making {event} in-scope")
+ source_trail = event.make_in_scope(set_scope_distance)
for s in source_trail:
- self.emit_event(s)
+ self.emit_event(s, _block=False, _force_submit=True)
+ else:
+ if event.scope_distance > self.scan.scope_report_distance:
+ log.debug(
+ f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})"
+ )
+ event.make_internal()
+ if not event.host or (event.always_emit and not event_is_duplicate):
+ log.debug(
+ f"Force-emitting {event} because it does not have identifying scope information or because always_emit was True"
+ )
+ source_trail = event.unmake_internal(force_output=True)
+ for s in source_trail:
+ self.emit_event(s, _block=False, _force_submit=True)
# now that the event is properly tagged, we can finally make decisions about it
- if callable(abort_if) and abort_if(event):
- log.debug(f"{event.module}: not raising event {event} due to custom criteria in abort_if()")
- return
+ if callable(abort_if):
+ abort_result = abort_if(event)
+ msg = f"{event.module}: not raising event {event} due to custom criteria in abort_if()"
+ with suppress(ValueError, TypeError):
+ abort_result, reason = abort_result
+ msg += f": {reason}"
+ if abort_result:
+ log.debug(msg)
+ return
if not self.accept_event(event):
return
@@ -219,16 +227,18 @@ def _emit_event(self, event, *args, **kwargs):
and not str(event.module) == "speculate"
):
source_module = self.scan.helpers._make_dummy_module("host", _type="internal")
+ source_module._priority = 4
source_event = self.scan.make_event(event.host, "DNS_NAME", module=source_module, source=event)
source_event.scope_distance = event.scope_distance
if "target" in event.tags:
- source_event.tags.add("target")
- self.emit_event(source_event)
+ source_event.add_tag("target")
+ self.emit_event(source_event, _block=False, _force_submit=True)
if self.dns_resolution and emit_children:
dns_child_events = []
if dns_children:
for rdtype, record in dns_children:
module = self.scan.helpers.dns._get_dummy_module(rdtype)
+ module._priority = 4
try:
child_event = self.scan.make_event(record, "DNS_NAME", module=module, source=source_event)
dns_child_events.append(child_event)
@@ -237,15 +247,18 @@ def _emit_event(self, event, *args, **kwargs):
f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}'
)
for child_event in dns_child_events:
- self.emit_event(child_event)
+ self.emit_event(child_event, _block=False, _force_submit=True)
except ValidationError as e:
log.warning(f"Event validation failed with args={args}, kwargs={kwargs}: {e}")
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
finally:
+ event._resolved.set()
if event_distributed:
self.scan.stats.event_distributed(event)
+ with self.event_emitted:
+ self.event_emitted.notify()
log.debug(f"{event.module}.emit_event() finished for {event}")
def hash_event(self, event):
@@ -287,32 +300,33 @@ def catch(self, callback, *args, **kwargs):
ret = None
on_finish_callback = kwargs.pop("_on_finish_callback", None)
force = kwargs.pop("_force", False)
- start_time = datetime.now()
- callback_name = f"{callback.__qualname__}({args}, {kwargs})"
+ fn = callback
+ for arg in args:
+ if callable(arg):
+ fn = arg
+ else:
+ break
try:
if not self.scan.stopping or force:
ret = callback(*args, **kwargs)
except ScanCancelledError as e:
- log.debug(f"ScanCancelledError in {callback.__qualname__}(): {e}")
+ log.debug(f"ScanCancelledError in {fn.__qualname__}(): {e}")
except BrokenPipeError as e:
- log.debug(f"BrokenPipeError in {callback.__qualname__}(): {e}")
+ log.debug(f"BrokenPipeError in {fn.__qualname__}(): {e}")
except Exception as e:
- log.error(f"Error in {callback.__qualname__}(): {e}")
- log.debug(traceback.format_exc())
+ log.error(f"Error in {fn.__qualname__}(): {e}")
+ log.trace(traceback.format_exc())
except KeyboardInterrupt:
log.debug(f"Interrupted")
self.scan.stop()
- finally:
- run_time = datetime.now() - start_time
- self.scan.stats.function_called(callback_name, run_time)
if callable(on_finish_callback):
try:
on_finish_callback()
except Exception as e:
log.error(
- f"Error in on_finish_callback {on_finish_callback.__qualname__}() after {callback.__qualname__}(): {e}"
+ f"Error in on_finish_callback {on_finish_callback.__qualname__}() after {fn.__qualname__}(): {e}"
)
- log.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
return ret
def distribute_event(self, *args, **kwargs):
@@ -320,10 +334,6 @@ def distribute_event(self, *args, **kwargs):
Queue event with modules
"""
event = self.scan.make_event(*args, **kwargs)
- try:
- self.queued_event_types[event.type] += 1
- except KeyError:
- self.queued_event_types[event.type] = 1
event_hash = hash(event)
dup = event_hash in self.events_distributed
@@ -340,7 +350,7 @@ def distribute_event(self, *args, **kwargs):
event_within_scope_distance = -1 < event.scope_distance <= self.scan.scope_search_distance
event_within_report_distance = -1 < event.scope_distance <= self.scan.scope_report_distance
if mod._type == "output":
- if event_within_report_distance or (event._force_output and mod.emit_graph_trail):
+ if event_within_report_distance or event._force_output:
mod.queue_event(event)
if not stats_recorded:
stats_recorded = True
@@ -349,78 +359,63 @@ def distribute_event(self, *args, **kwargs):
if event_within_scope_distance:
mod.queue_event(event)
- def loop_until_finished(self, status_frequency=10):
- iteration = 0
+ def loop_until_finished(self):
modules = list(self.scan.modules.values())
- num_modules = len(modules)
activity = True
- timedelta_2secs = timedelta(seconds=status_frequency)
- last_log_time = datetime.now()
try:
-
self.scan.dispatcher.on_start(self.scan)
while 1:
# abort if we're aborting
if self.scan.aborting:
- # Empty incoming and outgoing module event queue
+ # Empty event queues
for module in self.scan.modules.values():
with suppress(queue.Empty):
while 1:
module.incoming_event_queue.get_nowait()
- with suppress(queue.Empty):
- while 1:
- module.outgoing_event_queue.get_nowait()
+ with suppress(queue.Empty):
+ while 1:
+ self.incoming_event_queue.get_nowait()
break
- # print status every 2 seconds
- now = datetime.now()
- time_since_last_log = now - last_log_time
- if time_since_last_log > timedelta_2secs:
- self.modules_status(_log=True, passes=1)
- last_log_time = now
-
if "python" in self.scan.modules:
events, finish, report = self.scan.modules["python"].events_waiting
yield from events
- # pause if the thread pool queue is full
- while (
- not self.scan.aborting and self.scan._event_thread_pool.qsize >= self.scan._event_thread_pool_qsize
- ):
- sleep(0.05)
-
- module_index = iteration % num_modules
- module = modules[module_index]
- end = module_index == num_modules - 1
try:
- event, kwargs = module.outgoing_event_queue.get_nowait()
- acceptable = self.emit_event(event, **kwargs)
- if acceptable:
- activity = True
+ self.log_status()
+ event, kwargs = self.incoming_event_queue.get_nowait()
+ while not self.scan.aborting:
+ try:
+ acceptable = self.emit_event(event, _block=False, **kwargs)
+ if acceptable:
+ activity = True
+ break
+ except queue.Full:
+ self.log_status()
+ with self.event_emitted:
+ self.event_emitted.wait(timeout=0.1)
except queue.Empty:
# if we're on the last module
- if end:
- finished = self.modules_status().get("finished", False)
- # And if the scan is finished
- if finished:
- # And if new events were generated since last time we were here
- if activity:
- activity = False
- self.scan.status = "FINISHING"
- # Trigger .finished() on every module and start over
- log.info("Finishing scan")
- for module in modules:
- module.queue_event("FINISHED")
- else:
- # Otherwise stop the scan if no new events were generated since last time
- break
- continue
- # save on CPU
- sleep(0.1)
- finally:
- iteration += 1
+ modules_status = self.modules_status()
+ finished = modules_status.get("finished", False)
+ # And if the scan is finished
+ if finished:
+ # And if new events were generated since last time we were here
+ if activity:
+ activity = False
+ self.scan.status = "FINISHING"
+ # Trigger .finished() on every module and start over
+ log.info("Finishing scan")
+ finished_event = self.scan.make_event("FINISHED", "FINISHED", dummy=True)
+ for module in modules:
+ module.queue_event(finished_event)
+ else:
+ # Otherwise stop the scan if no new events were generated since last time
+ break
+ with self.incoming_event_queue.not_empty:
+ self.incoming_event_queue.not_empty.wait(timeout=0.1)
except KeyboardInterrupt:
self.scan.stop()
@@ -429,12 +424,20 @@ def loop_until_finished(self, status_frequency=10):
log.critical(traceback.format_exc())
finally:
- # Run .report() on every module and start over
+ # Run .report() on every module
for mod in self.scan.modules.values():
self.catch(mod.report, _force=True)
- def modules_status(self, _log=False, passes=None):
+ def log_status(self, frequency=10):
+ # print status every 10 seconds
+ timedelta_secs = timedelta(seconds=frequency)
+ now = datetime.now()
+ time_since_last_log = now - self.last_log_time
+ if time_since_last_log > timedelta_secs:
+ self.modules_status(_log=True, passes=1)
+ self.last_log_time = now
+ def modules_status(self, _log=False, passes=None):
# If scan looks to be finished, check an additional five times to ensure that it really is
# There is a tiny chance of a race condition, which this helps to avoid
if passes is None:
@@ -444,7 +447,6 @@ def modules_status(self, _log=False, passes=None):
finished = True
while passes > 0:
-
status = {"modules": {}, "scan": self.scan.status_detailed}
for num_tasks in status["scan"]["queued_tasks"].values():
@@ -473,7 +475,6 @@ def modules_status(self, _log=False, passes=None):
modules_errored = [m for m, s in status["modules"].items() if s["errored"]]
if _log:
-
modules_status = []
for m, s in status["modules"].items():
incoming = s["events"]["incoming"]
@@ -486,23 +487,64 @@ def modules_status(self, _log=False, passes=None):
modules_status = [s for s in modules_status if s[-2] or s[-1] > 0][:5]
if modules_status:
modules_status_str = ", ".join([f"{m}({i:,}:{t:,}:{o:,})" for m, i, o, t, _ in modules_status])
- self.scan.info(f"Modules: {modules_status_str}")
- event_type_summary = sorted(self.queued_event_types.items(), key=lambda x: x[-1], reverse=True)
- self.scan.info(f'Events: {", ".join([f"{k}: {v}" for k,v in event_type_summary])}')
+ running_modules_str = ", ".join([m[0] for m in modules_status])
+ self.scan.info(f"{self.scan.name}: Running modules: {running_modules_str}")
+ self.scan.verbose(
+ f"{self.scan.name}: Modules status (incoming:processing:outgoing) {modules_status_str}"
+ )
+ event_type_summary = sorted(
+ self.scan.stats.events_emitted_by_type.items(), key=lambda x: x[-1], reverse=True
+ )
+ if event_type_summary:
+ self.scan.info(
+ f'{self.scan.name}: Events produced so far: {", ".join([f"{k}: {v}" for k,v in event_type_summary])}'
+ )
+ else:
+ self.scan.info(f"{self.scan.name}: No events produced yet")
total_tasks = status["scan"]["queued_tasks"]["total"]
- dns_tasks = status["scan"]["queued_tasks"]["dns"]
event_tasks = status["scan"]["queued_tasks"]["event"]
- main_tasks = status["scan"]["queued_tasks"]["main"]
internal_tasks = status["scan"]["queued_tasks"]["internal"]
- self.scan.info(
- f"Tasks: {total_tasks:,} (Main: {main_tasks:,}, Event: {event_tasks:,}, DNS: {dns_tasks:,}, Internal: {internal_tasks:,})"
+ self.scan.verbose(
+ f"{self.scan.name}: Thread pool tasks: {total_tasks:,} (Event: {event_tasks:,}, Internal: {internal_tasks:,})"
)
if modules_errored:
self.scan.verbose(
- f'Modules errored: {len(modules_errored):,} ({", ".join([m for m in modules_errored])})'
+ f'{self.scan.name}: Modules errored: {len(modules_errored):,} ({", ".join([m for m in modules_errored])})'
+ )
+
+ queued_events_by_type = [(k, v) for k, v in self.incoming_event_queue.event_types.items() if v > 0]
+ if queued_events_by_type:
+ queued_events_by_type.sort(key=lambda x: x[-1], reverse=True)
+ queued_events_by_type_str = ", ".join(f"{m}: {t:,}" for m, t in queued_events_by_type)
+ self.scan.info(
+ f"{self.scan.name}: {self.incoming_event_queue.qsize():,} events in queue ({queued_events_by_type_str})"
)
+ else:
+ self.scan.info(f"{self.scan.name}: No events in queue")
+
+ if self.scan.log_level <= logging.DEBUG:
+ threadpool_names = [
+ "_internal_thread_pool",
+ "_event_thread_pool",
+ "_thread_pool",
+ ]
+ for threadpool_name in threadpool_names:
+ threadpool = getattr(self.scan, threadpool_name)
+ for thread_status in threadpool.threads_status:
+ self.scan.debug(f"scan.{threadpool_name}: {thread_status}")
+
+ # Uncomment these lines to enable debugging of event queues
+
+ # queued_events = self.incoming_event_queue.events
+ # if queued_events:
+ # queued_events_str = ", ".join(str(e) for e in queued_events)
+ # self.scan.verbose(f"Queued events: {queued_events_str}")
+ # queued_events_by_module = [(k, v) for k, v in self.incoming_event_queue.modules.items() if v > 0]
+ # queued_events_by_module.sort(key=lambda x: x[-1], reverse=True)
+ # queued_events_by_module_str = ", ".join(f"{m}: {t:,}" for m, t in queued_events_by_module)
+ # self.scan.verbose(f"{self.scan.name}: Queued events by module: {queued_events_by_module_str}")
status.update({"modules_errored": len(modules_errored)})
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index 26778bab2e..647c29a7d6 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -17,12 +17,12 @@
from .dispatcher import Dispatcher
from bbot.modules import module_loader
from bbot.core.event import make_event
-from bbot.core.logger import init_logging
from bbot.core.helpers.misc import sha1, rand_string
from bbot.core.helpers.helper import ConfigAwareHelper
+from bbot.core.logger import init_logging, get_log_level
from bbot.core.helpers.names_generator import random_name
-from bbot.core.helpers.threadpool import ThreadPoolWrapper
from bbot.core.configurator.environ import prepare_environment
+from bbot.core.helpers.threadpool import ThreadPoolWrapper, BBOTThreadPoolExecutor
from bbot.core.errors import BBOTError, ScanError, ScanCancelledError, ValidationError
log = logging.getLogger("bbot.scanner")
@@ -31,7 +31,6 @@
class Scanner:
-
_status_codes = {
"NOT_STARTED": 0,
"STARTING": 1,
@@ -85,14 +84,15 @@ def __init__(
self._status_code = 0
# Set up thread pools
- max_workers = max(1, self.config.get("max_threads", 100))
+ max_workers = max(1, self.config.get("max_threads", 25))
# Shared thread pool, for module use
- self._thread_pool = ThreadPoolWrapper(concurrent.futures.ThreadPoolExecutor(max_workers=max_workers))
+ self._thread_pool = BBOTThreadPoolExecutor(max_workers=max_workers)
# Event thread pool, for event emission
- self._event_thread_pool = ThreadPoolWrapper(concurrent.futures.ThreadPoolExecutor(max_workers=max_workers * 2))
- self._event_thread_pool_qsize = 1000
+ self._event_thread_pool = ThreadPoolWrapper(
+ BBOTThreadPoolExecutor(max_workers=max_workers * 2), qsize=max_workers
+ )
# Internal thread pool, for handle_event(), module setup, cleanup callbacks, etc.
- self._internal_thread_pool = ThreadPoolWrapper(concurrent.futures.ThreadPoolExecutor(max_workers=max_workers))
+ self._internal_thread_pool = ThreadPoolWrapper(BBOTThreadPoolExecutor(max_workers=max_workers))
self.process_pool = ThreadPoolWrapper(concurrent.futures.ProcessPoolExecutor())
self.helpers = ConfigAwareHelper(config=self.config, scan=self)
@@ -134,6 +134,13 @@ def __init__(
)
self.scope_report_distance = int(self.config.get("scope_report_distance", 1))
+ # custom HTTP headers warning
+ self.custom_http_headers = self.config.get("http_headers", {})
+ if self.custom_http_headers:
+ self.warning(
+ "You have enabled custom HTTP headers. These will be attached to all in-scope requests and all requests made by httpx."
+ )
+
self._prepped = False
self._cleanedup = False
@@ -162,7 +169,6 @@ def start_without_generator(self):
deque(self.start(), maxlen=0)
def start(self):
-
self.prep()
failed = True
@@ -212,13 +218,12 @@ def start(self):
except BBOTError as e:
self.critical(f"Error during scan: {e}")
- self.debug(traceback.format_exc())
+ self.trace()
except Exception:
self.critical(f"Unexpected error during scan:\n{traceback.format_exc()}")
finally:
-
self.cleanup()
self.shutdown_threadpools(wait=True)
@@ -376,16 +381,12 @@ def status(self, status):
@property
def status_detailed(self):
- main_tasks = self._thread_pool.num_tasks
- dns_tasks = self.helpers.dns._thread_pool.num_tasks
event_threadpool_tasks = self._event_thread_pool.num_tasks
internal_tasks = self._internal_thread_pool.num_tasks
process_tasks = self.process_pool.num_tasks
- total_tasks = main_tasks + dns_tasks + event_threadpool_tasks + internal_tasks + process_tasks
+ total_tasks = event_threadpool_tasks + internal_tasks + process_tasks
status = {
"queued_tasks": {
- "main": main_tasks,
- "dns": dns_tasks,
"internal": internal_tasks,
"process": process_tasks,
"event": event_threadpool_tasks,
@@ -459,23 +460,23 @@ def hugesuccess(self, *args, **kwargs):
def warning(self, *args, **kwargs):
log.warning(*args, extra={"scan_id": self.id}, **kwargs)
- self._log_traceback()
+ self.trace()
def hugewarning(self, *args, **kwargs):
log.hugewarning(*args, extra={"scan_id": self.id}, **kwargs)
- self._log_traceback()
+ self.trace()
def error(self, *args, **kwargs):
log.error(*args, extra={"scan_id": self.id}, **kwargs)
- self._log_traceback()
-
- def critical(self, *args, **kwargs):
- log.critical(*args, extra={"scan_id": self.id}, **kwargs)
+ self.trace()
- def _log_traceback(self):
+ def trace(self):
e_type, e_val, e_traceback = exc_info()
if e_type is not None:
- self.debug(traceback.format_exc())
+ log.trace(traceback.format_exc())
+
+ def critical(self, *args, **kwargs):
+ log.critical(*args, extra={"scan_id": self.id}, **kwargs)
def _internal_modules(self):
for modname in module_loader.preloaded(type="internal"):
@@ -483,9 +484,7 @@ def _internal_modules(self):
yield modname
def load_modules(self):
-
if not self._modules_loaded:
-
all_modules = list(set(self._scan_modules + self._output_modules + self._internal_modules))
if not all_modules:
self.warning(f"No modules to load")
@@ -553,8 +552,11 @@ def fail_setup(self, msg):
else:
raise ScanError(msg)
- def _load_modules(self, modules):
+ @property
+ def log_level(self):
+ return get_log_level()
+ def _load_modules(self, modules):
modules = [str(m) for m in modules]
loaded_modules = {}
failed = set()
diff --git a/bbot/scanner/stats.py b/bbot/scanner/stats.py
index cba296a7d7..f067e3b50d 100644
--- a/bbot/scanner/stats.py
+++ b/bbot/scanner/stats.py
@@ -3,19 +3,22 @@
log = logging.getLogger("bbot.scanner.stats")
+def _increment(d, k):
+ try:
+ d[k] += 1
+ except KeyError:
+ d[k] = 1
+
+
class ScanStats:
def __init__(self, scan):
self.scan = scan
self.module_stats = {}
+ self.events_emitted_by_type = {}
self.perf_stats = []
- def function_called(self, qualname, runtime):
- # uncomment the line below to track durations for function calls
- # this is helpful for debugging elusive performance issues
- # self.perf_stats.append((qualname, runtime))
- pass
-
def event_distributed(self, event):
+ _increment(self.events_emitted_by_type, event.type)
module_stat = self.get(event.module)
if module_stat is not None:
module_stat.increment_emitted(event)
@@ -56,7 +59,7 @@ def table(self):
consumed_str = f"{mstat.consumed_total:,}"
consumed = sorted(mstat.consumed.items(), key=lambda x: x[0])
if consumed:
- consumed_str = " (" + ", ".join(f"{c:,} {t}" for t, c in consumed) + ")"
+ consumed_str += " (" + ", ".join(f"{c:,} {t}" for t, c in consumed) + ")"
table_row.append(consumed_str)
table.append(table_row)
table.sort(key=lambda x: self.module_stats[x[0]].produced_total, reverse=True)
@@ -85,19 +88,13 @@ def __init__(self, module):
def increment_emitted(self, event):
self.emitted_total += 1
- self._increment(self.emitted, event.type)
+ _increment(self.emitted, event.type)
def increment_produced(self, event):
self.produced_total += 1
- self._increment(self.produced, event.type)
+ _increment(self.produced, event.type)
def increment_consumed(self, event):
- self.consumed_total += 1
- self._increment(self.consumed, event.type)
-
- @staticmethod
- def _increment(d, k):
- try:
- d[k] += 1
- except KeyError:
- d[k] = 1
+ if event.type not in ("FINISHED",):
+ self.consumed_total += 1
+ _increment(self.consumed, event.type)
diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
index 8d84bd6631..825e521f84 100644
--- a/bbot/scanner/target.py
+++ b/bbot/scanner/target.py
@@ -1,5 +1,6 @@
import logging
import ipaddress
+from contextlib import suppress
from bbot.core.errors import *
from bbot.core.event import make_event
@@ -45,24 +46,29 @@ def copy(self):
self_copy._events = dict(self._events)
return self_copy
- def _contains(self, other):
+ def get(self, host):
+ """
+ Get the matching target for a specified host. If not found, return None
+ """
try:
- other = make_event(other, dummy=True)
+ other = make_event(host, dummy=True)
except ValidationError:
- return False
- if other in self.events:
- return True
+ return
if other.host:
- if other.host in self._events:
- return True
+ with suppress(KeyError, StopIteration):
+ return next(iter(self._events[other.host]))
if self.scan.helpers.is_ip_type(other.host):
for n in self.scan.helpers.ip_network_parents(other.host, include_self=True):
- if n in self._events:
- return True
+ with suppress(KeyError, StopIteration):
+ return next(iter(self._events[n]))
elif not self.strict_scope:
for h in self.scan.helpers.domain_parents(other.host):
- if h in self._events:
- return True
+ with suppress(KeyError, StopIteration):
+ return next(iter(self._events[h]))
+
+ def _contains(self, other):
+ if self.get(other) is not None:
+ return True
return False
def __str__(self):
diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py
index 7c712a0048..6ad61b914b 100644
--- a/bbot/test/bbot_fixtures.py
+++ b/bbot/test/bbot_fixtures.py
@@ -41,10 +41,14 @@ def match_data(self, request: Request) -> bool:
log = logging.getLogger(f"bbot.test")
+# silence pytest_httpserver
+log = logging.getLogger("werkzeug")
+log.setLevel(logging.CRITICAL)
+
# silence stdout
root_logger = logging.getLogger()
for h in root_logger.handlers:
- h.addFilter(lambda x: x.levelno != 100)
+ h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE"))
tldextract.extract("www.evilcorp.com")
@@ -53,7 +57,6 @@ def match_data(self, request: Request) -> bool:
@pytest.fixture
def patch_commands():
-
import subprocess
sample_output = [
@@ -218,35 +221,47 @@ def helpers(scan):
@pytest.fixture
def events(scan):
class bbot_events:
- localhost = scan.make_event("127.0.0.1", dummy=True)
- ipv4 = scan.make_event("8.8.8.8", dummy=True)
- netv4 = scan.make_event("8.8.8.8/30", dummy=True)
- ipv6 = scan.make_event("2001:4860:4860::8888", dummy=True)
- netv6 = scan.make_event("2001:4860:4860::8888/126", dummy=True)
- domain = scan.make_event("publicAPIs.org", dummy=True)
- subdomain = scan.make_event("api.publicAPIs.org", dummy=True)
- email = scan.make_event("bob@evilcorp.co.uk", "EMAIL_ADDRESS", dummy=True)
- open_port = scan.make_event("api.publicAPIs.org:443", dummy=True)
- protocol = scan.make_event({"host": "api.publicAPIs.org:443", "protocol": "HTTP"}, "PROTOCOL", dummy=True)
- ipv4_open_port = scan.make_event("8.8.8.8:443", dummy=True)
- ipv6_open_port = scan.make_event("[2001:4860:4860::8888]:443", "OPEN_TCP_PORT", dummy=True)
- url_unverified = scan.make_event("https://api.publicAPIs.org:443/hellofriend", dummy=True)
- ipv4_url_unverified = scan.make_event("https://8.8.8.8:443/hellofriend", dummy=True)
- ipv6_url_unverified = scan.make_event("https://[2001:4860:4860::8888]:443/hellofriend", dummy=True)
- url = scan.make_event("https://api.publicAPIs.org:443/hellofriend", "URL", dummy=True)
- ipv4_url = scan.make_event("https://8.8.8.8:443/hellofriend", "URL", dummy=True)
- ipv6_url = scan.make_event("https://[2001:4860:4860::8888]:443/hellofriend", "URL", dummy=True)
- url_hint = scan.make_event("https://api.publicAPIs.org:443/hello.ash", "URL_HINT", dummy=True)
+ localhost = scan.make_event("127.0.0.1", source=scan.root_event)
+ ipv4 = scan.make_event("8.8.8.8", source=scan.root_event)
+ netv4 = scan.make_event("8.8.8.8/30", source=scan.root_event)
+ ipv6 = scan.make_event("2001:4860:4860::8888", source=scan.root_event)
+ netv6 = scan.make_event("2001:4860:4860::8888/126", source=scan.root_event)
+ domain = scan.make_event("publicAPIs.org", source=scan.root_event)
+ subdomain = scan.make_event("api.publicAPIs.org", source=scan.root_event)
+ email = scan.make_event("bob@evilcorp.co.uk", "EMAIL_ADDRESS", source=scan.root_event)
+ open_port = scan.make_event("api.publicAPIs.org:443", source=scan.root_event)
+ protocol = scan.make_event(
+ {"host": "api.publicAPIs.org", "port": 443, "protocol": "HTTP"}, "PROTOCOL", source=scan.root_event
+ )
+ ipv4_open_port = scan.make_event("8.8.8.8:443", source=scan.root_event)
+ ipv6_open_port = scan.make_event("[2001:4860:4860::8888]:443", "OPEN_TCP_PORT", source=scan.root_event)
+ url_unverified = scan.make_event("https://api.publicAPIs.org:443/hellofriend", source=scan.root_event)
+ ipv4_url_unverified = scan.make_event("https://8.8.8.8:443/hellofriend", source=scan.root_event)
+ ipv6_url_unverified = scan.make_event("https://[2001:4860:4860::8888]:443/hellofriend", source=scan.root_event)
+ url = scan.make_event(
+ "https://api.publicAPIs.org:443/hellofriend", "URL", tags=["status-200"], source=scan.root_event
+ )
+ ipv4_url = scan.make_event(
+ "https://8.8.8.8:443/hellofriend", "URL", tags=["status-200"], source=scan.root_event
+ )
+ ipv6_url = scan.make_event(
+ "https://[2001:4860:4860::8888]:443/hellofriend", "URL", tags=["status-200"], source=scan.root_event
+ )
+ url_hint = scan.make_event("https://api.publicAPIs.org:443/hello.ash", "URL_HINT", source=url)
vulnerability = scan.make_event(
- {"host": "evilcorp.com", "severity": "INFO", "description": "asdf"}, "VULNERABILITY", dummy=True
+ {"host": "evilcorp.com", "severity": "INFO", "description": "asdf"},
+ "VULNERABILITY",
+ source=scan.root_event,
)
- finding = scan.make_event({"host": "evilcorp.com", "description": "asdf"}, "FINDING", dummy=True)
- vhost = scan.make_event({"host": "evilcorp.com", "vhost": "www.evilcorp.com"}, "VHOST", dummy=True)
- http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", dummy=True)
+ finding = scan.make_event({"host": "evilcorp.com", "description": "asdf"}, "FINDING", source=scan.root_event)
+ vhost = scan.make_event({"host": "evilcorp.com", "vhost": "www.evilcorp.com"}, "VHOST", source=scan.root_event)
+ http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", source=scan.root_event)
storage_bucket = scan.make_event(
- {"name": "storage", "url": "https://storage.blob.core.windows.net"}, "STORAGE_BUCKET", dummy=True
+ {"name": "storage", "url": "https://storage.blob.core.windows.net"},
+ "STORAGE_BUCKET",
+ source=scan.root_event,
)
- emoji = scan.make_event("💩", "WHERE_IS_YOUR_GOD_NOW", dummy=True)
+ emoji = scan.make_event("💩", "WHERE_IS_YOUR_GOD_NOW", source=scan.root_event)
bbot_events.all = [ # noqa: F841
bbot_events.localhost,
@@ -284,7 +299,6 @@ class bbot_events:
@pytest.fixture
def agent(monkeypatch, websocketapp, bbot_config):
-
from bbot import agent
from bbot.modules.output.websocket import Websocket
diff --git a/bbot/test/helpers.py b/bbot/test/helpers.py
index f3e810a19e..3621b1893d 100644
--- a/bbot/test/helpers.py
+++ b/bbot/test/helpers.py
@@ -5,14 +5,21 @@
class MockHelper:
targets = ["blacklanternsecurity.com"]
+ blacklist = None
+ whitelist = None
config_overrides = {}
additional_modules = []
- def __init__(self, config, bbot_scanner, *args):
- self.name = self.__class__.__name__.lower()
+ def __init__(self, config, bbot_scanner, *args, **kwargs):
+ self.name = kwargs.get("module_name", self.__class__.__name__.lower())
self.config = OmegaConf.merge(config, OmegaConf.create(self.config_overrides))
self.scan = bbot_scanner(
- *self.targets, modules=[self.name] + self.additional_modules, name=f"{self.name}_test", config=self.config
+ *self.targets,
+ modules=[self.name] + self.additional_modules,
+ name=f"{self.name}_test",
+ config=self.config,
+ whitelist=self.whitelist,
+ blacklist=self.blacklist,
)
self.patch_scan(self.scan)
self.scan.prep()
@@ -56,12 +63,11 @@ def run(self):
class HttpxMockHelper(MockHelper):
-
targets = ["http://127.0.0.1:8888/"]
- def __init__(self, config, bbot_scanner, bbot_httpserver):
+ def __init__(self, config, bbot_scanner, bbot_httpserver, *args, **kwargs):
self.bbot_httpserver = bbot_httpserver
- super().__init__(config, bbot_scanner)
+ super().__init__(config, bbot_scanner, *args, **kwargs)
self.mock_args()
@abstractmethod
diff --git a/bbot/test/modules_test_classes.py b/bbot/test/modules_test_classes.py
index 4ecf7db31e..c5ad7d7fa6 100644
--- a/bbot/test/modules_test_classes.py
+++ b/bbot/test/modules_test_classes.py
@@ -1,18 +1,24 @@
import re
import json
+import logging
from .helpers import *
+log = logging.getLogger(f"bbot.test")
+
class Httpx(HttpxMockHelper):
def mock_args(self):
- respond_args = {"response_data": json.dumps({"foo": "bar"})}
- self.set_expect_requests(respond_args=respond_args)
+ request_args = dict(headers={"test": "header"})
+ respond_args = dict(response_data=json.dumps({"foo": "bar"}))
+ self.set_expect_requests(request_args, respond_args)
def check_events(self, events):
for e in events:
- if e.type == "HTTP_RESPONSE" and json.loads(e.data["body"])["foo"] == "bar":
- return True
+ if e.type == "HTTP_RESPONSE":
+ j = json.loads(e.data["body"])
+ if j.get("foo", "") == "bar":
+ return True
return False
@@ -39,6 +45,107 @@ def check_events(self, events):
return False
+class Excavate(HttpxMockHelper):
+ additional_modules = ["httpx"]
+ targets = ["http://127.0.0.1:8888/", "test.notreal"]
+
+ def mock_args(self):
+ response_data = """
+ ftp://ftp.test.notreal
+ \\nhttps://www1.test.notreal
+ \\x3dhttps://www2.test.notreal
+ %a2https://www3.test.notreal
+ \\uac20https://www4.test.notreal
+ \nwww5.test.notreal
+ \\x3dwww6.test.notreal
+ %a2www7.test.notreal
+ \\uac20www8.test.notreal
+
+ """
+ expect_args = {"method": "GET", "uri": "/"}
+ respond_args = {"response_data": response_data}
+ self.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
+
+ def check_events(self, events):
+ event_data = [e.data for e in events]
+ assert "https://www1.test.notreal/" in event_data
+ assert "https://www2.test.notreal/" in event_data
+ assert "https://www3.test.notreal/" in event_data
+ assert "https://www4.test.notreal/" in event_data
+ assert "www1.test.notreal" in event_data
+ assert "www2.test.notreal" in event_data
+ assert "www3.test.notreal" in event_data
+ assert "www4.test.notreal" in event_data
+ assert "www5.test.notreal" in event_data
+ assert "www6.test.notreal" in event_data
+ assert "www7.test.notreal" in event_data
+ assert "www8.test.notreal" in event_data
+ assert "http://www9.test.notreal/" in event_data
+
+ assert "nhttps://www1.test.notreal/" not in event_data
+ assert "x3dhttps://www2.test.notreal/" not in event_data
+ assert "a2https://www3.test.notreal/" not in event_data
+ assert "uac20https://www4.test.notreal/" not in event_data
+ assert "nwww5.test.notreal" not in event_data
+ assert "x3dwww6.test.notreal" not in event_data
+ assert "a2www7.test.notreal" not in event_data
+ assert "uac20www8.test.notreal" not in event_data
+
+ assert any(
+ e.type == "FINDING" and e.data.get("description", "") == "Non-HTTP URI: ftp://ftp.test.notreal"
+ for e in events
+ )
+ assert any(
+ e.type == "PROTOCOL"
+ and e.data.get("protocol", "") == "FTP"
+ and e.data.get("host", "") == "ftp.test.notreal"
+ for e in events
+ )
+ return True
+
+
+class Subdomain_Hijack(HttpxMockHelper):
+ additional_modules = ["httpx", "excavate"]
+
+ def mock_args(self):
+ fingerprints = self.module.fingerprints
+ assert fingerprints, "No subdomain hijacking fingerprints available"
+ fingerprint = next(iter(fingerprints))
+ rand_string = self.scan.helpers.rand_string(length=15, digits=False)
+ self.rand_subdomain = f"{rand_string}.{next(iter(fingerprint.domains))}"
+ respond_args = {"response_data": f''}
+ self.set_expect_requests(respond_args=respond_args)
+
+ def check_events(self, events):
+ for event in events:
+ if (
+ event.type == "FINDING"
+ and event.data["description"].startswith("Hijackable Subdomain")
+ and self.rand_subdomain in event.data["description"]
+ and event.data["host"] == self.rand_subdomain
+ ):
+ return True
+ return False
+
+
+class Fingerprintx(HttpxMockHelper):
+ targets = ["127.0.0.1:8888"]
+
+ def mock_args(self):
+ pass
+
+ def check_events(self, events):
+ for event in events:
+ if (
+ event.type == "PROTOCOL"
+ and event.host == self.scan.helpers.make_ip_type("127.0.0.1")
+ and event.port == 8888
+ and event.data["protocol"] == "HTTP"
+ ):
+ return True
+ return False
+
+
class Otx(RequestMockHelper):
def mock_args(self):
for t in self.targets:
@@ -88,8 +195,7 @@ def check_events(self, events):
class Badsecrets(HttpxMockHelper):
-
- targets = ["http://127.0.0.1:8888/", "http://127.0.0.1:8888/test.aspx"]
+ targets = ["http://127.0.0.1:8888/", "http://127.0.0.1:8888/test.aspx", "http://127.0.0.1:8888/cookie.aspx"]
sample_viewstate = """
|