Merge pull request #11 from laluka/jtof-fap-headers-ip

Add new IP/Host headers and improve http_headers_ip mode

Author: jtof-fap

bypass_url_parser.py

@@ -143,10 +143,12 @@ def __init__(self, config_dict=None, verbose=False, debug=False, debug_class=Fal
         self.base_curl = []
         self.user_agent_suffix = ""
         self.curl_items = []
+        self.curl_ips = []
         self.bypass_results = defaultdict(defaultdict)
         self.to_retry_items = []
         self.clean_output = ""
         self.pbar_queue = Queue(maxsize=1)
+        self.url_resolved_ip = ""
 
         # Init properties
         self.binary_name = Bypasser.DEFAULT_BINARY_NAME
@@ -169,6 +171,27 @@ def __init__(self, config_dict=None, verbose=False, debug=False, debug_class=Fal
 
     # *** Protected methods *** #
 
+    def _build_curl_ips(self, resolved_ip=None):
+        """ Build internal IP list from spoof_ips, const_internal_ip and the resolved target IP address.
+        :param str resolved_ip: Public (or private) IP address related to the url subdomain
+        """
+        self.curl_ips.clear()
+        # Adds user's custom IP addresses (-s, --spoof-ip)
+        if self.spoof_ips:
+            for spoof_ip in self.spoof_ips:
+                if spoof_ip not in self.curl_ips:
+                    self.curl_ips.append(spoof_ip)
+
+        # Append mode (by default and in any case if self.spoof_ips is empty)
+        if not self.spoof_ip_replace:
+            # Internal IP addresses
+            for const_internal_ip in self.const_internal_ips:
+                if const_internal_ip not in self.curl_ips:
+                    self.curl_ips.append(const_internal_ip)
+            # Public (or private) IP address
+            if resolved_ip and resolved_ip not in self.curl_ips:
+                self.curl_ips.append(resolved_ip)
+
     def _init_debug_level(self, level):
         if level:
             self.verbose = True
@@ -199,12 +222,18 @@ def _generate_curls(self, url_obj: ParseResult):
         # Reset curl list
         self.curl_items.clear()
 
-        # Get the public IP of this URL
-        url_public_ip = socket.gethostbyname(str(url_obj.hostname))
+        # Resolves public (or private) IP of target URL
+        try:
+            self.url_resolved_ip = socket.gethostbyname(str(url_obj.hostname))
+        except (socket.error, socket.gaierror):
+            error_msg = f"Unable to resolve the subdomain '{url_obj.hostname}'. Please check the url or your " \
+                        f"host's DNS resolvers"
+            self.logger.error(error_msg)
+            raise ValueError(error_msg)
 
         # Original request
         cmd = [*self.base_curl, target_url]
-        item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="original_request", target_ip=url_public_ip,
+        item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="original_request", target_ip=self.url_resolved_ip,
                         debug=self.debug, ext_logger=self.logger)
         if item not in self.curl_items:
             self.curl_items.append(item)
@@ -213,8 +242,8 @@ def _generate_curls(self, url_obj: ParseResult):
         if any(mode in ["all", "http_methods"] for mode in self.current_bypass_modes):
             for const_http_method in self.const_http_methods:
                 cmd = [*self.base_curl, "-X", const_http_method, target_url]
-                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="http_methods", target_ip=url_public_ip,
-                                debug=self.debug, ext_logger=self.logger)
+                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="http_methods",
+                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                 if item not in self.curl_items:
                     self.curl_items.append(item)
 
@@ -223,8 +252,7 @@ def _generate_curls(self, url_obj: ParseResult):
             for http_version in CurlItem.CURL_HTTP_VERSIONS[:-1]:
                 cmd = [*self.get_curl_base(forced_http_version=http_version), target_url]
                 item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="http_versions",
-                                target_ip=url_public_ip,
-                                debug=self.debug, ext_logger=self.logger)
+                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                 if item not in self.curl_items:
                     self.curl_items.append(item)
 
@@ -234,29 +262,31 @@ def _generate_curls(self, url_obj: ParseResult):
                 for const_http_method in self.const_http_methods:
                     cmd = [*self.base_curl, "-H", f"{const_header_method}: {const_http_method}", target_url]
                     item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="http_headers_method",
-                                    target_ip=url_public_ip, debug=self.debug, ext_logger=self.logger)
+                                    target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                     if item not in self.curl_items:
                         self.curl_items.append(item)
 
         # [http_headers_ip] - Custom host injection headers
         if any(mode in ["all", "http_headers_ip"] for mode in self.current_bypass_modes):
+            self._build_curl_ips(resolved_ip=self.url_resolved_ip)
             commands = set()
             for const_header_host in self.const_header_hosts:
-                if self.spoof_ips:
-                    # Custom IP addresses
-                    for spoof_ip in self.spoof_ips:
-                        commands.add(tuple([*self.base_curl, "-H", f"{const_header_host}: {spoof_ip}", target_url]))
-                if not self.spoof_ip_replace:  # False in any case if self.spoof_ips is empty
-                    # Internal IP addresses
-                    for const_internal_ip in self.const_internal_ips:
-                        commands.add(
-                            tuple([*self.base_curl, "-H", f"{const_header_host}: {const_internal_ip}", target_url]))
-                    # Public IP address related to the url subdomain
-                    commands.add(tuple([*self.base_curl, "-H", f"{const_header_host}: {url_public_ip}", target_url]))
+                # Header which takes 1 as value
+                if const_header_host == "X-AppEngine-Trusted-IP-Request":
+                    commands.add(tuple([*self.base_curl, "-H", f"{const_header_host}: 1", target_url]))
+                    continue
+                # Specific rule for header 'Forwarded: for='
+                for ip in self.curl_ips:
+                    if const_header_host == "Forwarded":
+                        commands.add(tuple([*self.base_curl, "-H", f"{const_header_host}: by={ip}", target_url]))
+                        commands.add(tuple([*self.base_curl, "-H", f"{const_header_host}: for={ip}", target_url]))
+                        commands.add(tuple([*self.base_curl, "-H", f"{const_header_host}: host={ip}", target_url]))
+                    else:
+                        commands.add(tuple([*self.base_curl, "-H", f"{const_header_host}: {ip}", target_url]))
             # Add items
             for command in commands:
                 item = CurlItem(url_obj, self.base_curl, list(command), bypass_mode="http_headers_ip",
-                                target_ip=url_public_ip, debug=self.debug, ext_logger=self.logger)
+                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                 if item not in self.curl_items:
                     self.curl_items.append(item)
 
@@ -275,7 +305,7 @@ def _generate_curls(self, url_obj: ParseResult):
             # Add items
             for command in commands:
                 item = CurlItem(url_obj, self.base_curl, list(command), bypass_mode="http_headers_scheme",
-                                target_ip=url_public_ip, debug=self.debug, ext_logger=self.logger)
+                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                 if item not in self.curl_items:
                     self.curl_items.append(item)
 
@@ -294,7 +324,7 @@ def _generate_curls(self, url_obj: ParseResult):
                 # Add items
                 for command in commands:
                     item = CurlItem(url_obj, self.base_curl, list(command), bypass_mode="http_headers_port",
-                                    target_ip=url_public_ip, debug=self.debug, ext_logger=self.logger)
+                                    target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                     if item not in self.curl_items:
                         self.curl_items.append(item)
 
@@ -314,7 +344,7 @@ def _generate_curls(self, url_obj: ParseResult):
             # Add items
             for command in commands:
                 item = CurlItem(url_obj, self.base_curl, list(command), bypass_mode="mid_paths",
-                                target_ip=url_public_ip, debug=self.debug, ext_logger=self.logger)
+                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                 if item not in self.curl_items:
                     self.curl_items.append(item)
 
@@ -337,7 +367,7 @@ def _generate_curls(self, url_obj: ParseResult):
                 # Add items
                 for command in commands:
                     item = CurlItem(url_obj, self.base_curl, list(command), bypass_mode="end_paths",
-                                    target_ip=url_public_ip, debug=self.debug, ext_logger=self.logger)
+                                    target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                     if item not in self.curl_items:
                         self.curl_items.append(item)
 
@@ -349,8 +379,8 @@ def _generate_curls(self, url_obj: ParseResult):
                 char_case = base_path[abc_index]
                 char_case = char_case.upper() if char_case.islower() else char_case.lower()
                 cmd = [*self.base_curl, f"{base_url}{base_path[:abc_index]}{char_case}{base_path[abc_index + 1:]}"]
-                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="case_substitution", target_ip=url_public_ip,
-                                debug=self.debug, ext_logger=self.logger)
+                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="case_substitution",
+                                target_ip=self.url_resolved_ip, debug=self.debug, ext_logger=self.logger)
                 if item not in self.curl_items:
                     self.curl_items.append(item)
 
@@ -359,7 +389,7 @@ def _generate_curls(self, url_obj: ParseResult):
                 char_urlencoded = format(ord(base_path[abc_index]), "02x")
                 cmd = [*self.base_curl,
                        f"{base_url}{base_path[:abc_index]}%{char_urlencoded}{base_path[abc_index + 1:]}"]
-                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="char_encode", target_ip=url_public_ip,
+                item = CurlItem(url_obj, self.base_curl, cmd, bypass_mode="char_encode", target_ip=self.url_resolved_ip,
                                 debug=self.debug, ext_logger=self.logger)
                 if item not in self.curl_items:
                     self.curl_items.append(item)

payloads/const_header_hosts.lst

@@ -1,53 +1,139 @@
-Access-Control-Allow-Origin
-Base-Url
-CF-Connecting_IP
+Ali-CDN-Real-IP
 CF-Connecting-IP
+Cdn-Real-IP
+Cdn-Src-IP
+Cf-Pseudo-IPv4
 Client-IP
+Cluster-Client-IP
+Cluster-IP
+Connection
+Contact
 Destination
-Forwarded-For-IP
-Forwarded-For
+Fastly-Client-IP
 Forwarded
+Forwarded-For
+Forwarded-For-IP
+Forwarded-Host
+Forwarder
+Forwarder-For
+Forwarder-Host
+Forwarding
+Forwarding-For
+Forwarding-Host
+From
+HTTP-Host
 Host
-Http-Url
+Host-IP
+IP
+Incap-Client-IP
 Origin
+Origin-Host
+Origin-IP
+Originating-IP
+Override-IP
 Profile
-Proxy-Host
-Proxy-Url
 Proxy
-Real-Ip
+Proxy-Client-IP
+Proxy-Host
+Proxy-IP
+Real-Client-IP
+Real-IP
 Redirect
-Referer
-Referrer
+Remote-Addr
+Remote-Host
+Remote-IP
 Request-Uri
+Server
+Server-IP
+Server-Name
+Source-IP
+True-Client
 True-Client-IP
-Uri
-Url
+Via
+WL-Proxy-Client-IP
+X-AppEngine-Trusted-IP-Request
+X-Appengine-User-IP
 X-Arbitrary
+X-Backend-Host
+X-Backend-Server
+X-BlueCoat-Via
+X-C-IP
+X-Cache-Info
+X-Client-Host
 X-Client-IP
+X-Cluster-Client-IP
+X-Cluster-IP
 X-Custom-IP-Authorization
+X-Dev-Host
+X-Ebay-Client-IP
+X-Fake-IP
+X-Fb-Host
+X-Fb-User-Remote-Addr
+X-Forward
 X-Forward-For
+X-Forwarded
 X-Forwarded-By
-X-Forwarded-For-Original
 X-Forwarded-For
+X-Forwarded-For-IP
+X-Forwarded-For-Original
+X-Forwarded-From
 X-Forwarded-Host
-X-Forwarded-Proto
 X-Forwarded-Server
-X-Forwarded
+X-Forwarder
 X-Forwarder-For
+X-Forwarder-Host
+X-Forwarding
+X-Forwarding-For
+X-Forwarding-Host
+X-Forwared-Host
+X-From
+X-From-IP
+X-Gateway-Host
+X-HTTP-Host-Override
 X-Host
-X-Http-Destinationurl
-X-HTTP-DestinationURL
+X-Host-IP
+X-Host-Override
 X-Http-Host-Override
+X-Http-Method-Override
+X-IP
+X-IP-Addr
+X-IP-Address
+X-IP-Trail
+X-MS-ADFS-Proxy-Client-IP
+X-MS-Forwarded-Client-IP
+X-Nokia-ipaddress
+X-Origin
+X-Origin-Host
+X-Origin-IP
+X-Original-For
+X-Original-Forwarded-For
+X-Original-Host
+X-Original-Hostname
+X-Original-IP
 X-Original-Remote-Addr
-X-Original-URL
 X-Originally-Forwarded-For
-X-Originating-
+X-Originating
+X-Originating-Host
 X-Originating-IP
-X-Proxy-Url
-X-ProxyUser-Ip
-X-Real-Ip
-X-Referrer
+X-Proxy
+X-Proxy-IP
+X-ProxyMesh-IP
+X-ProxyUser-IP
+X-Real-Client-IP
+X-Real-IP
 X-Remote-Addr
+X-Remote-Host
 X-Remote-IP
-X-Rewrite-URL
-X-WAP-Profile
+X-Served-By
+X-Server
+X-Server-IP
+X-Server-Name
+X-Sp-Edge-Host
+X-Sp-Forwarded-IP
+X-True-Client
+X-True-Client-IP
+X-True-IP
+X-WAP-Network-Client-IP
+X-Wap-Profile
+X-YWBCLO-UIP
+Z-Forwarded-For