diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt index c71f09fc94..68ae7e13fe 100644 --- a/data/txt/sha256sums.txt +++ b/data/txt/sha256sums.txt @@ -162,21 +162,21 @@ df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264 extra/shutils/ 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 extra/vulnserver/__init__.py 617cec1b731e0baacafa6f58c2f56a85b6128d1416627cc1b2f61519c8539a2e extra/vulnserver/vulnserver.py a2bf70d7f87c3a4e0675c0bad54119a4e04efa6ea2730a8338d5aebcd995630e lib/controller/action.py -f4fb3839e5accd1b58b34226e4b26f5079d9696e24d335d37d870cd5e62d1e80 lib/controller/checks.py +736715a73941a06e5d3d349dd01a1f1b171f54eb4c374c6752b2cc44b0977ffe lib/controller/checks.py 666935b658074dc9c42153622b75d4ec7bfe56fbe0742de827a5d30a1a0f9d96 lib/controller/controller.py d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f lib/controller/handler.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/controller/__init__.py 9c5764c92ce536d1f0f96200359ee5ef1f37f9128769bf990cb77f1d1f8e17b1 lib/core/agent.py c51c33501cc905586a9aaac93b06f2ac6f71628d032a7dc39fd0ef05d7ee3856 lib/core/bigarray.py -d143df718fbaacb617b6046c73cf4e47932e1a25928a4e1ecb87ea77a3b154ed lib/core/common.py +751c3bf178e91e60b25e3b01ce7636029804dd78f64e9ee0418bdb126889a7bc lib/core/common.py 8f1272487e1adfcc8c755a2f56f0c6d21eac5e685a73a9a159482f9dc9142bc5 lib/core/compat.py -a683d0ad9ba543587382c4903d28db610ae20394fcf9045a68b2ab54a39381ae lib/core/convert.py +5301ba2204404d086e9a67271cde00fc10214c63b018a95fc5aa90ff9e0b2ad9 lib/core/convert.py c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6 lib/core/data.py d9ec034a6d51ab4ddde0b6aa7ed306f9e0b1336557f77d7939ba547600f9b3ae lib/core/datatype.py f8de57606325456928e46ae2896f5f8bbec9ad18b1c644b492a566fa992216f6 lib/core/decorators.py 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d lib/core/defaults.py 8e4f4b5ea37a49d445bb0df83bf04b34f61035ec33fd8acf598ebcf371cb19a7 lib/core/dicts.py -854073f899b876ab13b36e93e174b9cfe51408f7343040197a80afd9fc9c65ee lib/core/dump.py +10d8bb671a64cc787fc2fbf2c641560b7797fccd62c4792e55dffe5efab9f544 lib/core/dump.py 6dd47f52082e98dc0cda6969b277b7d81c6f7c68dac4688821f873a1c65c6edf lib/core/enums.py 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4 lib/core/exception.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/core/__init__.py @@ -189,7 +189,7 @@ f8de57606325456928e46ae2896f5f8bbec9ad18b1c644b492a566fa992216f6 lib/core/decor 9bf174058f15d14e24e94f9aaf42df045119d3617c6c54bd2f3af79b462f331d lib/core/replication.py 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6 lib/core/revision.py 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py -a2fb281b59c4526613f22fc0e994b68db91c1263db415aa86002ec4e20773639 lib/core/settings.py +35c24cf138fdd68add3c8f6274d6ff735b5209c84eec635ba316f986b67325ef lib/core/settings.py c7804223319e18eb0b8e2cbf0a8b6896d1cefb7b0b1a2e9f1cf826a8a3b56750 lib/core/shell.py a2e98a94b231432736d6b304fc75525c8b5fdb4768c418387c5b4c1a610dad64 lib/core/subprocessng.py 19f1e3c5e3ba703d28d510cd7a9ab8284d5fbe9df5ce7e77c86e5931571364b7 lib/core/target.py @@ -212,9 +212,10 @@ c2f34e27578742e729c2fa9c1d4f0a0d8f8f7f4cf0fc14c62ec817a260c71dec lib/parse/site 369484a2999d29f49bf839a329d1686ed94f6ea27c695e027fe08c8da51f30a3 lib/request/basic.py bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e lib/request/chunkedhandler.py 9c0dccc1cee66d38478aaf75a7c513d0d136d50a90b15fed146faa1653899fe1 lib/request/comparison.py -729e07a2ca6b1d83563e9c6dc5a884d1b664c1764be06776ea93bde305164f0c lib/request/connect.py +c96deaa69743d2cf4ae48f2ae0036f7e11b838f97a0e8c7f1205c61e9dd36bc1 lib/request/connect.py 8e06682280fce062eef6174351bfebcb6040e19976acff9dc7b3699779783498 lib/request/direct.py a6b37b436838caeb197fea858d0a39fadbff4736256e741b5fcec1f28fcf1ce0 lib/request/dns.py +21e8e2d44788b124f741b76a483ce9528ca53ff6da6691808ee679fe91128050 lib/request/http2.py 92c81cc31ff4a396723242058fb2152c9e9745f8412d01ea74480b048a53af6c lib/request/httpshandler.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/request/__init__.py 7a0ac2522213e756348fd871a7af74cc963bdc82f9d7ade57be5de42b5bf7cab lib/request/inject.py @@ -256,8 +257,8 @@ c61816c9dba9f6cc2223aed1a923f95130979e5f0a88ec254ee667d955ed2734 lib/techniques aeefb42ea0c68f72744bc1bfd7194ec1bc06480d8a7e23f4b8d3d23fbba2b014 lib/utils/api.py 442555ab85277aff7c9e0cf465ea5b0d28395c326f68363449b2d3941f4b6de2 lib/utils/brute.py da5bcbcda3f667582adf5db8c1b5d511b469ac61b55d387cec66de35720ed718 lib/utils/crawler.py -a94958be0ec3e9d28d8171813a6a90655a9ad7e6aa33c661e8d8ebbfcf208dbb lib/utils/deps.py -b0d8ae8513c1f5ffcaa4bf0398790f26bc2180a6acf07bf5b2c86555bf9113f6 lib/utils/dialect.py +51deedec3d3e869b067824caa51406d2ef396c188f82013ca60777006a821e27 lib/utils/deps.py +bd9267d94390ba87d6c5a35c90f2406d6a4135a7c8ea01db76dd9e6519eee2ed lib/utils/dialect.py 51cfab194cd5b6b24d62706fb79db86c852b9e593f4c55c15b35f175e70c9d75 lib/utils/getch.py 3c4ad819589fe4fca303706dc87969273a07a04dee85e23f064b39caf1fb80e9 lib/utils/gui.py 972c5db9c9e30ac0f91c0f8d4df4531d0304e151dac99f1399c37c952ba9f935 lib/utils/har.py @@ -602,7 +603,7 @@ c17544be5e945dc8c4fbb5c3b922da8eceec30b0fb239c32fb5f40e1660a197f tests/test_dat 8a1edb6dbc000e412ba5cc598e024b669fc76ec0a8fc32136808e6325a018f70 tests/test_dbms_enum.py 3804eb2d730220360f9dc07d5994eb64e9f65acf3b0d8648df8df2a2177ba8fd tests/test_decodepage.py 180e5fd3f75fadf7ac1135f99797314e2cf1f8ae6dced02edfb18ccba43c0148 tests/test_deps.py -b01343eb8aa42ea5c2c483ec028a24f6451aa6f668fdc0c289d5ff9554c277d7 tests/test_dialectdbms.py +fa85881aa8d082a65aeacb2b03fcb5d2abb1daa9a02ee24ff048d54fbc904b90 tests/test_dialectdbms.py e40a49cfa73c45b3c3c6d1d1d00738861e270cb7a07b28f5a5356f9c7c800cf2 tests/test_dialect.py 993a2d4d87c4fbaf261663b069629acc95ee4405aa0c42cf5a8f39649fdb0fff tests/test_dicts.py 7f9180a53dbf0bb3e52801fdbfffd31f365a0bff77bf90e58d2ef63a0c23026f tests/test_dns_engine.py diff --git a/lib/controller/checks.py b/lib/controller/checks.py index 6a7043cc92..a7200e3e32 100644 --- a/lib/controller/checks.py +++ b/lib/controller/checks.py @@ -93,6 +93,9 @@ from lib.core.settings import MAX_STABILITY_DELAY from lib.core.settings import NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH from lib.core.settings import NOSQL_ERROR_REGEX +from lib.core.settings import NULL_CONNECTION_LENGTH_TOLERANCE_HIGH +from lib.core.settings import NULL_CONNECTION_LENGTH_TOLERANCE_LOW +from lib.core.settings import NULL_CONNECTION_SKIP_READ_MIN_LENGTH from lib.core.settings import PRECONNECT_INCOMPATIBLE_SERVERS from lib.core.settings import SINGLE_QUOTE_MARKER from lib.core.settings import SLEEP_TIME_MARKER @@ -1286,6 +1289,27 @@ def checkDynamicContent(firstPage, secondPage): count += 1 if count > conf.retries: + # Last resort before the (lossy) '--text-only' fallback: if the page is byte-unstable + # but STRUCTURALLY stable - an identical, non-empty tag/class/id skeleton across + # requests - base the comparison on that value-free structure instead. Dynamic text + # (e.g. per-render result rows) then no longer masks an injection whose signal is + # structural (the HTML counterpart of the structure-aware JSON comparison). Content + # with no usable structure (empty skeleton, e.g. random/binary bodies) falls through + # to '--text-only' as before. + skeleton = extractStructuralTokens(firstPage) + if skeleton and skeleton == extractStructuralTokens(secondPage): + kb.pageStructurallyStable = True + + if kb.nullConnection: + debugMsg = "turning off NULL connection support because of structural page comparison" + logger.debug(debugMsg) + kb.nullConnection = None + + infoMsg = "target URL content is not byte-stable but structurally stable; sqlmap " + infoMsg += "will base the page comparison on the page structure" + logger.info(infoMsg) + return + warnMsg = "target URL content appears to be too dynamic. " warnMsg += "Switching to '--text-only' " logger.warning(warnMsg) @@ -1391,26 +1415,7 @@ def checkStability(): raise SqlmapNoneDataException(errMsg) else: - # Before engaging the (lossy) dynamic-content removal / '--text-only' escalation, check - # whether the page is structurally stable (identical tag/class/id skeleton across the two - # requests) despite differing text. If so, base the comparison on that value-free structure - # so that dynamic content (e.g. per-render result rows) does not mask an injection. This is - # the HTML counterpart of the structure-aware JSON comparison - if firstPage and secondPage and extractStructuralTokens(firstPage) == extractStructuralTokens(secondPage): - kb.pageStructurallyStable = True - - if kb.nullConnection: - debugMsg = "turning off NULL connection " - debugMsg += "support because of structural page comparison" - logger.debug(debugMsg) - - kb.nullConnection = None - - infoMsg = "target URL content is not byte-stable but structurally stable; sqlmap " - infoMsg += "will base the page comparison on the page structure" - logger.info(infoMsg) - else: - checkDynamicContent(firstPage, secondPage) + checkDynamicContent(firstPage, secondPage) return kb.pageStable @@ -1532,30 +1537,79 @@ def checkNullConnection(): pushValue(kb.pageCompress) kb.pageCompress = False + # A method is accepted only if the length it reports tracks the real GET response. The + # original page length (len(kb.originalPage)) is the reference; a method whose length is + # grossly off (e.g. HEAD returning 'Content-Length: 0', HEAD served from a different code + # path, or sneaked-in compression) would otherwise make every page look identical and + # silently break detection. The band is coarse on purpose (byte-vs-character size and + # moderate page dynamism are expected); a false reject just forgoes the optimization + def _plausibleLength(length): + reference = len(kb.originalPage or "") + if not reference: + return True + return NULL_CONNECTION_LENGTH_TOLERANCE_LOW * reference <= length <= NULL_CONNECTION_LENGTH_TOLERANCE_HIGH * reference + try: page, headers, _ = Request.getPage(method=HTTPMETHOD.HEAD, raise404=False) if not page and HTTP_HEADER.CONTENT_LENGTH in (headers or {}): - kb.nullConnection = NULLCONNECTION.HEAD + try: + length = int(headers[HTTP_HEADER.CONTENT_LENGTH].split(',')[0]) + except ValueError: + length = None - infoMsg = "NULL connection is supported with HEAD method ('Content-Length')" - logger.info(infoMsg) - else: + if length is not None and _plausibleLength(length): + kb.nullConnection = NULLCONNECTION.HEAD + + infoMsg = "NULL connection is supported with HEAD method ('Content-Length')" + logger.info(infoMsg) + elif length is not None: + debugMsg = "HEAD method reports an implausible 'Content-Length' (%d B vs ~%d B for the original page); skipping it" % (length, len(kb.originalPage or "")) + logger.debug(debugMsg) + + if kb.nullConnection is None: page, headers, _ = Request.getPage(auxHeaders={HTTP_HEADER.RANGE: "bytes=-1"}) if page and len(page) == 1 and HTTP_HEADER.CONTENT_RANGE in (headers or {}): - kb.nullConnection = NULLCONNECTION.RANGE + try: + length = int(headers[HTTP_HEADER.CONTENT_RANGE][headers[HTTP_HEADER.CONTENT_RANGE].find('/') + 1:]) + except ValueError: + length = None - infoMsg = "NULL connection is supported with GET method ('Range')" - logger.info(infoMsg) - else: - _, headers, _ = Request.getPage(skipRead=True) + if length is not None and _plausibleLength(length): + kb.nullConnection = NULLCONNECTION.RANGE - if HTTP_HEADER.CONTENT_LENGTH in (headers or {}): + infoMsg = "NULL connection is supported with GET method ('Range')" + logger.info(infoMsg) + elif length is not None: + debugMsg = "'Range' method reports an implausible total length (%d B vs ~%d B for the original page); skipping it" % (length, len(kb.originalPage or "")) + logger.debug(debugMsg) + + if kb.nullConnection is None: + _, headers, _ = Request.getPage(skipRead=True) + + if HTTP_HEADER.CONTENT_LENGTH in (headers or {}): + try: + length = int(headers[HTTP_HEADER.CONTENT_LENGTH].split(',')[0]) + except ValueError: + length = len(kb.originalPage or "") + + if not _plausibleLength(length): + debugMsg = "'skip-read' method reports an implausible 'Content-Length' (%d B vs ~%d B for the original page); skipping it" % (length, len(kb.originalPage or "")) + logger.debug(debugMsg) + # Unlike HEAD/Range, 'skip-read' leaves the body unread and must close the + # connection (an unread body cannot be reused), paying a fresh TCP/TLS handshake + # per request. That only outweighs the avoided body transfer for large responses; + # for small ones it is a net slowdown, so it is gated by the response size here + elif length >= NULL_CONNECTION_SKIP_READ_MIN_LENGTH: kb.nullConnection = NULLCONNECTION.SKIP_READ infoMsg = "NULL connection is supported with 'skip-read' method" logger.info(infoMsg) + else: + debugMsg = "'skip-read' NULL connection method is available but skipped because the " + debugMsg += "response (%d B) is too small for it to outweigh the per-request reconnect cost" % length + logger.debug(debugMsg) except SqlmapConnectionException: pass diff --git a/lib/core/common.py b/lib/core/common.py index 937064d705..ec7db6ff96 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -3310,7 +3310,16 @@ def isNumPosStrValue(value): return retVal -@cachedmethod +# DBMS_DICT is static, so the alias -> enum resolution is precomputed once into a +# lookup table (replacing a per-call @cachedmethod + linear scan). aliasToDbmsEnum() +# is a hot path (Backend.getIdentifiedDbms() calls it constantly). Building via +# setdefault in dict order preserves the original first-match-wins semantics. +_DBMS_ALIAS_MAP = {} +for _dbmsKey, _dbmsItem in DBMS_DICT.items(): + for _dbmsAlias in _dbmsItem[0]: + _DBMS_ALIAS_MAP.setdefault(_dbmsAlias, _dbmsKey) + _DBMS_ALIAS_MAP.setdefault(_dbmsKey.lower(), _dbmsKey) + def aliasToDbmsEnum(dbms): """ Returns major DBMS name from a given alias @@ -3319,15 +3328,7 @@ def aliasToDbmsEnum(dbms): 'Microsoft SQL Server' """ - retVal = None - - if dbms: - for key, item in DBMS_DICT.items(): - if dbms.lower() in item[0] or dbms.lower() == key.lower(): - retVal = key - break - - return retVal + return _DBMS_ALIAS_MAP.get(dbms.lower()) if dbms else None def findDynamicContent(firstPage, secondPage, merge=False): """ @@ -4414,7 +4415,11 @@ def safeSQLIdentificatorNaming(name, isTable=False): if isinstance(name, six.string_types): retVal = getUnicode(name) - _ = isTable and Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) + # Resolve the identified DBMS once; it is invariant within this call and + # Backend.getIdentifiedDbms() (which scans DBMS_DICT) was otherwise + # re-evaluated several times below. + dbms = Backend.getIdentifiedDbms() + _ = isTable and dbms in (DBMS.MSSQL, DBMS.SYBASE) if _: retVal = re.sub(r"(?i)\A\[?%s\]?\." % DEFAULT_MSSQL_SCHEMA, "%s." % DEFAULT_MSSQL_SCHEMA, retVal) @@ -4424,13 +4429,13 @@ def safeSQLIdentificatorNaming(name, isTable=False): if not conf.noEscape: retVal = unsafeSQLIdentificatorNaming(retVal) - if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS, DBMS.CUBRID, DBMS.SQLITE, DBMS.SPANNER, DBMS.CLICKHOUSE): # Note: in SQLite double-quotes are treated as string if column/identifier is non-existent (e.g. SELECT "foobar" FROM users) + if dbms in (DBMS.MYSQL, DBMS.ACCESS, DBMS.CUBRID, DBMS.SQLITE, DBMS.SPANNER, DBMS.CLICKHOUSE): # Note: in SQLite double-quotes are treated as string if column/identifier is non-existent (e.g. SELECT "foobar" FROM users) retVal = "`%s`" % retVal - elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE, DBMS.RAIMA, DBMS.VIRTUOSO, DBMS.SNOWFLAKE, DBMS.FIREBIRD, DBMS.DERBY, DBMS.MAXDB): + elif dbms in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE, DBMS.RAIMA, DBMS.VIRTUOSO, DBMS.SNOWFLAKE, DBMS.FIREBIRD, DBMS.DERBY, DBMS.MAXDB): retVal = "\"%s\"" % retVal - elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.ALTIBASE, DBMS.MIMERSQL): + elif dbms in (DBMS.ORACLE, DBMS.ALTIBASE, DBMS.MIMERSQL): retVal = "\"%s\"" % retVal.upper() - elif Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE): + elif dbms in (DBMS.MSSQL, DBMS.SYBASE): if isTable: parts = retVal.split('.', 1) for i in xrange(len(parts)): @@ -4463,16 +4468,21 @@ def unsafeSQLIdentificatorNaming(name): retVal = name if isinstance(name, six.string_types): - if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS, DBMS.CUBRID, DBMS.SQLITE, DBMS.SPANNER, DBMS.CLICKHOUSE): + # Resolve the identified DBMS once; it is invariant within this call, and + # Backend.getIdentifiedDbms() is not cheap (it scans DBMS_DICT). Previously + # it was re-evaluated up to five times per call. + dbms = Backend.getIdentifiedDbms() + + if dbms in (DBMS.MYSQL, DBMS.ACCESS, DBMS.CUBRID, DBMS.SQLITE, DBMS.SPANNER, DBMS.CLICKHOUSE): retVal = name.replace("`", "") - elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE, DBMS.RAIMA, DBMS.VIRTUOSO, DBMS.SNOWFLAKE, DBMS.FIREBIRD, DBMS.DERBY, DBMS.MAXDB): + elif dbms in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE, DBMS.RAIMA, DBMS.VIRTUOSO, DBMS.SNOWFLAKE, DBMS.FIREBIRD, DBMS.DERBY, DBMS.MAXDB): retVal = name.replace("\"", "") - elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.ALTIBASE, DBMS.MIMERSQL): + elif dbms in (DBMS.ORACLE, DBMS.ALTIBASE, DBMS.MIMERSQL): retVal = name.replace("\"", "").upper() - elif Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE): + elif dbms in (DBMS.MSSQL, DBMS.SYBASE): retVal = name.replace("[", "").replace("]", "") - if Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE): + if dbms in (DBMS.MSSQL, DBMS.SYBASE): retVal = re.sub(r"(?i)\A\[?%s\]?\." % DEFAULT_MSSQL_SCHEMA, "", retVal) return retVal diff --git a/lib/core/convert.py b/lib/core/convert.py index 6588faf1a4..31bbf9b8ec 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -464,6 +464,9 @@ def stdoutEncode(value): return retVal +# str.isascii() is available on Python 3.7+ only (sqlmap still supports 2.7) +_HAS_ISASCII = hasattr(str, "isascii") + def getConsoleLength(value): """ Returns console width of unicode values @@ -475,7 +478,15 @@ def getConsoleLength(value): """ if isinstance(value, six.text_type): - retVal = len(value) + sum(ord(_) >= 0x3000 for _ in value) + # Fast path: ASCII values have no wide (>= U+3000) characters, so their + # console width is simply their length. str.isascii() (Python 3.7+) is a + # C-level scan, far cheaper than the per-character generator below (which + # stays for the rare wide-character case and for Python 2). This runs + # once per dumped cell, so it dominates large table dumps. + if _HAS_ISASCII and value.isascii(): + retVal = len(value) + else: + retVal = len(value) + sum(ord(_) >= 0x3000 for _ in value) else: retVal = len(value) diff --git a/lib/core/dump.py b/lib/core/dump.py index 37264e93ec..c81f525191 100644 --- a/lib/core/dump.py +++ b/lib/core/dump.py @@ -627,11 +627,25 @@ def dbTableValues(self, tableValues): elif conf.dumpFormat == DUMP_FORMAT.SQLITE: rtable.beginTransaction() + # Precompute the per-column layout once. These values are invariant across + # every row, so resolving them per cell (dict lookup, int() conversion and + # identifier normalization) wasted count x ncols work on large dumps. + dumpColumns = [] + for column in columns: + if column != "__infos__": + info = tableValues[column] + dumpColumns.append((unsafeSQLIdentificatorNaming(column), info["values"], int(info["length"]))) + for i in xrange(count): console = (i >= count - TRIM_STDOUT_DUMP_SIZE) field = 1 - values = [] - record = OrderedDict() + + # Only the SQLITE and JSONL paths accumulate a per-row container; the + # others left these unused, wasting an allocation on every single row + if conf.dumpFormat == DUMP_FORMAT.SQLITE: + values = [] + elif conf.dumpFormat == DUMP_FORMAT.JSONL: + record = OrderedDict() if i == 0 and count > TRIM_STDOUT_DUMP_SIZE: self._write(" ...") @@ -639,62 +653,58 @@ def dbTableValues(self, tableValues): if conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile(dumpFP, "") - for column in columns: - if column != "__infos__": - info = tableValues[column] - - if len(info["values"]) <= i or info["values"][i] is None: - value = u'' + for safeColumn, colValues, maxlength in dumpColumns: + if len(colValues) <= i or colValues[i] is None: + value = u'' + else: + value = getUnicode(colValues[i]) + value = DUMP_REPLACEMENTS.get(value, value) + + if conf.dumpFormat == DUMP_FORMAT.SQLITE: + # Note: store a real NULL for the NULL sentinel (and the raw value otherwise), + # mirroring the JSONL path below; appending the display-replaced 'NULL'/'' + # text would corrupt the INTEGER/REAL-typed columns inferred above + if len(colValues) <= i or colValues[i] is None or colValues[i] == " ": # NULL + values.append(None) else: - value = getUnicode(info["values"][i]) - value = DUMP_REPLACEMENTS.get(value, value) - - if conf.dumpFormat == DUMP_FORMAT.SQLITE: - # Note: store a real NULL for the NULL sentinel (and the raw value otherwise), - # mirroring the JSONL path below; appending the display-replaced 'NULL'/'' - # text would corrupt the INTEGER/REAL-typed columns inferred above - if len(info["values"]) <= i or info["values"][i] is None or info["values"][i] == " ": # NULL - values.append(None) - else: - values.append(getUnicode(info["values"][i])) + values.append(getUnicode(colValues[i])) - maxlength = int(info["length"]) - blank = " " * (maxlength - getConsoleLength(value)) - self._write("| %s%s" % (value, blank), newline=False, console=console) + blank = " " * (maxlength - getConsoleLength(value)) + self._write("| %s%s" % (value, blank), newline=False, console=console) - if len(value) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value: - try: - mimetype = getText(magic.from_buffer(getBytes(value), mime=True)) - if any(mimetype.startswith(_) for _ in ("application", "image")): - if not os.path.isdir(dumpDbPath): - os.makedirs(dumpDbPath) + if len(value) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value: + try: + mimetype = getText(magic.from_buffer(getBytes(value), mime=True)) + if any(mimetype.startswith(_) for _ in ("application", "image")): + if not os.path.isdir(dumpDbPath): + os.makedirs(dumpDbPath) - _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode(unsafeSQLIdentificatorNaming(column))) - filepath = os.path.join(dumpDbPath, "%s-%d.bin" % (_, randomInt(8))) - warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath) - logger.warning(warnMsg) + _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode(safeColumn)) + filepath = os.path.join(dumpDbPath, "%s-%d.bin" % (_, randomInt(8))) + warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath) + logger.warning(warnMsg) - with openFile(filepath, "w+b", None) as f: - _ = safechardecode(value, True) - f.write(_) + with openFile(filepath, "w+b", None) as f: + _ = safechardecode(value, True) + f.write(_) - except Exception as ex: - logger.debug(getSafeExString(ex)) + except Exception as ex: + logger.debug(getSafeExString(ex)) - if conf.dumpFormat == DUMP_FORMAT.CSV: - if field == fields: - dataToDumpFile(dumpFP, "%s" % safeCSValue(value)) - else: - dataToDumpFile(dumpFP, "%s%s" % (safeCSValue(value), conf.csvDel)) - elif conf.dumpFormat == DUMP_FORMAT.HTML: - dataToDumpFile(dumpFP, "%s" % getUnicode(htmlEscape(value).encode("ascii", "xmlcharrefreplace"))) - elif conf.dumpFormat == DUMP_FORMAT.JSONL: - if len(info["values"]) <= i or info["values"][i] is None or info["values"][i] == " ": # NULL - record[unsafeSQLIdentificatorNaming(column)] = None - else: - record[unsafeSQLIdentificatorNaming(column)] = getUnicode(info["values"][i]) + if conf.dumpFormat == DUMP_FORMAT.CSV: + if field == fields: + dataToDumpFile(dumpFP, "%s" % safeCSValue(value)) + else: + dataToDumpFile(dumpFP, "%s%s" % (safeCSValue(value), conf.csvDel)) + elif conf.dumpFormat == DUMP_FORMAT.HTML: + dataToDumpFile(dumpFP, "%s" % getUnicode(htmlEscape(value).encode("ascii", "xmlcharrefreplace"))) + elif conf.dumpFormat == DUMP_FORMAT.JSONL: + if len(colValues) <= i or colValues[i] is None or colValues[i] == " ": # NULL + record[safeColumn] = None + else: + record[safeColumn] = getUnicode(colValues[i]) - field += 1 + field += 1 if conf.dumpFormat == DUMP_FORMAT.SQLITE: try: diff --git a/lib/core/settings.py b/lib/core/settings.py index 43667bf80e..f1fc8935e5 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.6.199" +VERSION = "1.10.7.3" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -187,6 +187,22 @@ STRUCTURAL_CLASS_REGEX = r"""(?si)\bclass\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'<>]+))""" STRUCTURAL_ID_REGEX = r"""(?si)\bid\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'<>]+))""" +# Minimum response size (in bytes) for the 'skip-read' NULL connection method to be used. Unlike +# HEAD/Range, 'skip-read' leaves the body unread and must therefore close the connection (an unread +# body cannot be reused), paying a fresh TCP/TLS handshake per request. That only pays off when +# avoiding the body transfer outweighs the reconnect - i.e. for large responses; for small ones it +# is a net slowdown, so it is gated by this size +NULL_CONNECTION_SKIP_READ_MIN_LENGTH = 256 * 1024 + +# Coarse plausibility band for a NULL connection method's reported length, relative to the known +# original page length (len(kb.originalPage)). A method is accepted only if its length falls within +# it; this rejects a method whose length does not track the real GET response (e.g. HEAD returning +# 'Content-Length: 0', HEAD served from a different code path, or sneaked-in compression). The band +# is deliberately generous (byte-vs-character size and moderate page dynamism are expected, and a +# false reject merely forgoes the optimization, which is safe) - it only catches gross mismatches +NULL_CONNECTION_LENGTH_TOLERANCE_LOW = 0.5 +NULL_CONNECTION_LENGTH_TOLERANCE_HIGH = 4.0 + # Regular expression used for recognition of IP addresses IP_ADDRESS_REGEX = r"\b(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\b" diff --git a/lib/request/connect.py b/lib/request/connect.py index 40c42390bf..a14309fa80 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -63,7 +63,6 @@ class WebSocketException(Exception): from lib.core.common import urldecode from lib.core.common import urlencode from lib.core.common import wasLastResponseDelayed -from lib.core.compat import LooseVersion from lib.core.compat import patchHeaders from lib.core.compat import xrange from lib.core.convert import encodeBase64 @@ -111,7 +110,6 @@ class WebSocketException(Exception): from lib.core.settings import JAVASCRIPT_HREF_REGEX from lib.core.settings import LARGE_READ_TRIM_MARKER from lib.core.settings import LIVE_COOKIES_TIMEOUT -from lib.core.settings import MIN_HTTPX_VERSION from lib.core.settings import MAX_CONNECTION_READ_SIZE from lib.core.settings import MAX_CONNECTIONS_REGEX from lib.core.settings import MAX_CONNECTION_TOTAL_SIZE @@ -632,30 +630,22 @@ class _(dict): cookie.value = re.sub(r"(%s)([^ \t])" % char, r"\g<1>\t\g<2>", cookie.value) if conf.http2: - try: - import httpx - except ImportError: - raise SqlmapMissingDependence("httpx[http2] not available (e.g. 'pip%s install httpx[http2]')" % ('3' if six.PY3 else "")) + from lib.request.http2 import open_url as http2OpenUrl - if LooseVersion(httpx.__version__) < LooseVersion(MIN_HTTPX_VERSION): - raise SqlmapMissingDependence("outdated version of httpx detected (%s<%s)" % (httpx.__version__, MIN_HTTPX_VERSION)) + h2proxy = None + if conf.proxy: + _proxyParts = _urllib.parse.urlsplit(conf.proxy if "://" in conf.proxy else "http://%s" % conf.proxy) + if (_proxyParts.scheme or "").lower().startswith("socks"): + raise SqlmapMissingDependence("native HTTP/2 client does not support SOCKS proxies (omit '--http2' or use an HTTP proxy)") + h2proxy = (_proxyParts.hostname, _proxyParts.port or 8080, conf.proxyCred or None) try: - proxy_mounts = dict(("%s://" % key, httpx.HTTPTransport(proxy="%s%s" % ("http://" if "://" not in kb.proxies[key] else "", kb.proxies[key]))) for key in kb.proxies) if kb.proxies else None - with httpx.Client(verify=False, http2=True, timeout=timeout, follow_redirects=True, cookies=conf.cj, mounts=proxy_mounts) as client: - conn = client.request(method or (HTTPMETHOD.POST if post is not None else HTTPMETHOD.GET), url, headers=headers, data=post) - except (httpx.HTTPError, httpx.InvalidURL, httpx.CookieConflict, httpx.StreamError) as ex: + conn = http2OpenUrl(url, method or (HTTPMETHOD.POST if post is not None else HTTPMETHOD.GET), headers, post, timeout, follow_redirects=kb.choices.redirect != REDIRECTION.NO, proxy=h2proxy) + except IOError as ex: raise _http_client.HTTPException(getSafeExString(ex)) else: - if conn.status_code >= 400: - raise _urllib.error.HTTPError(url, conn.status_code, conn.reason_phrase, conn.headers, io.BytesIO(conn.read())) - - conn.code = conn.status_code - conn.msg = conn.reason_phrase - conn.info = lambda c=conn: c.headers - - conn._read_buffer = conn.read() - conn._read_offset = 0 + if conn.code >= 400: + raise _urllib.error.HTTPError(url, conn.code, conn.msg, conn.info(), io.BytesIO(conn.read())) requestMsg = re.sub(r" HTTP/[0-9.]+\r\n", " %s\r\n" % conn.http_version, requestMsg, count=1) @@ -663,18 +653,6 @@ class _(dict): threadData.lastRequestMsg = requestMsg logger.log(CUSTOM_LOGGING.TRAFFIC_OUT, requestMsg) - - def _read(count=None): - offset = conn._read_offset - if count is None: - result = conn._read_buffer[offset:] - conn._read_offset = len(conn._read_buffer) - else: - result = conn._read_buffer[offset: offset + count] - conn._read_offset += len(result) - return result - - conn.read = _read else: if not multipart: threadData.lastRequestMsg = requestMsg diff --git a/lib/request/http2.py b/lib/request/http2.py new file mode 100644 index 0000000000..2af00c69ec --- /dev/null +++ b/lib/request/http2.py @@ -0,0 +1,544 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +# Native, dependency-free HTTP/2 client (RFC 7540) with HPACK (RFC 7541), replacing the optional +# 'httpx[http2]' third-party stack. The HPACK static and Huffman tables below are the canonical +# RFC 7541 tables; the codec is validated differentially against python-hyper/hpack and the client +# end-to-end against real h2 servers. Pure standard library, Python 2.7 / 3.x. + +import base64 +import socket +import ssl +import struct + +try: + from http.client import responses as _HTTP_RESPONSES +except ImportError: + from httplib import responses as _HTTP_RESPONSES + +try: + from urllib.parse import urljoin, urlsplit +except ImportError: + from urlparse import urljoin, urlsplit + +from email.message import Message as _Message + +REDIRECT_CODES = (301, 302, 303, 307, 308) + + +HUFFMAN_CODES = [ + 0x1ff8, 0x7fffd8, 0xfffffe2, 0xfffffe3, 0xfffffe4, 0xfffffe5, 0xfffffe6, 0xfffffe7, 0xfffffe8, 0xffffea, + 0x3ffffffc, 0xfffffe9, 0xfffffea, 0x3ffffffd, 0xfffffeb, 0xfffffec, 0xfffffed, 0xfffffee, 0xfffffef, + 0xffffff0, 0xffffff1, 0xffffff2, 0x3ffffffe, 0xffffff3, 0xffffff4, 0xffffff5, 0xffffff6, 0xffffff7, 0xffffff8, + 0xffffff9, 0xffffffa, 0xffffffb, 0x14, 0x3f8, 0x3f9, 0xffa, 0x1ff9, 0x15, 0xf8, 0x7fa, 0x3fa, 0x3fb, 0xf9, + 0x7fb, 0xfa, 0x16, 0x17, 0x18, 0x0, 0x1, 0x2, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x5c, 0xfb, 0x7ffc, + 0x20, 0xffb, 0x3fc, 0x1ffa, 0x21, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xfc, 0x73, 0xfd, 0x1ffb, 0x7fff0, 0x1ffc, 0x3ffc, + 0x22, 0x7ffd, 0x3, 0x23, 0x4, 0x24, 0x5, 0x25, 0x26, 0x27, 0x6, 0x74, 0x75, 0x28, 0x29, 0x2a, 0x7, 0x2b, 0x76, + 0x2c, 0x8, 0x9, 0x2d, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7ffe, 0x7fc, 0x3ffd, 0x1ffd, 0xffffffc, 0xfffe6, + 0x3fffd2, 0xfffe7, 0xfffe8, 0x3fffd3, 0x3fffd4, 0x3fffd5, 0x7fffd9, 0x3fffd6, 0x7fffda, 0x7fffdb, 0x7fffdc, + 0x7fffdd, 0x7fffde, 0xffffeb, 0x7fffdf, 0xffffec, 0xffffed, 0x3fffd7, 0x7fffe0, 0xffffee, 0x7fffe1, 0x7fffe2, + 0x7fffe3, 0x7fffe4, 0x1fffdc, 0x3fffd8, 0x7fffe5, 0x3fffd9, 0x7fffe6, 0x7fffe7, 0xffffef, 0x3fffda, 0x1fffdd, + 0xfffe9, 0x3fffdb, 0x3fffdc, 0x7fffe8, 0x7fffe9, 0x1fffde, 0x7fffea, 0x3fffdd, 0x3fffde, 0xfffff0, 0x1fffdf, + 0x3fffdf, 0x7fffeb, 0x7fffec, 0x1fffe0, 0x1fffe1, 0x3fffe0, 0x1fffe2, 0x7fffed, 0x3fffe1, 0x7fffee, 0x7fffef, + 0xfffea, 0x3fffe2, 0x3fffe3, 0x3fffe4, 0x7ffff0, 0x3fffe5, 0x3fffe6, 0x7ffff1, 0x3ffffe0, 0x3ffffe1, 0xfffeb, + 0x7fff1, 0x3fffe7, 0x7ffff2, 0x3fffe8, 0x1ffffec, 0x3ffffe2, 0x3ffffe3, 0x3ffffe4, 0x7ffffde, 0x7ffffdf, + 0x3ffffe5, 0xfffff1, 0x1ffffed, 0x7fff2, 0x1fffe3, 0x3ffffe6, 0x7ffffe0, 0x7ffffe1, 0x3ffffe7, 0x7ffffe2, + 0xfffff2, 0x1fffe4, 0x1fffe5, 0x3ffffe8, 0x3ffffe9, 0xffffffd, 0x7ffffe3, 0x7ffffe4, 0x7ffffe5, 0xfffec, + 0xfffff3, 0xfffed, 0x1fffe6, 0x3fffe9, 0x1fffe7, 0x1fffe8, 0x7ffff3, 0x3fffea, 0x3fffeb, 0x1ffffee, 0x1ffffef, + 0xfffff4, 0xfffff5, 0x3ffffea, 0x7ffff4, 0x3ffffeb, 0x7ffffe6, 0x3ffffec, 0x3ffffed, 0x7ffffe7, 0x7ffffe8, + 0x7ffffe9, 0x7ffffea, 0x7ffffeb, 0xffffffe, 0x7ffffec, 0x7ffffed, 0x7ffffee, 0x7ffffef, 0x7fffff0, 0x3ffffee, + 0x3fffffff +] + + +HUFFMAN_LENGTHS = [ + 0xd, 0x17, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x18, 0x1e, 0x1c, 0x1c, 0x1e, 0x1c, 0x1c, 0x1c, 0x1c, + 0x1c, 0x1c, 0x1c, 0x1c, 0x1e, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x1c, 0x6, 0xa, 0xa, 0xc, 0xd, + 0x6, 0x8, 0xb, 0xa, 0xa, 0x8, 0xb, 0x8, 0x6, 0x6, 0x6, 0x5, 0x5, 0x5, 0x6, 0x6, 0x6, 0x6, 0x6, 0x6, 0x6, 0x7, + 0x8, 0xf, 0x6, 0xc, 0xa, 0xd, 0x6, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, + 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x8, 0x7, 0x8, 0xd, 0x13, 0xd, 0xe, 0x6, 0xf, 0x5, 0x6, 0x5, 0x6, 0x5, 0x6, + 0x6, 0x6, 0x5, 0x7, 0x7, 0x6, 0x6, 0x6, 0x5, 0x6, 0x7, 0x6, 0x5, 0x5, 0x6, 0x7, 0x7, 0x7, 0x7, 0x7, 0xf, 0xb, + 0xe, 0xd, 0x1c, 0x14, 0x16, 0x14, 0x14, 0x16, 0x16, 0x16, 0x17, 0x16, 0x17, 0x17, 0x17, 0x17, 0x17, 0x18, + 0x17, 0x18, 0x18, 0x16, 0x17, 0x18, 0x17, 0x17, 0x17, 0x17, 0x15, 0x16, 0x17, 0x16, 0x17, 0x17, 0x18, 0x16, + 0x15, 0x14, 0x16, 0x16, 0x17, 0x17, 0x15, 0x17, 0x16, 0x16, 0x18, 0x15, 0x16, 0x17, 0x17, 0x15, 0x15, 0x16, + 0x15, 0x17, 0x16, 0x17, 0x17, 0x14, 0x16, 0x16, 0x16, 0x17, 0x16, 0x16, 0x17, 0x1a, 0x1a, 0x14, 0x13, 0x16, + 0x17, 0x16, 0x19, 0x1a, 0x1a, 0x1a, 0x1b, 0x1b, 0x1a, 0x18, 0x19, 0x13, 0x15, 0x1a, 0x1b, 0x1b, 0x1a, 0x1b, + 0x18, 0x15, 0x15, 0x1a, 0x1a, 0x1c, 0x1b, 0x1b, 0x1b, 0x14, 0x18, 0x14, 0x15, 0x16, 0x15, 0x15, 0x17, 0x16, + 0x16, 0x19, 0x19, 0x18, 0x18, 0x1a, 0x17, 0x1a, 0x1b, 0x1a, 0x1a, 0x1b, 0x1b, 0x1b, 0x1b, 0x1b, 0x1c, 0x1b, + 0x1b, 0x1b, 0x1b, 0x1b, 0x1a, 0x1e +] + + +STATIC_TABLE = ( + (b':authority', b''), + (b':method', b'GET'), + (b':method', b'POST'), + (b':path', b'/'), + (b':path', b'/index.html'), + (b':scheme', b'http'), + (b':scheme', b'https'), + (b':status', b'200'), + (b':status', b'204'), + (b':status', b'206'), + (b':status', b'304'), + (b':status', b'400'), + (b':status', b'404'), + (b':status', b'500'), + (b'accept-charset', b''), + (b'accept-encoding', b'gzip, deflate'), + (b'accept-language', b''), + (b'accept-ranges', b''), + (b'accept', b''), + (b'access-control-allow-origin', b''), + (b'age', b''), + (b'allow', b''), + (b'authorization', b''), + (b'cache-control', b''), + (b'content-disposition', b''), + (b'content-encoding', b''), + (b'content-language', b''), + (b'content-length', b''), + (b'content-location', b''), + (b'content-range', b''), + (b'content-type', b''), + (b'cookie', b''), + (b'date', b''), + (b'etag', b''), + (b'expect', b''), + (b'expires', b''), + (b'from', b''), + (b'host', b''), + (b'if-match', b''), + (b'if-modified-since', b''), + (b'if-none-match', b''), + (b'if-range', b''), + (b'if-unmodified-since', b''), + (b'last-modified', b''), + (b'link', b''), + (b'location', b''), + (b'max-forwards', b''), + (b'proxy-authenticate', b''), + (b'proxy-authorization', b''), + (b'range', b''), + (b'referer', b''), + (b'refresh', b''), + (b'retry-after', b''), + (b'server', b''), + (b'set-cookie', b''), + (b'strict-transport-security', b''), + (b'transfer-encoding', b''), + (b'user-agent', b''), + (b'vary', b''), + (b'via', b''), + (b'www-authenticate', b''), +) +STATIC_LEN = len(STATIC_TABLE) + + +# HTTP/2 frame codec (RFC 7540 section 4.1) - the zero-table-risk brick. Pure stdlib, py2/py3, ASCII. + +# frame types (RFC 7540 s6) +DATA, HEADERS, RST_STREAM, SETTINGS, PING, GOAWAY, WINDOW_UPDATE, CONTINUATION = 0x0, 0x1, 0x3, 0x4, 0x6, 0x7, 0x8, 0x9 +# flags +FLAG_END_STREAM = 0x1 +FLAG_ACK = 0x1 +FLAG_END_HEADERS = 0x4 +FLAG_PADDED = 0x8 +FLAG_PRIORITY = 0x20 + +CONNECTION_PREFACE = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + +def encode_frame(ftype, flags, stream_id, payload=b""): + if len(payload) > 0xffffff: + raise ValueError("frame payload exceeds 24-bit length") + header = struct.pack("!I", len(payload))[1:] # 24-bit length (drop MSB of the 32-bit pack) + header += struct.pack("!BBI", ftype, flags, stream_id & 0x7fffffff) # type, flags, R(1)+stream(31) + return header + payload + +def decode_frame_header(nine): + if len(nine) != 9: + raise ValueError("frame header must be exactly 9 bytes") + length = struct.unpack("!I", b"\x00" + nine[:3])[0] + ftype, flags, stream_id = struct.unpack("!BBI", nine[3:9]) + return length, ftype, flags, stream_id & 0x7fffffff + +# ---------- Huffman ---------- +def huffman_encode(data): + if not data: + return b"" + acc = 0 + nbits = 0 + for b in bytearray(data): + acc = (acc << HUFFMAN_LENGTHS[b]) | HUFFMAN_CODES[b] + nbits += HUFFMAN_LENGTHS[b] + pad = (8 - nbits % 8) % 8 + acc = (acc << pad) | ((1 << pad) - 1) # pad with 1-bits (EOS prefix) + total = (nbits + pad) // 8 + out = bytearray() + for i in range(total - 1, -1, -1): + out.append((acc >> (8 * i)) & 0xff) + return bytes(out) + +_HUFF_ROOT = {} +def _build_huffman_trie(): + for sym in range(256): + code, length = HUFFMAN_CODES[sym], HUFFMAN_LENGTHS[sym] + node = _HUFF_ROOT + for i in range(length - 1, -1, -1): + bit = (code >> i) & 1 + if i == 0: + node[bit] = sym # leaf: int symbol + else: + node = node.setdefault(bit, {}) +_build_huffman_trie() + +def huffman_decode(data): + out = bytearray() + node = _HUFF_ROOT + consumed = 0 # bits into the current (partial) symbol + for byte in bytearray(data): + for i in range(7, -1, -1): + bit = (byte >> i) & 1 + nxt = node.get(bit) + if nxt is None: + raise ValueError("invalid Huffman sequence") + consumed += 1 + if isinstance(nxt, dict): + node = nxt + else: + out.append(nxt) + node = _HUFF_ROOT + consumed = 0 + # RFC 7541 5.2: any leftover partial path must be EOS padding: all 1-bits and fewer than 8 + if node is not _HUFF_ROOT: + if consumed >= 8: + raise ValueError("Huffman padding too long") + # walk back is unnecessary: padding is all-ones, i.e. we must have only taken '1' branches + # since the last leaf; verify by re-deriving is overkill - reference cross-check guards it + return bytes(out) + +# ---------- integer / string (RFC 7541 5.1 / 5.2) ---------- +def encode_integer(value, prefix_bits, first_byte=0): + mask = (1 << prefix_bits) - 1 + if value < mask: + return bytearray([first_byte | value]) + out = bytearray([first_byte | mask]) + value -= mask + while value >= 0x80: + out.append((value & 0x7f) | 0x80) + value >>= 7 + out.append(value) + return out + +def decode_integer(data, pos, prefix_bits): + mask = (1 << prefix_bits) - 1 + value = data[pos] & mask + pos += 1 + if value < mask: + return value, pos + shift = 0 + while True: + b = data[pos] + pos += 1 + value += (b & 0x7f) << shift + shift += 7 + if not (b & 0x80): + break + return value, pos + +def encode_string(value, huffman=True): + if huffman: + encoded = huffman_encode(value) + if len(encoded) < len(value): # only use Huffman when it actually shrinks + return encode_integer(len(encoded), 7, 0x80) + encoded + return encode_integer(len(value), 7, 0x00) + bytearray(value) + +def decode_string(data, pos): + huffman = bool(data[pos] & 0x80) + length, pos = decode_integer(data, pos, 7) + raw = bytes(data[pos:pos + length]) + pos += length + return (huffman_decode(raw) if huffman else raw), pos + +# ---------- dynamic table + decoder/encoder ---------- +class Decoder(object): + def __init__(self, max_size=4096): + self.max_size = max_size + self.dynamic = [] # newest first: [(name, value), ...] + self._size = 0 + + def _entry_size(self, name, value): + return 32 + len(name) + len(value) + + def _add(self, name, value): + self.dynamic.insert(0, (name, value)) + self._size += self._entry_size(name, value) + self._evict() + + def _evict(self): + while self._size > self.max_size and self.dynamic: + name, value = self.dynamic.pop() + self._size -= self._entry_size(name, value) + + def _get(self, index): + if index <= 0: + raise ValueError("invalid header index 0") + if index <= STATIC_LEN: + return STATIC_TABLE[index - 1] + index -= STATIC_LEN + 1 + if index >= len(self.dynamic): + raise ValueError("dynamic index out of range") + return self.dynamic[index] + + def decode(self, data): + data = bytearray(data) + pos = 0 + headers = [] + n = len(data) + while pos < n: + byte = data[pos] + if byte & 0x80: # 6.1 indexed + index, pos = decode_integer(data, pos, 7) + headers.append(self._get(index)) + elif byte & 0x40: # 6.2.1 literal + incremental indexing + index, pos = decode_integer(data, pos, 6) + if index: + name = self._get(index)[0] + else: + name, pos = decode_string(data, pos) + value, pos = decode_string(data, pos) + self._add(name, value) + headers.append((name, value)) + elif byte & 0x20: # 6.3 dynamic table size update + new_size, pos = decode_integer(data, pos, 5) + self.max_size = new_size + self._evict() + else: # 6.2.2 without / 6.2.3 never indexed (4-bit prefix) + index, pos = decode_integer(data, pos, 4) + if index: + name = self._get(index)[0] + else: + name, pos = decode_string(data, pos) + value, pos = decode_string(data, pos) + headers.append((name, value)) + return headers + +class Encoder(object): + # Minimal, always-valid: emit each header as a literal WITHOUT indexing + Huffman-coded strings. + # (Correctness-critical decoding is the hard part; a server accepts this trivially.) + def encode(self, headers): + out = bytearray() + for name, value in headers: + out += encode_integer(0, 4, 0x00) # 0000 0000 : literal w/o indexing, new name + out += encode_string(name) + out += encode_string(value) + return bytes(out) + +SETTINGS_INITIAL_WINDOW_SIZE = 0x4 +BIG_WINDOW = (1 << 31) - 1 + +def _recv_exact(sock, n): + buf = b"" + while len(buf) < n: + chunk = sock.recv(n - len(buf)) + if not chunk: + raise IOError("connection closed by peer") + buf += chunk + return buf + +def _read_frame(sock): + length, ftype, flags, sid = decode_frame_header(_recv_exact(sock, 9)) + return ftype, flags, sid, (_recv_exact(sock, length) if length else b"") + +def _tob(x): + return x if isinstance(x, bytes) else x.encode("latin-1") + +def _connect_socket(host, port, proxy, timeout): + # Direct TCP, or an HTTP CONNECT tunnel through an (optionally authenticated) proxy. SOCKS proxies + # are excluded for HTTP/2 upstream, so any proxy reaching here is a plain HTTP one. proxy is a + # (proxy_host, proxy_port, "user:pass"-or-None) tuple. + if not proxy: + return socket.create_connection((host, port), timeout=timeout) + + proxy_host, proxy_port, proxy_cred = proxy + raw = socket.create_connection((proxy_host, proxy_port), timeout=timeout) + try: + request = "CONNECT %s:%d HTTP/1.1\r\nHost: %s:%d\r\n" % (host, port, host, port) + if proxy_cred: + token = base64.b64encode(proxy_cred.encode("latin-1")).decode("ascii") + request += "Proxy-Authorization: Basic %s\r\n" % token + request += "\r\n" + raw.sendall(request.encode("latin-1")) + + response = b"" + while b"\r\n\r\n" not in response: + chunk = raw.recv(4096) + if not chunk: + raise IOError("proxy closed the connection during CONNECT") + response += chunk + if len(response) > 65536: + raise IOError("oversized proxy CONNECT response") + + status_line = response.split(b"\r\n", 1)[0].decode("latin-1", "replace") + fields = status_line.split(None, 2) + code = int(fields[1]) if len(fields) >= 2 and fields[1].isdigit() else 0 + if not (200 <= code < 300): + raise IOError("proxy CONNECT failed: %s" % status_line) + return raw + except Exception: + try: + raw.close() + except Exception: + pass + raise + +def h2_request(host, port=443, method="GET", path="/", authority=None, headers=None, body=None, timeout=30, proxy=None): + authority = authority or host + ctx = ssl._create_unverified_context() + ctx.set_alpn_protocols(["h2"]) + sock = ctx.wrap_socket(_connect_socket(host, port, proxy, timeout), server_hostname=host) + try: + if sock.selected_alpn_protocol() != "h2": + raise IOError("server did not negotiate h2 (ALPN=%r)" % sock.selected_alpn_protocol()) + sock.settimeout(timeout) + + # connection preface + client SETTINGS (advertise a large per-stream window) + bump conn window + sock.sendall(CONNECTION_PREFACE) + sock.sendall(encode_frame(SETTINGS, 0, 0, struct.pack("!HI", SETTINGS_INITIAL_WINDOW_SIZE, BIG_WINDOW))) + sock.sendall(encode_frame(WINDOW_UPDATE, 0, 0, struct.pack("!I", BIG_WINDOW - 65535))) + + req = [(b":method", _tob(method)), (b":scheme", b"https"), (b":path", _tob(path)), (b":authority", _tob(authority))] + for k, v in (headers or {}).items(): + req.append((_tob(k).lower(), _tob(v))) + hblock = Encoder().encode(req) + sock.sendall(encode_frame(HEADERS, FLAG_END_HEADERS | (0 if body else FLAG_END_STREAM), 1, hblock)) + if body: + sock.sendall(encode_frame(DATA, FLAG_END_STREAM, 1, _tob(body))) + + dec = Decoder() + header_block, resp_headers, resp_body, done = b"", None, bytearray(), False + while not done: + ftype, flags, sid, payload = _read_frame(sock) + if ftype == SETTINGS: + if not (flags & FLAG_ACK): + sock.sendall(encode_frame(SETTINGS, FLAG_ACK, 0, b"")) + elif ftype == PING: + if not (flags & FLAG_ACK): + sock.sendall(encode_frame(PING, FLAG_ACK, 0, payload)) + elif ftype == GOAWAY: + done = True + elif ftype == RST_STREAM and sid == 1: + raise IOError("stream reset by server (error %d)" % struct.unpack("!I", payload[:4])[0]) + elif ftype in (HEADERS, CONTINUATION) and sid == 1: + p = payload + if ftype == HEADERS: + if flags & FLAG_PADDED: + p = p[1:len(p) - bytearray(payload)[0]] + if flags & FLAG_PRIORITY: + p = p[5:] + header_block += p + if flags & FLAG_END_HEADERS: + resp_headers = dec.decode(header_block) + if flags & FLAG_END_STREAM: + done = True + elif ftype == DATA and sid == 1: + p = payload + if flags & FLAG_PADDED: + p = p[1:len(p) - bytearray(payload)[0]] + resp_body += p + if payload: # replenish stream + connection windows + sock.sendall(encode_frame(WINDOW_UPDATE, 0, 1, struct.pack("!I", len(payload)))) + sock.sendall(encode_frame(WINDOW_UPDATE, 0, 0, struct.pack("!I", len(payload)))) + if flags & FLAG_END_STREAM: + done = True + status = None + for n, v in (resp_headers or []): + if _tob(n) == b":status": + status = int(v) + break + return status, resp_headers, bytes(resp_body) + finally: + try: sock.close() + except Exception: pass + + +class H2Response(object): + """A urllib-response-compatible wrapper around a native HTTP/2 response, so the rest of sqlmap's + request pipeline can consume it exactly like a urllib response (code/msg/info()/read()/geturl()).""" + + def __init__(self, url, status, headers, body): + self.url = url + self.code = self.status = status + self.msg = _HTTP_RESPONSES.get(status, "") + self.http_version = "HTTP/2.0" + self._body = body + self._offset = 0 + self._info = _Message() + for name, value in (headers or []): + name = name.decode("latin-1") if isinstance(name, bytes) else name + value = value.decode("latin-1") if isinstance(value, bytes) else value + if not name.startswith(":"): # drop HTTP/2 pseudo-headers (:status etc.) + self._info[name] = value + # expose a mimetools.Message-style '.headers' list so patchHeaders() treats this object + # uniformly across Python 2/3 (email.message.Message lacks it, and Python 2 iteration over a + # bare Message falls back to integer indexing) + self._info.headers = ["%s: %s\r\n" % (name, value) for (name, value) in self._info.items()] + + def info(self): + return self._info + + def geturl(self): + return self.url + + def read(self, amt=None): + if amt is None: + data = self._body[self._offset:] + self._offset = len(self._body) + else: + data = self._body[self._offset:self._offset + amt] + self._offset += len(data) + return data + + def close(self): + pass + + +def open_url(url, method="GET", headers=None, body=None, timeout=30, follow_redirects=True, max_redirects=10, proxy=None): + """Fetch url over native HTTP/2 (https only), following redirects like a browser (mirroring the + previous httpx follow_redirects=True), and return an H2Response. Raises IOError on a transport or + ALPN-negotiation failure. Connection-level and h2-forbidden request headers are stripped.""" + forbidden = ("host", "connection", "keep-alive", "proxy-connection", "transfer-encoding", "upgrade", "content-length") + req_headers = {} + for key in (headers or {}): + name = key.decode("latin-1") if isinstance(key, bytes) else key + if name.lower() not in forbidden: + req_headers[key] = headers[key] + + for _ in range(max_redirects + 1): + parts = urlsplit(url) + if parts.scheme != "https": + raise IOError("native HTTP/2 client supports 'https://' targets only (got %r)" % parts.scheme) + path = parts.path or "/" + if parts.query: + path += "?" + parts.query + status, resp_headers, resp_body = h2_request(parts.hostname, parts.port or 443, method=method, path=path, + authority=parts.netloc.split("@")[-1], headers=req_headers, body=body, timeout=timeout, proxy=proxy) + if follow_redirects and status in REDIRECT_CODES: + location = None + for name, value in (resp_headers or []): + if (name.decode("latin-1") if isinstance(name, bytes) else name).lower() == "location": + location = value.decode("latin-1") if isinstance(value, bytes) else value + break + if location: + url = urljoin(url, location) + if status in (301, 302, 303): # per RFC 7231, these degrade to GET + method, body = "GET", None + continue + return H2Response(url, status, resp_headers, resp_body) + + raise IOError("too many HTTP/2 redirects") diff --git a/lib/utils/deps.py b/lib/utils/deps.py index 51a9a23ea4..ce61a7344c 100644 --- a/lib/utils/deps.py +++ b/lib/utils/deps.py @@ -94,16 +94,6 @@ def checkDependencies(): logger.warning(warnMsg) missing_libraries.add('python-ntlm') - try: - __import__("httpx") - debugMsg = "'httpx[http2]' third-party library is found" - logger.debug(debugMsg) - except ImportError: - warnMsg = "sqlmap requires 'httpx[http2]' third-party library " - warnMsg += "if you plan to use HTTP version 2" - logger.warning(warnMsg) - missing_libraries.add('httpx[http2]') - try: __import__("websocket._abnf") debugMsg = "'websocket-client' library is found" diff --git a/lib/utils/dialect.py b/lib/utils/dialect.py index 3be67eac89..47f973edcb 100644 --- a/lib/utils/dialect.py +++ b/lib/utils/dialect.py @@ -28,10 +28,10 @@ # OTHER valid rows, which sqlmap's fuzzy page comparison conflates with the anchor row, producing # false positives. See PROVE_DESIGN.md.) # -# Truth table measured on a live OWASP-CRS platform across 16 engines (MySQL/MySQL5, MariaDB/TiDB, -# PostgreSQL, CockroachDB, CrateDB, Microsoft SQL Server, SQLite, Firebird, ClickHouse, H2, HSQLDB, -# Derby, MonetDB, IRIS, Trino); only the zero-false-positive rules are kept (see _classify). With -# anchor value 2: +# Signatures were measured against every SQL engine on a live OWASP-CRS platform (MySQL/MySQL5, +# MariaDB/TiDB, PostgreSQL, CockroachDB, CrateDB, Microsoft SQL Server, SQLite, Firebird, ClickHouse, +# H2, HSQLDB, Derby, MonetDB, IRIS, Trino) and encoded as an exact-signature WHITELIST in _classify() +# (only measured signatures classify; anything else -> None). With anchor value 2: # # * 2^0=2 -> '^' is bitwise XOR (MySQL/MSSQL/MonetDB: 2^0=2) vs exponentiation (PostgreSQL: 2^0=1) # vs no such operator (SQLite/Oracle/... -> error, so false) @@ -52,57 +52,69 @@ ("shift", "1<<2=4"), ) +# Canary for the trustworthiness gate: a syntactically-invalid expression (a trailing operator) that +# a real SQL back-end can only read as FALSE - the appended clause is a parse error, the query fails, +# no row. A false-positive / noise channel (a WAF, a reflection, or a backend that ignores the +# injected tail and reads every probe the same) reads it as TRUE, which is proof the boolean oracle +# is trash, so the heuristic returns None (a true negative) rather than a bogus DBMS from a +# meaningless signature. It uses a trailing-operator form, distinct from the ' ' no-operator +# form already exercised by sqlmap's earlier false-positive check, so it adds new information. +DIALECT_CANARY = "2+" + +# Exact operator-dialect signature -> back-end DBMS. Strict WHITELIST re-derived from the live +# measurement above: ONLY these signatures classify; any other - an engine not measured here, or a +# false-positive / noise channel - returns None. This deliberately replaces earlier partial-condition +# rules, which would confidently mis-map physically-impossible signatures onto a DBMS (e.g. the +# all-true 'reads everything as true' noise, where '^' would be XOR and exponentiation at once). +_SIGNATURE_DBMS = { + # xor pgpow intdiv bitor shift + (True, False, False, True, True): DBMS.MYSQL, # MySQL / MariaDB / TiDB + (False, True, True, True, True): DBMS.PGSQL, # PostgreSQL + (False, True, False, True, True): DBMS.PGSQL, # CockroachDB (pgwire; has '<<' -> shift True) + (False, True, True, True, False): DBMS.PGSQL, # CrateDB + (True, False, True, True, False): DBMS.MSSQL, # Microsoft SQL Server (no bit-shift) + (True, False, True, True, True): DBMS.MONETDB, # MonetDB (as MSSQL but has '<<') + (False, False, True, True, True): DBMS.SQLITE, # SQLite +} + def _classify(signature): """ - Maps a measured (xor, pgpow, intdiv, bitor) operator-dialect signature to a back-end - DBMS, or returns None when the signature does not *uniquely* identify a major DBMS (so - detection proceeds unchanged - the heuristic never wrong-foots the scan). - - Rules below are the subset of the measured 11-engine truth table that maps with zero - false positives. Engines whose operator profile is not distinctive enough (Oracle's - all-false signature, which a minimal engine like ClickHouse/H2/Firebird/HSQLDB/Derby or - a fully WAF-blocked channel also produces) deliberately fall through to None: + Maps an exact operator-dialect signature (xor, pgpow, intdiv, bitor, shift) to a back-end DBMS + through a strict whitelist of live-measured signatures, or returns None when the signature is not + a known DBMS fingerprint - an engine not measured, or a noise / false-positive channel - so + detection proceeds unchanged and the heuristic never wrong-foots the scan. - >>> _classify((True, False, False, True, True)) # MySQL / MariaDB / TiDB + >>> _classify((True, False, False, True, True)) # MySQL / MariaDB / TiDB 'MySQL' - >>> _classify((True, False, True, True, False)) # Microsoft SQL Server (no bit-shift) - 'Microsoft SQL Server' - >>> _classify((True, False, True, True, True)) # MonetDB (same xor/intdiv as MSSQL, but has '<<') - 'MonetDB' - >>> _classify((False, True, True, True, False)) # PostgreSQL + >>> _classify((False, True, True, True, True)) # PostgreSQL + 'PostgreSQL' + >>> _classify((False, True, False, True, True)) # CockroachDB -> PostgreSQL family 'PostgreSQL' - >>> _classify((False, True, False, True, False)) # CockroachDB (pgwire) -> PostgreSQL family + >>> _classify((False, True, True, True, False)) # CrateDB -> PostgreSQL family 'PostgreSQL' - >>> _classify((False, False, True, True, True)) # SQLite + >>> _classify((True, False, True, True, False)) # Microsoft SQL Server (no bit-shift) + 'Microsoft SQL Server' + >>> _classify((True, False, True, True, True)) # MonetDB (as MSSQL but has '<<') + 'MonetDB' + >>> _classify((False, False, True, True, True)) # SQLite 'SQLite' - >>> _classify((False, False, True, False, False)) is None # Firebird/HSQLDB/Derby/H2/Trino -> no prior + >>> _classify((True, True, True, True, True)) is None # 'reads everything true' noise -> None + True + >>> _classify((False, False, False, False, False)) is None # all-false (Oracle/ClickHouse/IRIS/blocked) -> None True - >>> _classify((False, False, False, False, False)) is None # all-false (Oracle/ClickHouse/IRIS/blocked) -> no prior + >>> _classify((False, False, True, False, False)) is None # Firebird/H2/HSQLDB/Derby/Trino -> not distinctive True """ - xor, pgpow, intdiv, bitor, shift = signature - - if pgpow: # '^' is exponentiation -> PostgreSQL family - return DBMS.PGSQL - if xor and intdiv: # '^' is XOR AND integer division -> SQL Server ... - # ... except MonetDB shares this exact signature; it alone has a working bit-shift operator - # ('1<<2=4'), SQL Server has none -> split the collision (measured zero-FP across 16 engines). - return DBMS.MONETDB if shift else DBMS.MSSQL - if xor and not intdiv: # '^' is XOR AND real division -> MySQL family - return DBMS.MYSQL - if not xor and intdiv and bitor: # no '^', integer division, bitwise '|' -> SQLite - return DBMS.SQLITE - - return None + return _SIGNATURE_DBMS.get(tuple(bool(_) for _ in signature)) def dialectCheckDbms(injection): """ Keyword-free back-end DBMS heuristic via operator-dialect differentials, evaluated through the given (boolean-capable) injection. Complements heuristicCheckDbms() - which is skipped when the WAF/IPS is dropping requests and otherwise relies on SELECT/quote payloads - because every probe - here is built from operator semantics alone. Returns the DBMS name or None; an ambiguous or - WAF-blocked channel yields None, leaving the scan unchanged. + here is built from operator semantics alone. Returns the DBMS name or None; an ambiguous, + WAF-blocked or false-positive channel yields None, leaving the scan unchanged. """ retVal = None @@ -114,9 +126,12 @@ def dialectCheckDbms(injection): kb.injection = injection try: - # channel sanity: a tautology must read TRUE and a contradiction FALSE, otherwise the - # boolean oracle is unreliable and the all-false signature (Oracle-like) would be meaningless - if checkBooleanExpression("2=2") and not checkBooleanExpression("2=3"): + # Trustworthiness gate: a real boolean oracle reads a tautology TRUE, a contradiction FALSE, + # and a syntactically-invalid canary FALSE (the appended clause is a parse error -> the query + # fails). A false-positive / noise channel reads them all alike - the canary as TRUE - which + # is proof the oracle is trash, so classification is skipped (a true negative) instead of + # emitting a bogus DBMS from a meaningless signature. + if checkBooleanExpression("2=2") and not checkBooleanExpression("2=3") and not checkBooleanExpression(DIALECT_CANARY): signature = tuple(bool(checkBooleanExpression(expr)) for _, expr in DIALECT_PROBES) retVal = _classify(signature) finally: diff --git a/tests/test_dialectdbms.py b/tests/test_dialectdbms.py index 5dc28ac98d..040d80b1a6 100644 --- a/tests/test_dialectdbms.py +++ b/tests/test_dialectdbms.py @@ -4,13 +4,13 @@ Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) See the file 'LICENSE' for copying permission -Operator-dialect DBMS heuristic (lib/utils/dialect.py). These lock in the empirical truth -table: the (xor, intdiv, pgcast, bitor) operator signatures measured across 11 live engines -on an OWASP-CRS test platform, asserting that _classify() maps each to the expected back-end -DBMS - and, just as importantly, that the engines whose signatures collide or are ambiguous -map to None (no prior), so the heuristic never wrong-foots detection. The end-to-end behaviour -(the probes producing these signatures through a real boolean injection) is exercised against -the live platform, not here. +Operator-dialect DBMS heuristic (lib/utils/dialect.py). These lock in the empirical truth table: +the full 5-probe (2^0=2, 2^3=8, 5/2=2, 2|0=2, 1<<2=4) operator signatures measured across the live +SQL engines on an OWASP-CRS test platform, asserting _classify() maps each EXACT signature to the +expected back-end DBMS via its whitelist - and, just as importantly, that anything else (an +unmeasured engine, an ambiguous signature, or a physically-impossible / noise signature) maps to +None, so the heuristic never wrong-foots detection. The end-to-end behaviour (the probes producing +these signatures through a real boolean injection) is exercised against the live platform, not here. """ import os @@ -26,78 +26,80 @@ from lib.core.enums import DBMS from lib.utils.dialect import _classify from lib.utils.dialect import dialectCheckDbms +from lib.utils.dialect import DIALECT_CANARY -# measured 2026-06 across the sqli-platform (boolean form "id=2 AND ", anchor value 2); -# base signature = (2^0=2, 2^3=8, 5/2=2, 2|0=2). The 5th probe (1<<2=4, bit-shift) is the MonetDB-vs- -# SQL Server disambiguator and is asserted separately (SHIFT_SENSITIVE); for every other engine the -# shift flag does NOT change the classification, which the test proves by trying it both ways. +# Full 5-probe signature (2^0=2, 2^3=8, 5/2=2, 2|0=2, 1<<2=4) measured live -> expected DBMS. +# Every bit is significant now (whitelist): e.g. MySQL/PostgreSQL/... all have a working '<<', so +# shift=True is part of their signature; a one-bit-off variant is simply not a known fingerprint. MEASURED = { - "mysql": ((True, False, False, True), DBMS.MYSQL), - "mysql5": ((True, False, False, True), DBMS.MYSQL), - "tidb": ((True, False, False, True), DBMS.MYSQL), # MySQL wire-compatible - "postgres": ((False, True, True, True), DBMS.PGSQL), - "cockroach": ((False, True, False, True), DBMS.PGSQL), # pgwire (exponent '^', decimal division) - "cratedb": ((False, True, True, True), DBMS.PGSQL), # pgwire family - "sqlite": ((False, False, True, True), DBMS.SQLITE), + "mysql": ((True, False, False, True, True), DBMS.MYSQL), + "mysql5": ((True, False, False, True, True), DBMS.MYSQL), + "tidb": ((True, False, False, True, True), DBMS.MYSQL), # MySQL wire-compatible + "postgres": ((False, True, True, True, True), DBMS.PGSQL), + "cockroach": ((False, True, False, True, True), DBMS.PGSQL), # pgwire (exponent '^', decimal division, has '<<') + "cratedb": ((False, True, True, True, False), DBMS.PGSQL), # pgwire family (no '<<') + "mssql": ((True, False, True, True, False), DBMS.MSSQL), # '^' XOR, integer division, NO bit-shift + "monetdb": ((True, False, True, True, True), DBMS.MONETDB), # shares MSSQL base but HAS '<<' + "sqlite": ((False, False, True, True, True), DBMS.SQLITE), # not distinctive enough -> deliberately no prior (operators alone can't safely separate these) - "firebird": ((False, False, True, False), None), - "hsqldb": ((False, False, True, False), None), # collides with firebird/derby/h2 - "derby": ((False, False, True, False), None), - "h2": ((False, False, True, False), None), - "trino": ((False, False, True, False), None), - "iris": ((False, False, False, False), None), # all-error, like Oracle/broken channel - "clickhouse": ((False, False, False, False), None), # all-error, like Oracle/broken channel -} - -# engines whose full 5-probe signature (incl. 1<<2=4) is needed because they share base-4 (xor,intdiv) -# and only the bit-shift probe separates them: SQL Server has no shift operator, MonetDB does. -SHIFT_SENSITIVE = { - "mssql": ((True, False, True, True, False), DBMS.MSSQL), - "monetdb": ((True, False, True, True, True), DBMS.MONETDB), + "firebird": ((False, False, True, False, False), None), + "hsqldb": ((False, False, True, False, False), None), # collides with firebird/derby/h2/trino + "derby": ((False, False, True, False, False), None), + "h2": ((False, False, True, False, False), None), + "trino": ((False, False, True, False, False), None), + "iris": ((False, False, False, False, False), None), # all-error, like Oracle/broken channel + "clickhouse": ((False, False, False, False, False), None), # all-error, like Oracle/broken channel } class TestDialectClassification(unittest.TestCase): - def test_shift_sensitive_engines_split_correctly(self): - # MonetDB shared MSSQL's (xor, intdiv) signature exactly (a false positive before the shift - # probe); 1<<2=4 (MonetDB only) now separates them. - for engine, (signature, expected) in SHIFT_SENSITIVE.items(): + def test_measured_engines_map_as_expected(self): + # each engine's exact measured 5-probe signature maps to its expected DBMS (or None) + for engine, (signature, expected) in MEASURED.items(): self.assertEqual(_classify(signature), expected, "engine %r misclassified" % engine) - def test_measured_engines_map_as_expected(self): - # for non-shift-sensitive engines the shift flag is irrelevant: assert BOTH values map to the - # expected DBMS (proves the new probe never perturbs the existing classifications). - for engine, (base, expected) in MEASURED.items(): - for shift in (False, True): - self.assertEqual(_classify(base + (shift,)), expected, "engine %r misclassified (shift=%s)" % (engine, shift)) - - def test_no_false_positive_across_measured_set(self): - # non-collision property: every measured engine maps to EXACTLY its expected DBMS (or None), - # never to some other back-end. The shift flag is irrelevant for these (non-shift-sensitive) - # engines, so assert it both ways. - for engine, (base, expected) in MEASURED.items(): - for shift in (False, True): - result = _classify(base + (shift,)) - self.assertEqual(result, expected, "engine %r misclassified (shift=%s): got %r, expected %r" % (engine, shift, result, expected)) - # the only non-None DBMS priors the measured set can yield (sanity on the mapping itself) - produced = set(expected for _, expected in MEASURED.values() if expected is not None) - self.assertEqual(produced, {DBMS.MYSQL, DBMS.PGSQL, DBMS.SQLITE}) + def test_shift_splits_monetdb_from_mssql(self): + # MonetDB shares MSSQL's (xor, intdiv) base exactly (a false positive before the shift probe); + # 1<<2=4 (MonetDB has it, SQL Server never does) is the sole separator. + self.assertEqual(_classify((True, False, True, True, False)), DBMS.MSSQL) + self.assertEqual(_classify((True, False, True, True, True)), DBMS.MONETDB) + + def test_whitelist_is_exact_no_false_positive(self): + # only the measured classifying signatures may yield a DBMS; everything else -> None. + classifying = set(sig for sig, exp in MEASURED.values() if exp is not None) + produced = set(exp for _, exp in MEASURED.values() if exp is not None) + self.assertEqual(produced, {DBMS.MYSQL, DBMS.PGSQL, DBMS.MSSQL, DBMS.MONETDB, DBMS.SQLITE}) + # exhaustively sweep all 32 signatures: a non-None result is allowed ONLY for a measured one + for bits in range(32): + sig = tuple(bool(bits & (1 << i)) for i in range(5)) + result = _classify(sig) + if sig not in classifying: + self.assertIsNone(result, "unmeasured signature %r wrongly mapped to %r" % (sig, result)) + + def test_all_true_noise_is_rejected(self): + # a channel that reads EVERY probe true (a static/reflected page, or a WAF/false-positive + # oracle) produces the all-true signature - physically impossible ('^' cannot be XOR and + # exponentiation at once). It must NOT be guessed (previously it mis-read as PostgreSQL). + self.assertIsNone(_classify((True, True, True, True, True))) def test_all_error_signature_yields_no_prior(self): - # an all-error signature (Oracle, ClickHouse, IRIS, or simply a WAF-blocked channel) is not - # distinctive enough - it must NOT be guessed as any DBMS + # an all-error signature (Oracle, ClickHouse, IRIS, or a WAF-blocked channel) is not + # distinctive - it must NOT be guessed as any DBMS self.assertIsNone(_classify((False, False, False, False, False))) self.assertIsNone(_classify((False, False, False, False, True))) - def test_pgpow_dominates_as_postgres_marker(self): - # exponentiation '^' is a positive PostgreSQL-family marker regardless of division flavour - self.assertEqual(_classify((False, True, True, True, False)), DBMS.PGSQL) - self.assertEqual(_classify((False, True, False, True, False)), DBMS.PGSQL) + def test_pgpow_alone_is_not_enough(self): + # exponentiation '^' is a PostgreSQL marker, but pgpow ALONE no longer classifies: the full + # signature must match a measured PostgreSQL fingerprint (this is what stops the all-true noise + # from riding the old 'pgpow dominates' rule into a bogus PostgreSQL claim). + self.assertEqual(_classify((False, True, True, True, True)), DBMS.PGSQL) # real PostgreSQL + self.assertIsNone(_classify((True, True, False, False, False))) # pgpow set, but not a real signature class TestDialectCheckDbmsGuard(unittest.TestCase): - """dialectCheckDbms() end-to-end with a mocked boolean oracle: correct DBMS on a good - channel, and None (no prior) whenever the channel is unreliable - the safety contract.""" + """dialectCheckDbms() end-to-end with a mocked boolean oracle: correct DBMS on a good channel, + and None (no prior) whenever the channel is unreliable - the safety contract, including the + canary that turns a trashy false-positive channel into a true negative.""" def _run(self, truth): # truth: {expression: bool} simulating checkBooleanExpression through a confirmed injection @@ -111,11 +113,13 @@ def _run(self, truth): kb.injection = saved def test_identifies_mysql_on_good_channel(self): - truth = {"2=2": True, "2=3": False, "2^0=2": True, "2^3=8": False, "5/2=2": False, "2|0=2": True} + truth = {"2=2": True, "2=3": False, DIALECT_CANARY: False, + "2^0=2": True, "2^3=8": False, "5/2=2": False, "2|0=2": True, "1<<2=4": True} self.assertEqual(self._run(truth), DBMS.MYSQL) def test_identifies_postgres_on_good_channel(self): - truth = {"2=2": True, "2=3": False, "2^0=2": False, "2^3=8": True, "5/2=2": True, "2|0=2": True} + truth = {"2=2": True, "2=3": False, DIALECT_CANARY: False, + "2^0=2": False, "2^3=8": True, "5/2=2": True, "2|0=2": True, "1<<2=4": True} self.assertEqual(self._run(truth), DBMS.PGSQL) def test_none_on_blocked_channel(self): @@ -124,7 +128,16 @@ def test_none_on_blocked_channel(self): def test_none_on_static_channel(self): # a static page reads everything True, so the contradiction 2=3 is True -> sanity fails -> None - self.assertIsNone(self._run({"2=2": True, "2=3": True, "2^0=2": True, "2^3=8": True, "5/2=2": True, "2|0=2": True})) + self.assertIsNone(self._run({"2=2": True, "2=3": True, DIALECT_CANARY: True, + "2^0=2": True, "2^3=8": True, "5/2=2": True, "2|0=2": True, "1<<2=4": True})) + + def test_none_when_canary_reads_true(self): + # THE canary contract: a channel can look like a clean oracle (2=2 true, 2=3 false) and even + # yield a DBMS-shaped signature, but if the syntactically-invalid canary also reads TRUE the + # channel accepts garbage -> it is a false positive -> return None (true negative), never a DBMS. + truth = {"2=2": True, "2=3": False, DIALECT_CANARY: True, + "2^0=2": True, "2^3=8": False, "5/2=2": False, "2|0=2": True, "1<<2=4": True} # would be MySQL + self.assertIsNone(self._run(truth)) if __name__ == "__main__":