OpenStack library utils
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

567 lines
20KB

  1. # Copyright 2011 OpenStack Foundation.
  2. # All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  5. # not use this file except in compliance with the License. You may obtain
  6. # a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13. # License for the specific language governing permissions and limitations
  14. # under the License.
  15. """
  16. System-level utilities and helper functions.
  17. """
  18. import collections
  19. import math
  20. import re
  21. import unicodedata
  22. import pyparsing as pp
  23. import six
  24. from six.moves import urllib
  25. from oslo_utils._i18n import _
  26. from oslo_utils import encodeutils
  27. UNIT_PREFIX_EXPONENT = {
  28. 'k': 1,
  29. 'K': 1,
  30. 'Ki': 1,
  31. 'M': 2,
  32. 'Mi': 2,
  33. 'G': 3,
  34. 'Gi': 3,
  35. 'T': 4,
  36. 'Ti': 4,
  37. }
  38. UNIT_SYSTEM_INFO = {
  39. 'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')),
  40. 'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')),
  41. 'mixed': (None, re.compile(r'(^[-+]?\d*\.?\d+)([kKMGT]i?)?(b|bit|B)$')),
  42. }
  43. TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
  44. FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
  45. SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
  46. SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
  47. # NOTE(flaper87): The following globals are used by `mask_password`
  48. _SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password',
  49. 'auth_token', 'new_pass', 'auth_password', 'secret_uuid',
  50. 'secret', 'sys_pswd', 'token', 'configdrive',
  51. 'CHAPPASSWORD', 'encrypted_key', 'private_key',
  52. 'encryption_key_id', 'fernetkey', 'sslkey', 'passphrase']
  53. # NOTE(ldbragst): Let's build a list of regex objects using the list of
  54. # _SANITIZE_KEYS we already have. This way, we only have to add the new key
  55. # to the list of _SANITIZE_KEYS and we can generate regular expressions
  56. # for XML and JSON automatically.
  57. _SANITIZE_PATTERNS_2 = {}
  58. _SANITIZE_PATTERNS_1 = {}
  59. # NOTE(amrith): Some regular expressions have only one parameter, some
  60. # have two parameters. Use different lists of patterns here.
  61. _FORMAT_PATTERNS_1 = [r'(%(key)s[0-9]*\s*[=]\s*)[^\s^\'^\"]+']
  62. _FORMAT_PATTERNS_2 = [r'(%(key)s[0-9]*\s*[=]\s*[\"\'])[^\"\']*([\"\'])',
  63. r'(%(key)s[0-9]*\s+[\"\'])[^\"\']*([\"\'])',
  64. r'([-]{2}%(key)s[0-9]*\s+)[^\'^\"^=^\s]+([\s]*)',
  65. r'(<%(key)s[0-9]*>)[^<]*(</%(key)s[0-9]*>)',
  66. r'([\"\']%(key)s[0-9]*[\"\']\s*:\s*[\"\'])[^\"\']*'
  67. '([\"\'])',
  68. r'([\'"][^"\']*%(key)s[0-9]*[\'"]\s*:\s*u?[\'"])[^\"\']*'
  69. '([\'"])',
  70. r'([\'"][^\'"]*%(key)s[0-9]*[\'"]\s*,\s*\'--?[A-z]+'
  71. '\'\s*,\s*u?[\'"])[^\"\']*([\'"])',
  72. r'(%(key)s[0-9]*\s*--?[A-z]+\s*)\S+(\s*)']
  73. # NOTE(dhellmann): Keep a separate list of patterns by key so we only
  74. # need to apply the substitutions for keys we find using a quick "in"
  75. # test.
  76. for key in _SANITIZE_KEYS:
  77. _SANITIZE_PATTERNS_1[key] = []
  78. _SANITIZE_PATTERNS_2[key] = []
  79. for pattern in _FORMAT_PATTERNS_2:
  80. reg_ex = re.compile(pattern % {'key': key}, re.DOTALL | re.IGNORECASE)
  81. _SANITIZE_PATTERNS_2[key].append(reg_ex)
  82. for pattern in _FORMAT_PATTERNS_1:
  83. reg_ex = re.compile(pattern % {'key': key}, re.DOTALL | re.IGNORECASE)
  84. _SANITIZE_PATTERNS_1[key].append(reg_ex)
  85. def int_from_bool_as_string(subject):
  86. """Interpret a string as a boolean and return either 1 or 0.
  87. Any string value in:
  88. ('True', 'true', 'On', 'on', '1')
  89. is interpreted as a boolean True.
  90. Useful for JSON-decoded stuff and config file parsing
  91. """
  92. return int(bool_from_string(subject))
  93. def bool_from_string(subject, strict=False, default=False):
  94. """Interpret a subject as a boolean.
  95. A subject can be a boolean, a string or an integer. Boolean type value
  96. will be returned directly, otherwise the subject will be converted to
  97. a string. A case-insensitive match is performed such that strings
  98. matching 't','true', 'on', 'y', 'yes', or '1' are considered True and,
  99. when `strict=False`, anything else returns the value specified by
  100. 'default'.
  101. Useful for JSON-decoded stuff and config file parsing.
  102. If `strict=True`, unrecognized values, including None, will raise a
  103. ValueError which is useful when parsing values passed in from an API call.
  104. Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
  105. """
  106. if isinstance(subject, bool):
  107. return subject
  108. if not isinstance(subject, six.string_types):
  109. subject = six.text_type(subject)
  110. lowered = subject.strip().lower()
  111. if lowered in TRUE_STRINGS:
  112. return True
  113. elif lowered in FALSE_STRINGS:
  114. return False
  115. elif strict:
  116. acceptable = ', '.join(
  117. "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
  118. msg = _("Unrecognized value '%(val)s', acceptable values are:"
  119. " %(acceptable)s") % {'val': subject,
  120. 'acceptable': acceptable}
  121. raise ValueError(msg)
  122. else:
  123. return default
  124. def is_valid_boolstr(value):
  125. """Check if the provided string is a valid bool string or not.
  126. :param value: value to verify
  127. :type value: string
  128. :returns: true if value is boolean string, false otherwise
  129. .. versionadded:: 3.17
  130. """
  131. boolstrs = TRUE_STRINGS + FALSE_STRINGS
  132. return str(value).lower() in boolstrs
  133. def string_to_bytes(text, unit_system='IEC', return_int=False):
  134. """Converts a string into an float representation of bytes.
  135. The units supported for IEC / mixed::
  136. Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
  137. KB, KiB, MB, MiB, GB, GiB, TB, TiB
  138. The units supported for SI ::
  139. kb(it), Mb(it), Gb(it), Tb(it)
  140. kB, MB, GB, TB
  141. SI units are interpreted as power-of-ten (e.g. 1kb = 1000b). Note
  142. that the SI unit system does not support capital letter 'K'
  143. IEC units are interpreted as power-of-two (e.g. 1MiB = 1MB =
  144. 1024b)
  145. Mixed units interpret the "i" to mean IEC, and no "i" to mean SI
  146. (e.g. 1kb = 1000b, 1kib == 1024b). Additionaly, mixed units
  147. interpret 'K' as power-of-ten. This mode is not particuarly
  148. useful for new code, but can help with compatability for parsers
  149. such as GNU parted.
  150. :param text: String input for bytes size conversion.
  151. :param unit_system: Unit system for byte size conversion.
  152. :param return_int: If True, returns integer representation of text
  153. in bytes. (default: decimal)
  154. :returns: Numerical representation of text in bytes.
  155. :raises ValueError: If text has an invalid value.
  156. """
  157. try:
  158. base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
  159. except KeyError:
  160. msg = _('Invalid unit system: "%s"') % unit_system
  161. raise ValueError(msg)
  162. match = reg_ex.match(text)
  163. if match:
  164. magnitude = float(match.group(1))
  165. unit_prefix = match.group(2)
  166. if match.group(3) in ['b', 'bit']:
  167. magnitude /= 8
  168. # In the mixed matcher, IEC units (with a trailing 'i') are
  169. # interpreted as power-of-two, others as power-of-ten
  170. if unit_system == 'mixed':
  171. if unit_prefix and not unit_prefix.endswith('i'):
  172. # For maximum compatability in mixed mode, we understand
  173. # "K" (which is not strict SI) as "k"
  174. if unit_prefix.startswith == 'K':
  175. unit_prefix = 'k'
  176. base = 1000
  177. else:
  178. base = 1024
  179. else:
  180. msg = _('Invalid string format: %s') % text
  181. raise ValueError(msg)
  182. if not unit_prefix:
  183. res = magnitude
  184. else:
  185. res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
  186. if return_int:
  187. return int(math.ceil(res))
  188. return res
  189. def to_slug(value, incoming=None, errors="strict"):
  190. """Normalize string.
  191. Convert to lowercase, remove non-word characters, and convert spaces
  192. to hyphens.
  193. Inspired by Django's `slugify` filter.
  194. :param value: Text to slugify
  195. :param incoming: Text's current encoding
  196. :param errors: Errors handling policy. See here for valid
  197. values http://docs.python.org/2/library/codecs.html
  198. :returns: slugified unicode representation of `value`
  199. :raises TypeError: If text is not an instance of str
  200. """
  201. value = encodeutils.safe_decode(value, incoming, errors)
  202. # NOTE(aababilov): no need to use safe_(encode|decode) here:
  203. # encodings are always "ascii", error handling is always "ignore"
  204. # and types are always known (first: unicode; second: str)
  205. value = unicodedata.normalize("NFKD", value).encode(
  206. "ascii", "ignore").decode("ascii")
  207. value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
  208. return SLUGIFY_HYPHENATE_RE.sub("-", value)
  209. # NOTE(dhellmann): Before submitting a patch to add a new argument to
  210. # this function to allow the caller to pass in "extra" or "additional"
  211. # or "replacement" patterns to be masked out, please note that we have
  212. # discussed that feature many times and always rejected it based on
  213. # the desire to have Oslo functions behave consistently across all
  214. # projects and *especially* to have security features work the same
  215. # way no matter where they are used. If every project adopted its own
  216. # set patterns for secret values, it would be very difficult to audit
  217. # the logging to ensure that everything is properly masked. So, please
  218. # either add your pattern to the module-level variables at the top of
  219. # this file or, even better, pick an existing pattern or key to use in
  220. # your application to ensure that the value is masked by this
  221. # function.
  222. def mask_password(message, secret="***"): # nosec
  223. """Replace password with *secret* in message.
  224. :param message: The string which includes security information.
  225. :param secret: value with which to replace passwords.
  226. :returns: The unicode value of message with the password fields masked.
  227. For example:
  228. >>> mask_password("'adminPass' : 'aaaaa'")
  229. "'adminPass' : '***'"
  230. >>> mask_password("'admin_pass' : 'aaaaa'")
  231. "'admin_pass' : '***'"
  232. >>> mask_password('"password" : "aaaaa"')
  233. '"password" : "***"'
  234. >>> mask_password("'original_password' : 'aaaaa'")
  235. "'original_password' : '***'"
  236. >>> mask_password("u'original_password' : u'aaaaa'")
  237. "u'original_password' : u'***'"
  238. .. versionadded:: 0.2
  239. .. versionchanged:: 1.1
  240. Replace also ``'auth_token'``, ``'new_pass'`` and ``'auth_password'``
  241. keys.
  242. .. versionchanged:: 1.1.1
  243. Replace also ``'secret_uuid'`` key.
  244. .. versionchanged:: 1.5
  245. Replace also ``'sys_pswd'`` key.
  246. .. versionchanged:: 2.6
  247. Replace also ``'token'`` key.
  248. .. versionchanged:: 2.7
  249. Replace also ``'secret'`` key.
  250. .. versionchanged:: 3.4
  251. Replace also ``'configdrive'`` key.
  252. .. versionchanged:: 3.8
  253. Replace also ``'CHAPPASSWORD'`` key.
  254. """
  255. try:
  256. message = six.text_type(message)
  257. except UnicodeDecodeError: # nosec
  258. # NOTE(jecarey): Temporary fix to handle cases where message is a
  259. # byte string. A better solution will be provided in Kilo.
  260. pass
  261. substitute1 = r'\g<1>' + secret
  262. substitute2 = r'\g<1>' + secret + r'\g<2>'
  263. # NOTE(ldbragst): Check to see if anything in message contains any key
  264. # specified in _SANITIZE_KEYS, if not then just return the message since
  265. # we don't have to mask any passwords.
  266. for key in _SANITIZE_KEYS:
  267. if key.lower() in message.lower():
  268. for pattern in _SANITIZE_PATTERNS_2[key]:
  269. message = re.sub(pattern, substitute2, message)
  270. for pattern in _SANITIZE_PATTERNS_1[key]:
  271. message = re.sub(pattern, substitute1, message)
  272. return message
  273. def mask_dict_password(dictionary, secret="***"): # nosec
  274. """Replace password with *secret* in a dictionary recursively.
  275. :param dictionary: The dictionary which includes secret information.
  276. :param secret: value with which to replace secret information.
  277. :returns: The dictionary with string substitutions.
  278. A dictionary (which may contain nested dictionaries) contains
  279. information (such as passwords) which should not be revealed, and
  280. this function helps detect and replace those with the 'secret'
  281. provided (or `***` if none is provided).
  282. Substitution is performed in one of three situations:
  283. If the key is something that is considered to be indicative of a
  284. secret, then the corresponding value is replaced with the secret
  285. provided (or `***` if none is provided).
  286. If a value in the dictionary is a string, then it is masked
  287. using the ``mask_password()`` function.
  288. Finally, if a value is a dictionary, this function will
  289. recursively mask that dictionary as well.
  290. For example:
  291. >>> mask_dict_password({'password': 'd81juxmEW_',
  292. >>> 'user': 'admin',
  293. >>> 'home-dir': '/home/admin'},
  294. >>> '???')
  295. {'password': '???', 'user': 'admin', 'home-dir': '/home/admin'}
  296. For example (the value is masked using mask_password())
  297. >>> mask_dict_password({'password': '--password d81juxmEW_',
  298. >>> 'user': 'admin',
  299. >>> 'home-dir': '/home/admin'},
  300. >>> '???')
  301. {'password': '--password ???', 'user': 'admin',
  302. 'home-dir': '/home/admin'}
  303. For example (a nested dictionary is masked):
  304. >>> mask_dict_password({"nested": {'password': 'd81juxmEW_',
  305. >>> 'user': 'admin',
  306. >>> 'home': '/home/admin'}},
  307. >>> '???')
  308. {"nested": {'password': '???', 'user': 'admin', 'home': '/home/admin'}}
  309. .. versionadded:: 3.4
  310. """
  311. if not isinstance(dictionary, collections.Mapping):
  312. raise TypeError("Expected a Mapping, got %s instead."
  313. % type(dictionary))
  314. out = {}
  315. for k, v in dictionary.items():
  316. if isinstance(v, collections.Mapping):
  317. out[k] = mask_dict_password(v, secret=secret)
  318. continue
  319. # NOTE(jlvillal): Check to see if anything in the dictionary 'key'
  320. # contains any key specified in _SANITIZE_KEYS.
  321. k_matched = False
  322. if isinstance(k, six.string_types):
  323. for sani_key in _SANITIZE_KEYS:
  324. if sani_key in k:
  325. out[k] = secret
  326. k_matched = True
  327. break
  328. if not k_matched:
  329. # We did not find a match for the key name in the
  330. # _SANITIZE_KEYS, so we fall through to here
  331. if isinstance(v, six.string_types):
  332. out[k] = mask_password(v, secret=secret)
  333. else:
  334. # Just leave it alone.
  335. out[k] = v
  336. return out
  337. def is_int_like(val):
  338. """Check if a value looks like an integer with base 10.
  339. :param val: Value to verify
  340. :type val: string
  341. :returns: bool
  342. .. versionadded:: 1.1
  343. """
  344. try:
  345. return six.text_type(int(val)) == six.text_type(val)
  346. except (TypeError, ValueError):
  347. return False
  348. def check_string_length(value, name=None, min_length=0, max_length=None):
  349. """Check the length of specified string.
  350. :param value: the value of the string
  351. :param name: the name of the string
  352. :param min_length: the min_length of the string
  353. :param max_length: the max_length of the string
  354. :raises TypeError, ValueError: For any invalid input.
  355. .. versionadded:: 3.7
  356. """
  357. if name is None:
  358. name = value
  359. if not isinstance(value, six.string_types):
  360. msg = _("%s is not a string or unicode") % name
  361. raise TypeError(msg)
  362. length = len(value)
  363. if length < min_length:
  364. msg = _("%(name)s has %(length)s characters, less than "
  365. "%(min_length)s.") % {'name': name, 'length': length,
  366. 'min_length': min_length}
  367. raise ValueError(msg)
  368. if max_length and length > max_length:
  369. msg = _("%(name)s has %(length)s characters, more than "
  370. "%(max_length)s.") % {'name': name, 'length': length,
  371. 'max_length': max_length}
  372. raise ValueError(msg)
  373. def validate_integer(value, name, min_value=None, max_value=None):
  374. """Make sure that value is a valid integer, potentially within range.
  375. :param value: value of the integer
  376. :param name: name of the integer
  377. :param min_value: min_value of the integer
  378. :param max_value: max_value of the integer
  379. :returns: integer
  380. :raises: ValueError if value is an invalid integer
  381. .. versionadded:: 3.33
  382. """
  383. try:
  384. value = int(str(value))
  385. except (ValueError, UnicodeEncodeError):
  386. msg = _('%(value_name)s must be an integer'
  387. ) % {'value_name': name}
  388. raise ValueError(msg)
  389. if min_value is not None and value < min_value:
  390. msg = _('%(value_name)s must be >= %(min_value)d'
  391. ) % {'value_name': name, 'min_value': min_value}
  392. raise ValueError(msg)
  393. if max_value is not None and value > max_value:
  394. msg = _('%(value_name)s must be <= %(max_value)d'
  395. ) % {'value_name': name, 'max_value': max_value}
  396. raise ValueError(msg)
  397. return value
  398. def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False):
  399. """Validate and split the given HTTP request path.
  400. **Examples**::
  401. ['a'] = _split_path('/a')
  402. ['a', None] = _split_path('/a', 1, 2)
  403. ['a', 'c'] = _split_path('/a/c', 1, 2)
  404. ['a', 'c', 'o/r'] = _split_path('/a/c/o/r', 1, 3, True)
  405. :param path: HTTP Request path to be split
  406. :param minsegs: Minimum number of segments to be extracted
  407. :param maxsegs: Maximum number of segments to be extracted
  408. :param rest_with_last: If True, trailing data will be returned as part
  409. of last segment. If False, and there is
  410. trailing data, raises ValueError.
  411. :returns: list of segments with a length of maxsegs (non-existent
  412. segments will return as None)
  413. :raises: ValueError if given an invalid path
  414. .. versionadded:: 3.11
  415. """
  416. if not maxsegs:
  417. maxsegs = minsegs
  418. if minsegs > maxsegs:
  419. raise ValueError(_('minsegs > maxsegs: %(min)d > %(max)d)') %
  420. {'min': minsegs, 'max': maxsegs})
  421. if rest_with_last:
  422. segs = path.split('/', maxsegs)
  423. minsegs += 1
  424. maxsegs += 1
  425. count = len(segs)
  426. if (segs[0] or count < minsegs or count > maxsegs or
  427. '' in segs[1:minsegs]):
  428. raise ValueError(_('Invalid path: %s') % urllib.parse.quote(path))
  429. else:
  430. minsegs += 1
  431. maxsegs += 1
  432. segs = path.split('/', maxsegs)
  433. count = len(segs)
  434. if (segs[0] or count < minsegs or count > maxsegs + 1 or
  435. '' in segs[1:minsegs] or
  436. (count == maxsegs + 1 and segs[maxsegs])):
  437. raise ValueError(_('Invalid path: %s') % urllib.parse.quote(path))
  438. segs = segs[1:maxsegs]
  439. segs.extend([None] * (maxsegs - 1 - len(segs)))
  440. return segs
  441. def split_by_commas(value):
  442. """Split values by commas and quotes according to api-wg
  443. :param value: value to be split
  444. .. versionadded:: 3.17
  445. """
  446. word = (pp.QuotedString(quoteChar='"', escChar='\\') |
  447. pp.Word(pp.printables, excludeChars='",'))
  448. grammar = pp.stringStart + pp.delimitedList(word) + pp.stringEnd
  449. try:
  450. return list(grammar.parseString(value))
  451. except pp.ParseException:
  452. raise ValueError("Invalid value: %s" % value)