---

lexer.py

import string import warnings from json import loads from jmespath.exceptions import LexerError, EmptyExpressionError class Lexer(object): START_IDENTIFIER = set(string.ascii_letters + '_') VALID_IDENTIFIER = set(string.ascii_letters + string.digits + '_') VALID_NUMBER = set(string.digits) WHITESPACE = set(" \t\n\r") SIMPLE_TOKENS = { '.': 'dot', '*': 'star', ']': 'rbracket', ',': 'comma', ':': 'colon', '@': 'current', '(': 'lparen', ')': 'rparen', '{': 'lbrace', '}': 'rbrace', } def tokenize(self, expression): self._initialize_for_expression(expression) while self._current is not None: if self._current in self.SIMPLE_TOKENS: yield {'type': self.SIMPLE_TOKENS[self._current], 'value': self._current, 'start': self._position, 'end': self._position + 1} self._next() elif self._current in self.START_IDENTIFIER: start = self._position buff = self._current while self._next() in self.VALID_IDENTIFIER: buff += self._current yield {'type': 'unquoted_identifier', 'value': buff, 'start': start, 'end': start + len(buff)} elif self._current in self.WHITESPACE: self._next() elif self._current == '[': start = self._position next_char = self._next() if next_char == ']': self._next() yield {'type': 'flatten', 'value': '[]', 'start': start, 'end': start + 2} elif next_char == '?': self._next() yield {'type': 'filter', 'value': '[?', 'start': start, 'end': start + 2} else: yield {'type': 'lbracket', 'value': '[', 'start': start, 'end': start + 1} elif self._current == "'": yield self._consume_raw_string_literal() elif self._current == '|': yield self._match_or_else('|', 'or', 'pipe') elif self._current == '&': yield self._match_or_else('&', 'and', 'expref') elif self._current == '`': yield self._consume_literal() elif self._current in self.VALID_NUMBER: start = self._position buff = self._consume_number() yield {'type': 'number', 'value': int(buff), 'start': start, 'end': start + len(buff)} elif self._current == '-': # Negative number. start = self._position buff = self._consume_number() if len(buff) > 1: yield {'type': 'number', 'value': int(buff), 'start': start, 'end': start + len(buff)} else: raise LexerError(lexer_position=start, lexer_value=buff, message="Unknown token '%s'" % buff) elif self._current == '"': yield self._consume_quoted_identifier() elif self._current == '<': yield self._match_or_else('=', 'lte', 'lt') elif self._current == '>': yield self._match_or_else('=', 'gte', 'gt') elif self._current == '!': yield self._match_or_else('=', 'ne', 'not') elif self._current == '=': if self._next() == '=': yield {'type': 'eq', 'value': '==', 'start': self._position - 1, 'end': self._position} self._next() else: if self._current is None: # If we're at the EOF, we never advanced # the position so we don't need to rewind # it back one location. position = self._position else: position = self._position - 1 raise LexerError( lexer_position=position, lexer_value='=', message="Unknown token '='") else: raise LexerError(lexer_position=self._position, lexer_value=self._current, message="Unknown token %s" % self._current) yield {'type': 'eof', 'value': '', 'start': self._length, 'end': self._length} def _consume_number(self): start = self._position buff = self._current while self._next() in self.VALID_NUMBER: buff += self._current return buff def _initialize_for_expression(self, expression): if not expression: raise EmptyExpressionError() self._position = 0 self._expression = expression self._chars = list(self._expression) self._current = self._chars[self._position] self._length = len(self._expression) def _next(self): if self._position == self._length - 1: self._current = None else: self._position += 1 self._current = self._chars[self._position] return self._current def _consume_until(self, delimiter): # Consume until the delimiter is reached, # allowing for the delimiter to be escaped with "\". start = self._position buff = '' self._next() while self._current != delimiter: if self._current == '\\': buff += '\\' self._next() if self._current is None: # We're at the EOF. raise LexerError(lexer_position=start, lexer_value=self._expression[start:], message="Unclosed %s delimiter" % delimiter) buff += self._current self._next() # Skip the closing delimiter. self._next() return buff def _consume_literal(self): start = self._position lexeme = self._consume_until('`').replace('\\`', '`') try: # Assume it is valid JSON and attempt to parse. parsed_json = loads(lexeme) except ValueError: try: # Invalid JSON values should be converted to quoted # JSON strings during the JEP-12 deprecation period. parsed_json = loads('"%s"' % lexeme.lstrip()) warnings.warn("deprecated string literal syntax", PendingDeprecationWarning) except ValueError: raise LexerError(lexer_position=start, lexer_value=self._expression[start:], message="Bad token %s" % lexeme) token_len = self._position - start return {'type': 'literal', 'value': parsed_json, 'start': start, 'end': token_len} def _consume_quoted_identifier(self): start = self._position lexeme = '"' + self._consume_until('"') + '"' try: token_len = self._position - start return {'type': 'quoted_identifier', 'value': loads(lexeme), 'start': start, 'end': token_len} except ValueError as e: error_message = str(e).split(':')[0] raise LexerError(lexer_position=start, lexer_value=lexeme, message=error_message) def _consume_raw_string_literal(self): start = self._position lexeme = self._consume_until("'").replace("\\'", "'") token_len = self._position - start return {'type': 'literal', 'value': lexeme, 'start': start, 'end': token_len} def _match_or_else(self, expected, match_type, else_type): start = self._position current = self._current next_char = self._next() if next_char == expected: self._next() return {'type': match_type, 'value': current + next_char, 'start': start, 'end': start + 1} return {'type': else_type, 'value': current, 'start': start, 'end': start}

---

docroot: /var/www/469ed2f6-4d4b-4a8a-9f5e-ee4ef04d3bc8/public_html

---

curdir: /usr/lib/python3/dist-packages/jmespath

	.		4096
	..		12288
X	__init__.py		281
	__pycache__		4096
X	ast.py		2130
X	compat.py		381
X	exceptions.py		4128
X	functions.py		12736
X	lexer.py		8574
X	parser.py		19088
X	visitor.py		10834