vendor/twig/twig/src/Lexer.php line 219

Open in your IDE?
  1. <?php
  3. /*
  4. * This file is part of Twig.
  5. *
  6. * (c) Fabien Potencier
  7. * (c) Armin Ronacher
  8. *
  9. * For the full copyright and license information, please view the LICENSE
  10. * file that was distributed with this source code.
  11. */
  13. namespace Twig;
  15. use Twig\Error\SyntaxError;
  17. /**
  18. * Lexes a template string.
  19. *
  20. * @author Fabien Potencier <fabien@symfony.com>
  21. */
  22. class Lexer
  23. {
  24. private $isInitialized = false;
  26. private $tokens;
  27. private $code;
  28. private $cursor;
  29. private $lineno;
  30. private $end;
  31. private $state;
  32. private $states;
  33. private $brackets;
  34. private $env;
  35. private $source;
  36. private $options;
  37. private $regexes;
  38. private $position;
  39. private $positions;
  40. private $currentVarBlockLine;
  42. public const STATE_DATA = 0;
  43. public const STATE_BLOCK = 1;
  44. public const STATE_VAR = 2;
  45. public const STATE_STRING = 3;
  46. public const STATE_INTERPOLATION = 4;
  48. public const REGEX_NAME = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A';
  49. public const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?([Ee][\+\-][0-9]+)?/A';
  50. public const REGEX_STRING = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
  51. public const REGEX_DQ_STRING_DELIM = '/"/A';
  52. public const REGEX_DQ_STRING_PART = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As';
  53. public const PUNCTUATION = '()[]{}?:.,|';
  55. public function __construct(Environment $env, array $options = [])
  56. {
  57. $this->env = $env;
  59. $this->options = array_merge([
  60. 'tag_comment' => ['{#', '#}'],
  61. 'tag_block' => ['{%', '%}'],
  62. 'tag_variable' => ['{{', '}}'],
  63. 'whitespace_trim' => '-',
  64. 'whitespace_line_trim' => '~',
  65. 'whitespace_line_chars' => ' \t\0\x0B',
  66. 'interpolation' => ['#{', '}'],
  67. ], $options);
  68. }
  70. private function initialize()
  71. {
  72. if ($this->isInitialized) {
  73. return;
  74. }
  76. $this->isInitialized = true;
  78. // when PHP 7.3 is the min version, we will be able to remove the '#' part in preg_quote as it's part of the default
  79. $this->regexes = [
  80. // }}
  81. 'lex_var' => '{
  82. \s*
  83. (?:'.
  84. preg_quote($this->options['whitespace_trim'].$this->options['tag_variable'][1], '#').'\s*'. // -}}\s*
  85. '|'.
  86. preg_quote($this->options['whitespace_line_trim'].$this->options['tag_variable'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~}}[ \t\0\x0B]*
  87. '|'.
  88. preg_quote($this->options['tag_variable'][1], '#'). // }}
  89. ')
  90. }Ax',
  92. // %}
  93. 'lex_block' => '{
  94. \s*
  95. (?:'.
  96. preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*\n?'. // -%}\s*\n?
  97. '|'.
  98. preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
  99. '|'.
  100. preg_quote($this->options['tag_block'][1], '#').'\n?'. // %}\n?
  101. ')
  102. }Ax',
  104. // {% endverbatim %}
  105. 'lex_raw_data' => '{'.
  106. preg_quote($this->options['tag_block'][0], '#'). // {%
  107. '('.
  108. $this->options['whitespace_trim']. // -
  109. '|'.
  110. $this->options['whitespace_line_trim']. // ~
  111. ')?\s*endverbatim\s*'.
  112. '(?:'.
  113. preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'. // -%}
  114. '|'.
  115. preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
  116. '|'.
  117. preg_quote($this->options['tag_block'][1], '#'). // %}
  118. ')
  119. }sx',
  121. 'operator' => $this->getOperatorRegex(),
  123. // #}
  124. 'lex_comment' => '{
  125. (?:'.
  126. preg_quote($this->options['whitespace_trim'].$this->options['tag_comment'][1], '#').'\s*\n?'. // -#}\s*\n?
  127. '|'.
  128. preg_quote($this->options['whitespace_line_trim'].$this->options['tag_comment'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~#}[ \t\0\x0B]*
  129. '|'.
  130. preg_quote($this->options['tag_comment'][1], '#').'\n?'. // #}\n?
  131. ')
  132. }sx',
  134. // verbatim %}
  135. 'lex_block_raw' => '{
  136. \s*verbatim\s*
  137. (?:'.
  138. preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'. // -%}\s*
  139. '|'.
  140. preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
  141. '|'.
  142. preg_quote($this->options['tag_block'][1], '#'). // %}
  143. ')
  144. }Asx',
  146. 'lex_block_line' => '{\s*line\s+(\d+)\s*'.preg_quote($this->options['tag_block'][1], '#').'}As',
  148. // {{ or {% or {#
  149. 'lex_tokens_start' => '{
  150. ('.
  151. preg_quote($this->options['tag_variable'][0], '#'). // {{
  152. '|'.
  153. preg_quote($this->options['tag_block'][0], '#'). // {%
  154. '|'.
  155. preg_quote($this->options['tag_comment'][0], '#'). // {#
  156. ')('.
  157. preg_quote($this->options['whitespace_trim'], '#'). // -
  158. '|'.
  159. preg_quote($this->options['whitespace_line_trim'], '#'). // ~
  160. ')?
  161. }sx',
  162. 'interpolation_start' => '{'.preg_quote($this->options['interpolation'][0], '#').'\s*}A',
  163. 'interpolation_end' => '{\s*'.preg_quote($this->options['interpolation'][1], '#').'}A',
  164. ];
  165. }
  167. public function tokenize(Source $source)
  168. {
  169. $this->initialize();
  171. $this->source = $source;
  172. $this->code = str_replace(["\r\n", "\r"], "\n", $source->getCode());
  173. $this->cursor = 0;
  174. $this->lineno = 1;
  175. $this->end = \strlen($this->code);
  176. $this->tokens = [];
  177. $this->state = self::STATE_DATA;
  178. $this->states = [];
  179. $this->brackets = [];
  180. $this->position = -1;
  182. // find all token starts in one go
  183. preg_match_all($this->regexes['lex_tokens_start'], $this->code, $matches, \PREG_OFFSET_CAPTURE);
  184. $this->positions = $matches;
  186. while ($this->cursor < $this->end) {
  187. // dispatch to the lexing functions depending
  188. // on the current state
  189. switch ($this->state) {
  190. case self::STATE_DATA:
  191. $this->lexData();
  192. break;
  194. case self::STATE_BLOCK:
  195. $this->lexBlock();
  196. break;
  198. case self::STATE_VAR:
  199. $this->lexVar();
  200. break;
  202. case self::STATE_STRING:
  203. $this->lexString();
  204. break;
  206. case self::STATE_INTERPOLATION:
  207. $this->lexInterpolation();
  208. break;
  209. }
  210. }
  212. $this->pushToken(/* Token::EOF_TYPE */ -1);
  214. if (!empty($this->brackets)) {
  215. list($expect, $lineno) = array_pop($this->brackets);
  216. throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
  217. }
  219. return new TokenStream($this->tokens, $this->source);
  220. }
  222. private function lexData()
  223. {
  224. // if no matches are left we return the rest of the template as simple text token
  225. if ($this->position == \count($this->positions[0]) - 1) {
  226. $this->pushToken(/* Token::TEXT_TYPE */ 0, substr($this->code, $this->cursor));
  227. $this->cursor = $this->end;
  229. return;
  230. }
  232. // Find the first token after the current cursor
  233. $position = $this->positions[0][++$this->position];
  234. while ($position[1] < $this->cursor) {
  235. if ($this->position == \count($this->positions[0]) - 1) {
  236. return;
  237. }
  238. $position = $this->positions[0][++$this->position];
  239. }
  241. // push the template text first
  242. $text = $textContent = substr($this->code, $this->cursor, $position[1] - $this->cursor);
  244. // trim?
  245. if (isset($this->positions[2][$this->position][0])) {
  246. if ($this->options['whitespace_trim'] === $this->positions[2][$this->position][0]) {
  247. // whitespace_trim detected ({%-, {{- or {#-)
  248. $text = rtrim($text);
  249. } elseif ($this->options['whitespace_line_trim'] === $this->positions[2][$this->position][0]) {
  250. // whitespace_line_trim detected ({%~, {{~ or {#~)
  251. // don't trim \r and \n
  252. $text = rtrim($text, " \t\0\x0B");
  253. }
  254. }
  255. $this->pushToken(/* Token::TEXT_TYPE */ 0, $text);
  256. $this->moveCursor($textContent.$position[0]);
  258. switch ($this->positions[1][$this->position][0]) {
  259. case $this->options['tag_comment'][0]:
  260. $this->lexComment();
  261. break;
  263. case $this->options['tag_block'][0]:
  264. // raw data?
  265. if (preg_match($this->regexes['lex_block_raw'], $this->code, $match, 0, $this->cursor)) {
  266. $this->moveCursor($match[0]);
  267. $this->lexRawData();
  268. // {% line \d+ %}
  269. } elseif (preg_match($this->regexes['lex_block_line'], $this->code, $match, 0, $this->cursor)) {
  270. $this->moveCursor($match[0]);
  271. $this->lineno = (int) $match[1];
  272. } else {
  273. $this->pushToken(/* Token::BLOCK_START_TYPE */ 1);
  274. $this->pushState(self::STATE_BLOCK);
  275. $this->currentVarBlockLine = $this->lineno;
  276. }
  277. break;
  279. case $this->options['tag_variable'][0]:
  280. $this->pushToken(/* Token::VAR_START_TYPE */ 2);
  281. $this->pushState(self::STATE_VAR);
  282. $this->currentVarBlockLine = $this->lineno;
  283. break;
  284. }
  285. }
  287. private function lexBlock()
  288. {
  289. if (empty($this->brackets) && preg_match($this->regexes['lex_block'], $this->code, $match, 0, $this->cursor)) {
  290. $this->pushToken(/* Token::BLOCK_END_TYPE */ 3);
  291. $this->moveCursor($match[0]);
  292. $this->popState();
  293. } else {
  294. $this->lexExpression();
  295. }
  296. }
  298. private function lexVar()
  299. {
  300. if (empty($this->brackets) && preg_match($this->regexes['lex_var'], $this->code, $match, 0, $this->cursor)) {
  301. $this->pushToken(/* Token::VAR_END_TYPE */ 4);
  302. $this->moveCursor($match[0]);
  303. $this->popState();
  304. } else {
  305. $this->lexExpression();
  306. }
  307. }
  309. private function lexExpression()
  310. {
  311. // whitespace
  312. if (preg_match('/\s+/A', $this->code, $match, 0, $this->cursor)) {
  313. $this->moveCursor($match[0]);
  315. if ($this->cursor >= $this->end) {
  316. throw new SyntaxError(sprintf('Unclosed "%s".', self::STATE_BLOCK === $this->state ? 'block' : 'variable'), $this->currentVarBlockLine, $this->source);
  317. }
  318. }
  320. // arrow function
  321. if ('=' === $this->code[$this->cursor] && '>' === $this->code[$this->cursor + 1]) {
  322. $this->pushToken(Token::ARROW_TYPE, '=>');
  323. $this->moveCursor('=>');
  324. }
  325. // operators
  326. elseif (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) {
  327. $this->pushToken(/* Token::OPERATOR_TYPE */ 8, preg_replace('/\s+/', ' ', $match[0]));
  328. $this->moveCursor($match[0]);
  329. }
  330. // names
  331. elseif (preg_match(self::REGEX_NAME, $this->code, $match, 0, $this->cursor)) {
  332. $this->pushToken(/* Token::NAME_TYPE */ 5, $match[0]);
  333. $this->moveCursor($match[0]);
  334. }
  335. // numbers
  336. elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, 0, $this->cursor)) {
  337. $number = (float) $match[0]; // floats
  338. if (ctype_digit($match[0]) && $number <= \PHP_INT_MAX) {
  339. $number = (int) $match[0]; // integers lower than the maximum
  340. }
  341. $this->pushToken(/* Token::NUMBER_TYPE */ 6, $number);
  342. $this->moveCursor($match[0]);
  343. }
  344. // punctuation
  345. elseif (false !== strpos(self::PUNCTUATION, $this->code[$this->cursor])) {
  346. // opening bracket
  347. if (false !== strpos('([{', $this->code[$this->cursor])) {
  348. $this->brackets[] = [$this->code[$this->cursor], $this->lineno];
  349. }
  350. // closing bracket
  351. elseif (false !== strpos(')]}', $this->code[$this->cursor])) {
  352. if (empty($this->brackets)) {
  353. throw new SyntaxError(sprintf('Unexpected "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
  354. }
  356. list($expect, $lineno) = array_pop($this->brackets);
  357. if ($this->code[$this->cursor] != strtr($expect, '([{', ')]}')) {
  358. throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
  359. }
  360. }
  362. $this->pushToken(/* Token::PUNCTUATION_TYPE */ 9, $this->code[$this->cursor]);
  363. ++$this->cursor;
  364. }
  365. // strings
  366. elseif (preg_match(self::REGEX_STRING, $this->code, $match, 0, $this->cursor)) {
  367. $this->pushToken(/* Token::STRING_TYPE */ 7, stripcslashes(substr($match[0], 1, -1)));
  368. $this->moveCursor($match[0]);
  369. }
  370. // opening double quoted string
  371. elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
  372. $this->brackets[] = ['"', $this->lineno];
  373. $this->pushState(self::STATE_STRING);
  374. $this->moveCursor($match[0]);
  375. }
  376. // unlexable
  377. else {
  378. throw new SyntaxError(sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
  379. }
  380. }
  382. private function lexRawData()
  383. {
  384. if (!preg_match($this->regexes['lex_raw_data'], $this->code, $match, \PREG_OFFSET_CAPTURE, $this->cursor)) {
  385. throw new SyntaxError('Unexpected end of file: Unclosed "verbatim" block.', $this->lineno, $this->source);
  386. }
  388. $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor);
  389. $this->moveCursor($text.$match[0][0]);
  391. // trim?
  392. if (isset($match[1][0])) {
  393. if ($this->options['whitespace_trim'] === $match[1][0]) {
  394. // whitespace_trim detected ({%-, {{- or {#-)
  395. $text = rtrim($text);
  396. } else {
  397. // whitespace_line_trim detected ({%~, {{~ or {#~)
  398. // don't trim \r and \n
  399. $text = rtrim($text, " \t\0\x0B");
  400. }
  401. }
  403. $this->pushToken(/* Token::TEXT_TYPE */ 0, $text);
  404. }
  406. private function lexComment()
  407. {
  408. if (!preg_match($this->regexes['lex_comment'], $this->code, $match, \PREG_OFFSET_CAPTURE, $this->cursor)) {
  409. throw new SyntaxError('Unclosed comment.', $this->lineno, $this->source);
  410. }
  412. $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
  413. }
  415. private function lexString()
  416. {
  417. if (preg_match($this->regexes['interpolation_start'], $this->code, $match, 0, $this->cursor)) {
  418. $this->brackets[] = [$this->options['interpolation'][0], $this->lineno];
  419. $this->pushToken(/* Token::INTERPOLATION_START_TYPE */ 10);
  420. $this->moveCursor($match[0]);
  421. $this->pushState(self::STATE_INTERPOLATION);
  422. } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor) && \strlen($match[0]) > 0) {
  423. $this->pushToken(/* Token::STRING_TYPE */ 7, stripcslashes($match[0]));
  424. $this->moveCursor($match[0]);
  425. } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
  426. list($expect, $lineno) = array_pop($this->brackets);
  427. if ('"' != $this->code[$this->cursor]) {
  428. throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
  429. }
  431. $this->popState();
  432. ++$this->cursor;
  433. } else {
  434. // unlexable
  435. throw new SyntaxError(sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
  436. }
  437. }
  439. private function lexInterpolation()
  440. {
  441. $bracket = end($this->brackets);
  442. if ($this->options['interpolation'][0] === $bracket[0] && preg_match($this->regexes['interpolation_end'], $this->code, $match, 0, $this->cursor)) {
  443. array_pop($this->brackets);
  444. $this->pushToken(/* Token::INTERPOLATION_END_TYPE */ 11);
  445. $this->moveCursor($match[0]);
  446. $this->popState();
  447. } else {
  448. $this->lexExpression();
  449. }
  450. }
  452. private function pushToken($type, $value = '')
  453. {
  454. // do not push empty text tokens
  455. if (/* Token::TEXT_TYPE */ 0 === $type && '' === $value) {
  456. return;
  457. }
  459. $this->tokens[] = new Token($type, $value, $this->lineno);
  460. }
  462. private function moveCursor($text)
  463. {
  464. $this->cursor += \strlen($text);
  465. $this->lineno += substr_count($text, "\n");
  466. }
  468. private function getOperatorRegex()
  469. {
  470. $operators = array_merge(
  471. ['='],
  472. array_keys($this->env->getUnaryOperators()),
  473. array_keys($this->env->getBinaryOperators())
  474. );
  476. $operators = array_combine($operators, array_map('strlen', $operators));
  477. arsort($operators);
  479. $regex = [];
  480. foreach ($operators as $operator => $length) {
  481. // an operator that ends with a character must be followed by
  482. // a whitespace, a parenthesis, an opening map [ or sequence {
  483. $r = preg_quote($operator, '/');
  484. if (ctype_alpha($operator[$length - 1])) {
  485. $r .= '(?=[\s()\[{])';
  486. }
  488. // an operator that begins with a character must not have a dot or pipe before
  489. if (ctype_alpha($operator[0])) {
  490. $r = '(?<![\.\|])'.$r;
  491. }
  493. // an operator with a space can be any amount of whitespaces
  494. $r = preg_replace('/\s+/', '\s+', $r);
  496. $regex[] = $r;
  497. }
  499. return '/'.implode('|', $regex).'/A';
  500. }
  502. private function pushState($state)
  503. {
  504. $this->states[] = $this->state;
  505. $this->state = $state;
  506. }
  508. private function popState()
  509. {
  510. if (0 === \count($this->states)) {
  511. throw new \LogicException('Cannot pop state without a previous state.');
  512. }
  514. $this->state = array_pop($this->states);
  515. }
  516. }
  518. class_alias('Twig\Lexer', 'Twig_Lexer');