You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

363 line
12KB

  1. <?php
  2. /**
  3. * Forgivingly lexes HTML (SGML-style) markup into tokens.
  4. *
  5. * A lexer parses a string of SGML-style markup and converts them into
  6. * corresponding tokens. It doesn't check for well-formedness, although its
  7. * internal mechanism may make this automatic (such as the case of
  8. * HTMLPurifier_Lexer_DOMLex). There are several implementations to choose
  9. * from.
  10. *
  11. * A lexer is HTML-oriented: it might work with XML, but it's not
  12. * recommended, as we adhere to a subset of the specification for optimization
  13. * reasons. This might change in the future. Also, most tokenizers are not
  14. * expected to handle DTDs or PIs.
  15. *
  16. * This class should not be directly instantiated, but you may use create() to
  17. * retrieve a default copy of the lexer. Being a supertype, this class
  18. * does not actually define any implementation, but offers commonly used
  19. * convenience functions for subclasses.
  20. *
  21. * @note The unit tests will instantiate this class for testing purposes, as
  22. * many of the utility functions require a class to be instantiated.
  23. * This means that, even though this class is not runnable, it will
  24. * not be declared abstract.
  25. *
  26. * @par
  27. *
  28. * @note
  29. * We use tokens rather than create a DOM representation because DOM would:
  30. *
  31. * @par
  32. * -# Require more processing and memory to create,
  33. * -# Is not streamable, and
  34. * -# Has the entire document structure (html and body not needed).
  35. *
  36. * @par
  37. * However, DOM is helpful in that it makes it easy to move around nodes
  38. * without a lot of lookaheads to see when a tag is closed. This is a
  39. * limitation of the token system and some workarounds would be nice.
  40. */
  41. class HTMLPurifier_Lexer
  42. {
  43. /**
  44. * Whether or not this lexer implements line-number/column-number tracking.
  45. * If it does, set to true.
  46. */
  47. public $tracksLineNumbers = false;
  48. // -- STATIC ----------------------------------------------------------
  49. /**
  50. * Retrieves or sets the default Lexer as a Prototype Factory.
  51. *
  52. * By default HTMLPurifier_Lexer_DOMLex will be returned. There are
  53. * a few exceptions involving special features that only DirectLex
  54. * implements.
  55. *
  56. * @note The behavior of this class has changed, rather than accepting
  57. * a prototype object, it now accepts a configuration object.
  58. * To specify your own prototype, set %Core.LexerImpl to it.
  59. * This change in behavior de-singletonizes the lexer object.
  60. *
  61. * @param HTMLPurifier_Config $config
  62. * @return HTMLPurifier_Lexer
  63. * @throws HTMLPurifier_Exception
  64. */
  65. public static function create($config)
  66. {
  67. if (!($config instanceof HTMLPurifier_Config)) {
  68. $lexer = $config;
  69. trigger_error(
  70. "Passing a prototype to
  71. HTMLPurifier_Lexer::create() is deprecated, please instead
  72. use %Core.LexerImpl",
  73. E_USER_WARNING
  74. );
  75. } else {
  76. $lexer = $config->get('Core.LexerImpl');
  77. }
  78. $needs_tracking =
  79. $config->get('Core.MaintainLineNumbers') ||
  80. $config->get('Core.CollectErrors');
  81. $inst = null;
  82. if (is_object($lexer)) {
  83. $inst = $lexer;
  84. } else {
  85. if (is_null($lexer)) {
  86. do {
  87. // auto-detection algorithm
  88. if ($needs_tracking) {
  89. $lexer = 'DirectLex';
  90. break;
  91. }
  92. if (class_exists('DOMDocument') &&
  93. method_exists('DOMDocument', 'loadHTML') &&
  94. !extension_loaded('domxml')
  95. ) {
  96. // check for DOM support, because while it's part of the
  97. // core, it can be disabled compile time. Also, the PECL
  98. // domxml extension overrides the default DOM, and is evil
  99. // and nasty and we shan't bother to support it
  100. $lexer = 'DOMLex';
  101. } else {
  102. $lexer = 'DirectLex';
  103. }
  104. } while (0);
  105. } // do..while so we can break
  106. // instantiate recognized string names
  107. switch ($lexer) {
  108. case 'DOMLex':
  109. $inst = new HTMLPurifier_Lexer_DOMLex();
  110. break;
  111. case 'DirectLex':
  112. $inst = new HTMLPurifier_Lexer_DirectLex();
  113. break;
  114. case 'PH5P':
  115. $inst = new HTMLPurifier_Lexer_PH5P();
  116. break;
  117. default:
  118. throw new HTMLPurifier_Exception(
  119. "Cannot instantiate unrecognized Lexer type " .
  120. htmlspecialchars($lexer)
  121. );
  122. }
  123. }
  124. if (!$inst) {
  125. throw new HTMLPurifier_Exception('No lexer was instantiated');
  126. }
  127. // once PHP DOM implements native line numbers, or we
  128. // hack out something using XSLT, remove this stipulation
  129. if ($needs_tracking && !$inst->tracksLineNumbers) {
  130. throw new HTMLPurifier_Exception(
  131. 'Cannot use lexer that does not support line numbers with ' .
  132. 'Core.MaintainLineNumbers or Core.CollectErrors (use DirectLex instead)'
  133. );
  134. }
  135. return $inst;
  136. }
  137. // -- CONVENIENCE MEMBERS ---------------------------------------------
  138. public function __construct()
  139. {
  140. $this->_entity_parser = new HTMLPurifier_EntityParser();
  141. }
  142. /**
  143. * Most common entity to raw value conversion table for special entities.
  144. * @type array
  145. */
  146. protected $_special_entity2str =
  147. array(
  148. '&quot;' => '"',
  149. '&amp;' => '&',
  150. '&lt;' => '<',
  151. '&gt;' => '>',
  152. '&#39;' => "'",
  153. '&#039;' => "'",
  154. '&#x27;' => "'"
  155. );
  156. /**
  157. * Parses special entities into the proper characters.
  158. *
  159. * This string will translate escaped versions of the special characters
  160. * into the correct ones.
  161. *
  162. * @warning
  163. * You should be able to treat the output of this function as
  164. * completely parsed, but that's only because all other entities should
  165. * have been handled previously in substituteNonSpecialEntities()
  166. *
  167. * @param string $string String character data to be parsed.
  168. * @return string Parsed character data.
  169. */
  170. public function parseData($string)
  171. {
  172. // following functions require at least one character
  173. if ($string === '') {
  174. return '';
  175. }
  176. // subtracts amps that cannot possibly be escaped
  177. $num_amp = substr_count($string, '&') - substr_count($string, '& ') -
  178. ($string[strlen($string) - 1] === '&' ? 1 : 0);
  179. if (!$num_amp) {
  180. return $string;
  181. } // abort if no entities
  182. $num_esc_amp = substr_count($string, '&amp;');
  183. $string = strtr($string, $this->_special_entity2str);
  184. // code duplication for sake of optimization, see above
  185. $num_amp_2 = substr_count($string, '&') - substr_count($string, '& ') -
  186. ($string[strlen($string) - 1] === '&' ? 1 : 0);
  187. if ($num_amp_2 <= $num_esc_amp) {
  188. return $string;
  189. }
  190. // hmm... now we have some uncommon entities. Use the callback.
  191. $string = $this->_entity_parser->substituteSpecialEntities($string);
  192. return $string;
  193. }
  194. /**
  195. * Lexes an HTML string into tokens.
  196. * @param $string String HTML.
  197. * @param HTMLPurifier_Config $config
  198. * @param HTMLPurifier_Context $context
  199. * @return HTMLPurifier_Token[] array representation of HTML.
  200. */
  201. public function tokenizeHTML($string, $config, $context)
  202. {
  203. trigger_error('Call to abstract class', E_USER_ERROR);
  204. }
  205. /**
  206. * Translates CDATA sections into regular sections (through escaping).
  207. * @param string $string HTML string to process.
  208. * @return string HTML with CDATA sections escaped.
  209. */
  210. protected static function escapeCDATA($string)
  211. {
  212. return preg_replace_callback(
  213. '/<!\[CDATA\[(.+?)\]\]>/s',
  214. array('HTMLPurifier_Lexer', 'CDATACallback'),
  215. $string
  216. );
  217. }
  218. /**
  219. * Special CDATA case that is especially convoluted for <script>
  220. * @param string $string HTML string to process.
  221. * @return string HTML with CDATA sections escaped.
  222. */
  223. protected static function escapeCommentedCDATA($string)
  224. {
  225. return preg_replace_callback(
  226. '#<!--//--><!\[CDATA\[//><!--(.+?)//--><!\]\]>#s',
  227. array('HTMLPurifier_Lexer', 'CDATACallback'),
  228. $string
  229. );
  230. }
  231. /**
  232. * Special Internet Explorer conditional comments should be removed.
  233. * @param string $string HTML string to process.
  234. * @return string HTML with conditional comments removed.
  235. */
  236. protected static function removeIEConditional($string)
  237. {
  238. return preg_replace(
  239. '#<!--\[if [^>]+\]>.*?<!\[endif\]-->#si', // probably should generalize for all strings
  240. '',
  241. $string
  242. );
  243. }
  244. /**
  245. * Callback function for escapeCDATA() that does the work.
  246. *
  247. * @warning Though this is public in order to let the callback happen,
  248. * calling it directly is not recommended.
  249. * @param array $matches PCRE matches array, with index 0 the entire match
  250. * and 1 the inside of the CDATA section.
  251. * @return string Escaped internals of the CDATA section.
  252. */
  253. protected static function CDATACallback($matches)
  254. {
  255. // not exactly sure why the character set is needed, but whatever
  256. return htmlspecialchars($matches[1], ENT_COMPAT, 'UTF-8');
  257. }
  258. /**
  259. * Takes a piece of HTML and normalizes it by converting entities, fixing
  260. * encoding, extracting bits, and other good stuff.
  261. * @param string $html HTML.
  262. * @param HTMLPurifier_Config $config
  263. * @param HTMLPurifier_Context $context
  264. * @return string
  265. * @todo Consider making protected
  266. */
  267. public function normalize($html, $config, $context)
  268. {
  269. // normalize newlines to \n
  270. if ($config->get('Core.NormalizeNewlines')) {
  271. $html = str_replace("\r\n", "\n", $html);
  272. $html = str_replace("\r", "\n", $html);
  273. }
  274. if ($config->get('HTML.Trusted')) {
  275. // escape convoluted CDATA
  276. $html = $this->escapeCommentedCDATA($html);
  277. }
  278. // escape CDATA
  279. $html = $this->escapeCDATA($html);
  280. $html = $this->removeIEConditional($html);
  281. // extract body from document if applicable
  282. if ($config->get('Core.ConvertDocumentToFragment')) {
  283. $e = false;
  284. if ($config->get('Core.CollectErrors')) {
  285. $e =& $context->get('ErrorCollector');
  286. }
  287. $new_html = $this->extractBody($html);
  288. if ($e && $new_html != $html) {
  289. $e->send(E_WARNING, 'Lexer: Extracted body');
  290. }
  291. $html = $new_html;
  292. }
  293. // expand entities that aren't the big five
  294. $html = $this->_entity_parser->substituteNonSpecialEntities($html);
  295. // clean into wellformed UTF-8 string for an SGML context: this has
  296. // to be done after entity expansion because the entities sometimes
  297. // represent non-SGML characters (horror, horror!)
  298. $html = HTMLPurifier_Encoder::cleanUTF8($html);
  299. // if processing instructions are to removed, remove them now
  300. if ($config->get('Core.RemoveProcessingInstructions')) {
  301. $html = preg_replace('#<\?.+?\?>#s', '', $html);
  302. }
  303. return $html;
  304. }
  305. /**
  306. * Takes a string of HTML (fragment or document) and returns the content
  307. * @todo Consider making protected
  308. */
  309. public function extractBody($html)
  310. {
  311. $matches = array();
  312. $result = preg_match('|(.*?)<body[^>]*>(.*)</body>|is', $html, $matches);
  313. if ($result) {
  314. // Make sure it's not in a comment
  315. $comment_start = strrpos($matches[1], '<!--');
  316. $comment_end = strrpos($matches[1], '-->');
  317. if ($comment_start === false ||
  318. ($comment_end !== false && $comment_end > $comment_start)) {
  319. return $matches[2];
  320. }
  321. }
  322. return $html;
  323. }
  324. }
  325. // vim: et sw=4 sts=4