tokenize.js 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312
  1. 'use strict';
  2. Object.defineProperty(exports, "__esModule", {
  3. value: true
  4. });
  5. exports.default = tokenize;
  6. var _input = require('./input');
  7. var _input2 = _interopRequireDefault(_input);
  8. var _tokenizeString2 = require('./tokenize-string');
  9. var _tokenizeString3 = _interopRequireDefault(_tokenizeString2);
  10. var _tokenizeComment2 = require('./tokenize-comment');
  11. var _tokenizeComment3 = _interopRequireDefault(_tokenizeComment2);
  12. var _tokenizeInterpolant2 = require('./tokenize-interpolant');
  13. var _tokenizeInterpolant3 = _interopRequireDefault(_tokenizeInterpolant2);
  14. function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
  15. var singleQuote = "'".charCodeAt(0),
  16. doubleQuote = '"'.charCodeAt(0),
  17. dollar = '$'.charCodeAt(0),
  18. hash = '#'.charCodeAt(0),
  19. backslash = '\\'.charCodeAt(0),
  20. slash = '/'.charCodeAt(0),
  21. newline = '\n'.charCodeAt(0),
  22. space = ' '.charCodeAt(0),
  23. feed = '\f'.charCodeAt(0),
  24. tab = '\t'.charCodeAt(0),
  25. cr = '\r'.charCodeAt(0),
  26. openBracket = '('.charCodeAt(0),
  27. closeBracket = ')'.charCodeAt(0),
  28. openCurly = '{'.charCodeAt(0),
  29. closeCurly = '}'.charCodeAt(0),
  30. semicolon = ';'.charCodeAt(0),
  31. asterisk = '*'.charCodeAt(0),
  32. colon = ':'.charCodeAt(0),
  33. at = '@'.charCodeAt(0),
  34. comma = ','.charCodeAt(0),
  35. plus = '+'.charCodeAt(0),
  36. minus = '-'.charCodeAt(0),
  37. decComb = '>'.charCodeAt(0),
  38. adjComb = '~'.charCodeAt(0),
  39. number = /[+-]?(\d+(\.\d+)?|\.\d+)|(e[+-]\d+)/gi,
  40. sQuoteEnd = /(.*?)[^\\](?=((#{)|'))/gm,
  41. dQuoteEnd = /(.*?)[^\\](?=((#{)|"))/gm,
  42. wordEnd = /[ \n\t\r\(\)\{\},:;@!'"\\]|\/(?=\*)|#(?={)/g,
  43. ident = /-?([a-z_]|\\[^\\])([a-z-_0-9]|\\[^\\])*/gi;
  44. function tokenize(input, l, p) {
  45. var tokens = [];
  46. var css = input.css.valueOf();
  47. var code = void 0,
  48. next = void 0,
  49. quote = void 0,
  50. lines = void 0,
  51. last = void 0,
  52. content = void 0,
  53. escape = void 0,
  54. nextLine = void 0,
  55. nextOffset = void 0,
  56. escaped = void 0,
  57. escapePos = void 0,
  58. inInterpolant = void 0,
  59. inComment = void 0,
  60. inString = void 0;
  61. var length = css.length;
  62. var offset = -1;
  63. var line = l || 1;
  64. var pos = p || 0;
  65. while (pos < length) {
  66. code = css.charCodeAt(pos);
  67. if (code === newline) {
  68. offset = pos;
  69. line += 1;
  70. }
  71. switch (code) {
  72. case space:
  73. case tab:
  74. case cr:
  75. case feed:
  76. next = pos;
  77. do {
  78. next += 1;
  79. code = css.charCodeAt(next);
  80. if (code === newline) {
  81. offset = next;
  82. line += 1;
  83. }
  84. } while (code === space || code === tab || code === cr || code === feed);
  85. tokens.push(['space', css.slice(pos, next)]);
  86. pos = next - 1;
  87. break;
  88. case newline:
  89. tokens.push(['newline', '\n', line, pos - offset]);
  90. break;
  91. case plus:
  92. tokens.push(['+', '+', line, pos - offset]);
  93. break;
  94. case minus:
  95. tokens.push(['-', '-', line, pos - offset]);
  96. break;
  97. case decComb:
  98. tokens.push(['>', '>', line, pos - offset]);
  99. break;
  100. case adjComb:
  101. tokens.push(['~', '~', line, pos - offset]);
  102. break;
  103. case openCurly:
  104. tokens.push(['{', '{', line, pos - offset]);
  105. break;
  106. case closeCurly:
  107. if (inInterpolant) {
  108. inInterpolant = false;
  109. tokens.push(['endInterpolant', '}', line, pos - offset]);
  110. } else {
  111. tokens.push(['}', '}', line, pos - offset]);
  112. }
  113. break;
  114. case comma:
  115. tokens.push([',', ',', line, pos - offset]);
  116. break;
  117. case dollar:
  118. tokens.push(['$', '$', line, pos - offset]);
  119. break;
  120. case colon:
  121. tokens.push([':', ':', line, pos - offset]);
  122. break;
  123. case semicolon:
  124. tokens.push([';', ';', line, pos - offset]);
  125. break;
  126. case openBracket:
  127. tokens.push(['(', '(', line, pos - offset]);
  128. break;
  129. case closeBracket:
  130. tokens.push([')', ')', line, pos - offset]);
  131. break;
  132. case singleQuote:
  133. case doubleQuote:
  134. quote = code === singleQuote ? "'" : '"';
  135. tokens.push([quote, quote, line, pos - offset]);
  136. next = pos + 1;
  137. var _tokenizeString = (0, _tokenizeString3.default)(input, line, next, offset, quote),
  138. t = _tokenizeString.tokens,
  139. _l = _tokenizeString.line,
  140. _p = _tokenizeString.pos,
  141. o = _tokenizeString.offset;
  142. tokens = tokens.concat(t);
  143. next = _p;
  144. line = _l;
  145. offset = o;
  146. pos = next;
  147. break;
  148. case at:
  149. tokens.push(['@', '@', line, pos - offset]);
  150. break;
  151. case backslash:
  152. next = pos;
  153. escape = true;
  154. while (css.charCodeAt(next + 1) === backslash) {
  155. next += 1;
  156. escape = !escape;
  157. }
  158. code = css.charCodeAt(next + 1);
  159. if (escape && code !== space && code !== newline && code !== tab && code !== cr && code !== feed) {
  160. next += 1;
  161. }
  162. tokens.push(['word', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  163. pos = next;
  164. break;
  165. default:
  166. ident.lastIndex = pos;
  167. number.lastIndex = pos;
  168. wordEnd.lastIndex = pos;
  169. if (code === slash && css.charCodeAt(pos + 1) === asterisk) {
  170. inComment = true;
  171. tokens.push(['startComment', '/*', line, pos + 1 - offset]);
  172. next = pos + 1;
  173. var _tokenizeComment = (0, _tokenizeComment3.default)(input, line, next + 1, offset),
  174. _t = _tokenizeComment.tokens,
  175. _l2 = _tokenizeComment.line,
  176. _p2 = _tokenizeComment.pos,
  177. _o = _tokenizeComment.offset;
  178. tokens = tokens.concat(_t);
  179. next = _p2;
  180. line = _l2;
  181. offset = _o;
  182. pos = next;
  183. break;
  184. }
  185. if (code === asterisk && css.charCodeAt(pos + 1) !== slash) {
  186. tokens.push(['*', '*', line, pos - offset]);
  187. break;
  188. }
  189. if (inComment && code === asterisk && css.charCodeAt(pos + 1) === slash) {
  190. inComment = false;
  191. tokens.push(['endComment', '*/', line, pos + 1 - offset]);
  192. pos += 2;
  193. break;
  194. }
  195. if (code === slash && css.charCodeAt(pos + 1) !== slash) {
  196. tokens.push(['/', '/', line, pos - offset]);
  197. break;
  198. }
  199. if (code === hash && css.charCodeAt(pos + 1) === openCurly) {
  200. inInterpolant = true;
  201. tokens.push(['startInterpolant', '#{', line, pos + 1 - offset]);
  202. next = pos + 1;
  203. var _tokenizeInterpolant = (0, _tokenizeInterpolant3.default)(input, line, next + 1, offset),
  204. _t2 = _tokenizeInterpolant.tokens,
  205. _l3 = _tokenizeInterpolant.line,
  206. _p3 = _tokenizeInterpolant.pos,
  207. _o2 = _tokenizeInterpolant.offset;
  208. tokens = tokens.concat(_t2);
  209. next = _p3;
  210. line = _l3;
  211. offset = _o2;
  212. pos = next;
  213. break;
  214. }
  215. if (code === slash && css.charCodeAt(pos + 1) === slash) {
  216. next = css.indexOf('\n', pos + 2);
  217. next = (next > 0 ? next : css.length) - 1;
  218. tokens.push(['scssComment', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  219. pos = next;
  220. break;
  221. }
  222. if (ident.test(css) && (ident.lastIndex = pos || 1) && ident.exec(css).index === pos) {
  223. next = ident.lastIndex - 1;
  224. tokens.push(['ident', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  225. pos = next;
  226. break;
  227. }
  228. if (number.test(css) && (number.lastIndex = pos || 1) && number.exec(css).index === pos) {
  229. next = number.lastIndex - 1;
  230. tokens.push(['number', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  231. pos = next;
  232. break;
  233. }
  234. wordEnd.lastIndex = pos + 1;
  235. wordEnd.test(css);
  236. if (wordEnd.lastIndex === 0) {
  237. next = css.length - 1;
  238. } else {
  239. next = wordEnd.lastIndex - 2;
  240. }
  241. tokens.push(['word', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  242. pos = next;
  243. break;
  244. }
  245. pos++;
  246. }
  247. return tokens;
  248. }