aboutsummaryrefslogtreecommitdiff
path: root/src/utf8.c
blob: 8e3c4bb1acb4e517bb3bd49b1ee05c111ce0753f (plain)
  1. #include <stdlib.h>
  2. #include <stdint.h>
  3. #include <assert.h>
  4. #include "utf8.h"
  5. static const int8_t utf8proc_utf8class[256] = {
  6. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  7. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  8. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  9. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  10. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  11. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  12. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  13. 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  14. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  15. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  16. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  17. 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  18. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  19. 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  20. 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
  21. 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
  22. static void encode_unknown(cmark_strbuf *buf)
  23. {
  24. static const uint8_t repl[] = {239, 191, 189};
  25. cmark_strbuf_put(buf, repl, 3);
  26. }
  27. static int utf8proc_charlen(const uint8_t *str, int str_len)
  28. {
  29. int length, i;
  30. if (!str_len)
  31. return 0;
  32. length = utf8proc_utf8class[str[0]];
  33. if (!length)
  34. return -1;
  35. if (str_len >= 0 && length > str_len)
  36. return -str_len;
  37. for (i = 1; i < length; i++) {
  38. if ((str[i] & 0xC0) != 0x80)
  39. return -i;
  40. }
  41. return length;
  42. }
  43. // Validate a single UTF-8 character according to RFC 3629.
  44. static int utf8proc_valid(const uint8_t *str, int str_len)
  45. {
  46. int length = utf8proc_charlen(str, str_len);
  47. if (length <= 0)
  48. return length;
  49. switch (length) {
  50. case 1:
  51. if (str[0] == 0x00) {
  52. // ASCII NUL is technically valid but rejected
  53. // for security reasons.
  54. return -length;
  55. }
  56. break;
  57. case 2:
  58. if (str[0] < 0xC2) {
  59. // Overlong
  60. return -length;
  61. }
  62. break;
  63. case 3:
  64. if (str[0] == 0xE0) {
  65. if (str[1] < 0xA0) {
  66. // Overlong
  67. return -length;
  68. }
  69. }
  70. else if (str[0] == 0xED) {
  71. if (str[1] >= 0xA0) {
  72. // Surrogate
  73. return -length;
  74. }
  75. }
  76. break;
  77. case 4:
  78. if (str[0] == 0xF0) {
  79. if (str[1] < 0x90) {
  80. // Overlong
  81. return -length;
  82. }
  83. }
  84. else if (str[0] >= 0xF4) {
  85. if (str[0] > 0xF4 || str[1] >= 0x90) {
  86. // Above 0x10FFFF
  87. return -length;
  88. }
  89. }
  90. break;
  91. }
  92. return length;
  93. }
  94. void utf8proc_detab(cmark_strbuf *ob, const uint8_t *line, size_t size)
  95. {
  96. static const uint8_t whitespace[] = " ";
  97. size_t i = 0, tab = 0;
  98. while (i < size) {
  99. size_t org = i;
  100. while (i < size && line[i] != '\t' && line[i] != '\0'
  101. && line[i] < 0x80) {
  102. i++; tab++;
  103. }
  104. if (i > org)
  105. cmark_strbuf_put(ob, line + org, i - org);
  106. if (i >= size)
  107. break;
  108. if (line[i] == '\t') {
  109. int numspaces = 4 - (tab % 4);
  110. cmark_strbuf_put(ob, whitespace, numspaces);
  111. i += 1;
  112. tab += numspaces;
  113. } else {
  114. int charlen = utf8proc_valid(line + i, size - i);
  115. if (charlen >= 0) {
  116. cmark_strbuf_put(ob, line + i, charlen);
  117. } else {
  118. encode_unknown(ob);
  119. charlen = -charlen;
  120. }
  121. i += charlen;
  122. tab += 1;
  123. }
  124. }
  125. }
  126. int utf8proc_iterate(const uint8_t *str, int str_len, int32_t *dst)
  127. {
  128. int length;
  129. int32_t uc = -1;
  130. *dst = -1;
  131. length = utf8proc_charlen(str, str_len);
  132. if (length < 0)
  133. return -1;
  134. switch (length) {
  135. case 1:
  136. uc = str[0];
  137. break;
  138. case 2:
  139. uc = ((str[0] & 0x1F) << 6) + (str[1] & 0x3F);
  140. if (uc < 0x80) uc = -1;
  141. break;
  142. case 3:
  143. uc = ((str[0] & 0x0F) << 12) + ((str[1] & 0x3F) << 6)
  144. + (str[2] & 0x3F);
  145. if (uc < 0x800 || (uc >= 0xD800 && uc < 0xE000) ||
  146. (uc >= 0xFDD0 && uc < 0xFDF0)) uc = -1;
  147. break;
  148. case 4:
  149. uc = ((str[0] & 0x07) << 18) + ((str[1] & 0x3F) << 12)
  150. + ((str[2] & 0x3F) << 6) + (str[3] & 0x3F);
  151. if (uc < 0x10000 || uc >= 0x110000) uc = -1;
  152. break;
  153. }
  154. if (uc < 0 || ((uc & 0xFFFF) >= 0xFFFE))
  155. return -1;
  156. *dst = uc;
  157. return length;
  158. }
  159. void utf8proc_encode_char(int32_t uc, cmark_strbuf *buf)
  160. {
  161. uint8_t dst[4];
  162. int len = 0;
  163. assert(uc >= 0);
  164. if (uc < 0x80) {
  165. dst[0] = uc;
  166. len = 1;
  167. } else if (uc < 0x800) {
  168. dst[0] = 0xC0 + (uc >> 6);
  169. dst[1] = 0x80 + (uc & 0x3F);
  170. len = 2;
  171. } else if (uc == 0xFFFF) {
  172. dst[0] = 0xFF;
  173. len = 1;
  174. } else if (uc == 0xFFFE) {
  175. dst[0] = 0xFE;
  176. len = 1;
  177. } else if (uc < 0x10000) {
  178. dst[0] = 0xE0 + (uc >> 12);
  179. dst[1] = 0x80 + ((uc >> 6) & 0x3F);
  180. dst[2] = 0x80 + (uc & 0x3F);
  181. len = 3;
  182. } else if (uc < 0x110000) {
  183. dst[0] = 0xF0 + (uc >> 18);
  184. dst[1] = 0x80 + ((uc >> 12) & 0x3F);
  185. dst[2] = 0x80 + ((uc >> 6) & 0x3F);
  186. dst[3] = 0x80 + (uc & 0x3F);
  187. len = 4;
  188. } else {
  189. encode_unknown(buf);
  190. return;
  191. }
  192. cmark_strbuf_put(buf, dst, len);
  193. }
  194. void utf8proc_case_fold(cmark_strbuf *dest, const uint8_t *str, int len)
  195. {
  196. int32_t c;
  197. #define bufpush(x) \
  198. utf8proc_encode_char(x, dest)
  199. while (len > 0) {
  200. int char_len = utf8proc_iterate(str, len, &c);
  201. if (char_len >= 0) {
  202. #include "case_fold_switch.inc"
  203. } else {
  204. encode_unknown(dest);
  205. char_len = -char_len;
  206. }
  207. str += char_len;
  208. len -= char_len;
  209. }
  210. }
  211. // matches anything in the Zs class, plus LF, CR, TAB, FF.
  212. int utf8proc_is_space(int32_t uc)
  213. {
  214. return (uc == 9 ||
  215. uc == 10 ||
  216. uc == 12 ||
  217. uc == 13 ||
  218. uc == 32 ||
  219. uc == 160 ||
  220. uc == 5760 ||
  221. (uc >= 8192 && uc <= 8202) ||
  222. uc == 8239 ||
  223. uc == 8287 ||
  224. uc == 12288);
  225. }
  226. // matches anything in the P[cdefios] classes.
  227. int utf8proc_is_punctuation(int32_t uc)
  228. {
  229. return ((uc >= 33 && uc <= 35) ||
  230. (uc >= 37 && uc <= 42) ||
  231. (uc >= 44 && uc <= 47) ||
  232. uc == 58 ||
  233. uc == 59 ||
  234. uc == 63 ||
  235. uc == 64 ||
  236. (uc >= 91 && uc <= 93) ||
  237. uc == 95 ||
  238. uc == 123 ||
  239. uc == 125 ||
  240. uc == 161 ||
  241. uc == 167 ||
  242. uc == 171 ||
  243. uc == 182 ||
  244. uc == 183 ||
  245. uc == 187 ||
  246. uc == 191 ||
  247. uc == 894 ||
  248. uc == 903 ||
  249. (uc >= 1370 && uc <= 1375) ||
  250. uc == 1417 ||
  251. uc == 1418 ||
  252. uc == 1470 ||
  253. uc == 1472 ||
  254. uc == 1475 ||
  255. uc == 1478 ||
  256. uc == 1523 ||
  257. uc == 1524 ||
  258. uc == 1545 ||
  259. uc == 1546 ||
  260. uc == 1548 ||
  261. uc == 1549 ||
  262. uc == 1563 ||
  263. uc == 1566 ||
  264. uc == 1567 ||
  265. (uc >= 1642 && uc <= 1645) ||
  266. uc == 1748 ||
  267. (uc >= 1792 && uc <= 1805) ||
  268. (uc >= 2039 && uc <= 2041) ||
  269. (uc >= 2096 && uc <= 2110) ||
  270. uc == 2142 ||
  271. uc == 2404 ||
  272. uc == 2405 ||
  273. uc == 2416 ||
  274. uc == 2800 ||
  275. uc == 3572 ||
  276. uc == 3663 ||
  277. uc == 3674 ||
  278. uc == 3675 ||
  279. (uc >= 3844 && uc <= 3858) ||
  280. uc == 3860 ||
  281. (uc >= 3898 && uc <= 3901) ||
  282. uc == 3973 ||
  283. (uc >= 4048 && uc <= 4052) ||
  284. uc == 4057 ||
  285. uc == 4058 ||
  286. (uc >= 4170 && uc <= 4175) ||
  287. uc == 4347 ||
  288. (uc >= 4960 && uc <= 4968) ||
  289. uc == 5120 ||
  290. uc == 5741 ||
  291. uc == 5742 ||
  292. uc == 5787 ||
  293. uc == 5788 ||
  294. (uc >= 5867 && uc <= 5869) ||
  295. uc == 5941 ||
  296. uc == 5942 ||
  297. (uc >= 6100 && uc <= 6102) ||
  298. (uc >= 6104 && uc <= 6106) ||
  299. (uc >= 6144 && uc <= 6154) ||
  300. uc == 6468 ||
  301. uc == 6469 ||
  302. uc == 6686 ||
  303. uc == 6687 ||
  304. (uc >= 6816 && uc <= 6822) ||
  305. (uc >= 6824 && uc <= 6829) ||
  306. (uc >= 7002 && uc <= 7008) ||
  307. (uc >= 7164 && uc <= 7167) ||
  308. (uc >= 7227 && uc <= 7231) ||
  309. uc == 7294 ||
  310. uc == 7295 ||
  311. (uc >= 7360 && uc <= 7367) ||
  312. uc == 7379 ||
  313. (uc >= 8208 && uc <= 8231) ||
  314. (uc >= 8240 && uc <= 8259) ||
  315. (uc >= 8261 && uc <= 8273) ||
  316. (uc >= 8275 && uc <= 8286) ||
  317. uc == 8317 ||
  318. uc == 8318 ||
  319. uc == 8333 ||
  320. uc == 8334 ||
  321. (uc >= 8968 && uc <= 8971) ||
  322. uc == 9001 ||
  323. uc == 9002 ||
  324. (uc >= 10088 && uc <= 10101) ||
  325. uc == 10181 ||
  326. uc == 10182 ||
  327. (uc >= 10214 && uc <= 10223) ||
  328. (uc >= 10627 && uc <= 10648) ||
  329. (uc >= 10712 && uc <= 10715) ||
  330. uc == 10748 ||
  331. uc == 10749 ||
  332. (uc >= 11513 && uc <= 11516) ||
  333. uc == 11518 ||
  334. uc == 11519 ||
  335. uc == 11632 ||
  336. (uc >= 11776 && uc <= 11822) ||
  337. (uc >= 11824 && uc <= 11842) ||
  338. (uc >= 12289 && uc <= 12291) ||
  339. (uc >= 12296 && uc <= 12305) ||
  340. (uc >= 12308 && uc <= 12319) ||
  341. uc == 12336 ||
  342. uc == 12349 ||
  343. uc == 12448 ||
  344. uc == 12539 ||
  345. uc == 42238 ||
  346. uc == 42239 ||
  347. (uc >= 42509 && uc <= 42511) ||
  348. uc == 42611 ||
  349. uc == 42622 ||
  350. (uc >= 42738 && uc <= 42743) ||
  351. (uc >= 43124 && uc <= 43127) ||
  352. uc == 43214 ||
  353. uc == 43215 ||
  354. (uc >= 43256 && uc <= 43258) ||
  355. uc == 43310 ||
  356. uc == 43311 ||
  357. uc == 43359 ||
  358. (uc >= 43457 && uc <= 43469) ||
  359. uc == 43486 ||
  360. uc == 43487 ||
  361. (uc >= 43612 && uc <= 43615) ||
  362. uc == 43742 ||
  363. uc == 43743 ||
  364. uc == 43760 ||
  365. uc == 43761 ||
  366. uc == 44011 ||
  367. uc == 64830 ||
  368. uc == 64831 ||
  369. (uc >= 65040 && uc <= 65049) ||
  370. (uc >= 65072 && uc <= 65106) ||
  371. (uc >= 65108 && uc <= 65121) ||
  372. uc == 65123 ||
  373. uc == 65128 ||
  374. uc == 65130 ||
  375. uc == 65131 ||
  376. (uc >= 65281 && uc <= 65283) ||
  377. (uc >= 65285 && uc <= 65290) ||
  378. (uc >= 65292 && uc <= 65295) ||
  379. uc == 65306 ||
  380. uc == 65307 ||
  381. uc == 65311 ||
  382. uc == 65312 ||
  383. (uc >= 65339 && uc <= 65341) ||
  384. uc == 65343 ||
  385. uc == 65371 ||
  386. uc == 65373 ||
  387. (uc >= 65375 && uc <= 65381) ||
  388. (uc >= 65792 && uc <= 65794) ||
  389. uc == 66463 ||
  390. uc == 66512 ||
  391. uc == 66927 ||
  392. uc == 67671 ||
  393. uc == 67871 ||
  394. uc == 67903 ||
  395. (uc >= 68176 && uc <= 68184) ||
  396. uc == 68223 ||
  397. (uc >= 68336 && uc <= 68342) ||
  398. (uc >= 68409 && uc <= 68415) ||
  399. (uc >= 68505 && uc <= 68508) ||
  400. (uc >= 69703 && uc <= 69709) ||
  401. uc == 69819 ||
  402. uc == 69820 ||
  403. (uc >= 69822 && uc <= 69825) ||
  404. (uc >= 69952 && uc <= 69955) ||
  405. uc == 70004 ||
  406. uc == 70005 ||
  407. (uc >= 70085 && uc <= 70088) ||
  408. uc == 70093 ||
  409. (uc >= 70200 && uc <= 70205) ||
  410. uc == 70854 ||
  411. (uc >= 71105 && uc <= 71113) ||
  412. (uc >= 71233 && uc <= 71235) ||
  413. (uc >= 74864 && uc <= 74868) ||
  414. uc == 92782 ||
  415. uc == 92783 ||
  416. uc == 92917 ||
  417. (uc >= 92983 && uc <= 92987) ||
  418. uc == 92996 ||
  419. uc == 113823);
  420. }