github
/
linguist
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Add support for the lean comment format to the tokenizer (#6625) 

* Add support for lean comments

* Update tests

* Cater for another scenario

* Add another example

* Update test/test_tokenizer.rb

Co-authored-by: Eric Wieser <wieser.eric@gmail.com>

---------

Co-authored-by: Eric Wieser <wieser.eric@gmail.com>
This commit is contained in:
Colin Seymour 2023-12-11 11:58:48 +00:00 коммит произвёл GitHub
Родитель 36052b7c32
Коммит 71a100d9bf
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
4 изменённых файлов: 330 добавлений и 284 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

573
ext/linguist/lex.linguist_yy.c
Просмотреть файл

@ -243,7 +243,7 @@
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
* if you want the limit (max/min) macros for int types.
* if you want the limit (max/min) macros for int types.
*/
#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS 1
@ -260,7 +260,7 @@ typedef uint32_t flex_uint32_t;
typedef signed char flex_int8_t;
typedef short int flex_int16_t;
typedef int flex_int32_t;
typedef unsigned char flex_uint8_t;
typedef unsigned char flex_uint8_t;
typedef unsigned short int flex_uint16_t;
typedef unsigned int flex_uint32_t;
@ -384,10 +384,10 @@ typedef size_t yy_size_t;
#define EOB_ACT_CONTINUE_SCAN 0
#define EOB_ACT_END_OF_FILE 1
#define EOB_ACT_LAST_MATCH 2
#define YY_LESS_LINENO(n)
#define YY_LINENO_REWIND_TO(ptr)
/* Return all but the first "n" matched characters back to the input stream. */
#define yyless(n) \
do \
@ -545,8 +545,8 @@ static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner );
yyg->yy_hold_char = *yy_cp; \
*yy_cp = '\0'; \
yyg->yy_c_buf_p = yy_cp;
#define YY_NUM_RULES 79
#define YY_END_OF_BUFFER 80
#define YY_NUM_RULES 82
#define YY_END_OF_BUFFER 83
/* This struct is not used in this scanner,
but its presence is necessary. */
struct yy_trans_info
@ -554,36 +554,36 @@ struct yy_trans_info
flex_int32_t yy_verify;
flex_int32_t yy_nxt;
};
static const flex_int16_t yy_accept[255] =
static const flex_int16_t yy_accept[261] =
{ 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 80, 78, 60, 32,
61, 62, 63, 64, 33, 39, 39, 65, 66, 67,
59, 68, 52, 34, 34, 69, 70, 52, 49, 52,
71, 72, 35, 41, 73, 41, 74, 75, 40, 76,
40, 77, 78, 32, 61, 62, 63, 33, 59, 68,
52, 70, 23, 23, 23, 23, 23, 23, 23, 23,
60, 48, 51, 54, 31, 61, 35, 62, 42, 0,
42, 63, 53, 64, 44, 45, 31, 39, 36, 19,
0, 0, 39, 65, 66, 67, 59, 46, 56, 57,
0, 0, 0, 0, 0, 0, 0, 0, 83, 81,
63, 35, 64, 65, 66, 67, 36, 42, 42, 68,
69, 70, 62, 71, 55, 37, 37, 72, 73, 55,
52, 55, 74, 75, 38, 44, 76, 44, 77, 78,
43, 79, 43, 80, 81, 35, 64, 65, 66, 36,
62, 71, 55, 73, 25, 25, 25, 25, 25, 25,
25, 25, 25, 63, 51, 54, 57, 34, 64, 38,
65, 45, 0, 45, 66, 56, 67, 47, 48, 34,
42, 39, 19, 0, 0, 42, 68, 69, 70, 62,
68, 16, 52, 34, 34, 35, 34, 34, 35, 55,
69, 70, 50, 50, 44, 47, 52, 52, 49, 52,
49, 44, 71, 72, 0, 41, 38, 73, 41, 74,
75, 18, 0, 40, 37, 76, 40, 77, 0, 0,
0, 0, 0, 0, 0, 8, 8, 3, 3, 0,
61, 0, 7, 7, 63, 0, 0, 59, 0, 0,
35, 5, 9, 9, 70, 24, 0, 26, 27, 0,
0, 0, 20, 42, 21, 0, 58, 56, 57, 15,
14, 0, 34, 34, 34, 34, 34, 0, 0, 0,
0, 5, 8, 3, 0, 2, 0, 2, 2, 12,
49, 59, 60, 71, 16, 24, 55, 37, 37, 38,
37, 37, 38, 58, 72, 73, 53, 53, 47, 50,
55, 55, 52, 55, 52, 47, 74, 75, 0, 44,
41, 76, 44, 77, 78, 18, 0, 43, 40, 79,
43, 80, 0, 0, 0, 0, 0, 0, 0, 8,
8, 3, 3, 0, 64, 0, 7, 7, 66, 0,
0, 62, 0, 0, 38, 5, 9, 9, 73, 26,
0, 28, 29, 0, 0, 0, 33, 20, 45, 21,
0, 61, 59, 60, 15, 14, 23, 0, 37, 37,
37, 37, 37, 0, 0, 0, 0, 5, 8, 3,
7, 11, 6, 6, 10, 35, 5, 5, 5, 5,
9, 25, 28, 29, 30, 0, 13, 34, 34, 34,
34, 34, 34, 34, 17, 0, 0, 0, 2, 12,
12, 11, 11, 6, 10, 10, 22, 4, 4, 43,
34, 34, 34, 0, 2, 4, 0, 0, 0, 0,
0, 1, 0, 0
0, 2, 0, 2, 2, 12, 7, 11, 6, 6,
10, 38, 5, 5, 5, 5, 9, 27, 30, 31,
32, 0, 13, 37, 37, 37, 37, 37, 37, 37,
17, 0, 0, 0, 2, 12, 12, 11, 11, 6,
10, 10, 22, 4, 4, 46, 37, 37, 37, 0,
2, 4, 0, 0, 0, 0, 0, 1, 0, 0
} ;
static const YY_CHAR yy_ec[256] =
@ -627,226 +627,226 @@ static const YY_CHAR yy_meta[50] =
7, 7, 7, 7, 7, 1, 1, 1, 1
} ;
static const flex_int16_t yy_base[272] =
static const flex_int16_t yy_base[278] =
{ 0,
0, 48, 598, 597, 593, 592, 591, 590, 592, 591,
566, 543, 524, 519, 511, 510, 522, 613, 48, 509,
501, 56, 51, 59, 482, 68, 479, 37, 63, 475,
73, 86, 89, 98, 445, 95, 466, 139, 85, 126,
95, 51, 475, 52, 445, 443, 58, 439, 107, 137,
429, 108, 183, 165, 169, 471, 179, 175, 182, 176,
195, 190, 613, 456, 457, 425, 459, 465, 459, 451,
67, 443, 613, 613, 461, 459, 458, 456, 451, 425,
415, 451, 434, 447, 423, 613, 437, 147, 434, 613,
428, 423, 425, 423, 421, 418, 109, 407, 80, 176,
0, 48, 604, 603, 599, 598, 597, 596, 598, 597,
604, 603, 597, 596, 588, 587, 587, 586, 602, 619,
48, 595, 593, 56, 51, 59, 588, 68, 585, 37,
63, 581, 73, 86, 91, 100, 551, 97, 572, 141,
87, 128, 97, 51, 587, 52, 557, 555, 58, 552,
109, 139, 510, 110, 185, 167, 171, 530, 181, 177,
184, 178, 198, 193, 619, 502, 499, 467, 501, 502,
490, 476, 460, 67, 453, 619, 619, 471, 469, 468,
460, 455, 429, 418, 454, 437, 451, 434, 619, 448,
149, 445, 619, 439, 434, 442, 440, 438, 436, 77,
414, 156, 191, 204, 386, 216, 209, 168, 225, 613,
408, 406, 411, 613, 410, 409, 217, 233, 171, 242,
202, 613, 398, 396, 405, 192, 385, 385, 383, 381,
379, 613, 399, 201, 368, 368, 365, 361, 0, 259,
266, 268, 392, 389, 271, 613, 0, 613, 0, 276,
278, 403, 613, 0, 280, 284, 400, 287, 296, 397,
223, 298, 613, 0, 289, 613, 373, 613, 613, 392,
386, 391, 613, 353, 613, 369, 613, 613, 613, 613,
367, 236, 274, 352, 288, 269, 315, 367, 365, 355,
307, 320, 0, 0, 348, 324, 324, 325, 299, 331,
425, 184, 110, 432, 158, 432, 202, 211, 403, 188,
223, 186, 218, 619, 425, 423, 428, 619, 426, 418,
231, 242, 209, 245, 83, 619, 407, 405, 414, 225,
394, 388, 386, 384, 381, 619, 401, 217, 370, 370,
368, 366, 0, 196, 233, 269, 397, 394, 271, 619,
0, 619, 0, 277, 273, 408, 619, 0, 279, 282,
405, 286, 283, 404, 211, 290, 619, 0, 288, 619,
383, 619, 619, 402, 396, 403, 619, 619, 368, 619,
386, 619, 619, 619, 619, 383, 619, 284, 269, 368,
300, 220, 316, 382, 377, 375, 301, 303, 0, 0,
0, 312, 613, 0, 303, 311, 296, 613, 327, 351,
0, 613, 613, 613, 613, 274, 613, 246, 320, 241,
371, 0, 322, 0, 613, 161, 133, 220, 110, 134,
613, 120, 613, 0, 102, 613, 613, 613, 72, 613,
613, 346, 0, 319, 362, 58, 352, 410, 374, 370,
377, 454, 378, 613, 499, 502, 506, 512, 519, 525,
532, 539, 546, 548, 555, 562, 569, 576, 583, 590,
597
322, 311, 370, 323, 341, 370, 0, 362, 619, 0,
352, 325, 341, 619, 319, 348, 0, 619, 619, 619,
619, 326, 619, 297, 333, 322, 357, 0, 323, 0,
619, 270, 198, 137, 199, 180, 619, 136, 619, 0,
125, 619, 619, 619, 72, 619, 619, 353, 0, 333,
367, 58, 291, 396, 377, 378, 381, 440, 390, 619,
485, 488, 492, 498, 505, 511, 518, 525, 532, 534,
541, 548, 555, 562, 569, 576, 583
} ;
static const flex_int16_t yy_def[272] =
static const flex_int16_t yy_def[278] =
{ 0,
254, 1, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 254, 254, 254, 254,
256, 256, 254, 254, 254, 254, 254, 254, 254, 254,
254, 256, 254, 254, 34, 254, 254, 254, 254, 254,
254, 256, 256, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 256, 22, 254, 254, 254, 32,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 256, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
260, 1, 261, 261, 261, 261, 261, 261, 261, 261,
261, 261, 261, 261, 261, 261, 261, 261, 260, 260,
260, 260, 262, 262, 260, 260, 260, 260, 260, 260,
260, 260, 260, 262, 260, 260, 36, 260, 260, 260,
260, 260, 260, 262, 262, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 262, 24, 260, 260,
260, 34, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 262,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
254, 254, 254, 254, 34, 256, 256, 256, 256, 254,
254, 254, 254, 254, 254, 254, 257, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 53, 254,
254, 254, 254, 254, 254, 254, 258, 254, 259, 260,
254, 254, 254, 261, 254, 254, 254, 254, 254, 254,
256, 262, 254, 263, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 256, 254, 254, 254, 254,
254, 262, 258, 259, 260, 195, 264, 195, 198, 265,
260, 260, 260, 260, 260, 260, 260, 260, 36, 262,
262, 262, 262, 260, 260, 260, 260, 260, 260, 260,
263, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 55, 260, 260, 260, 260, 260, 260, 260,
264, 260, 265, 266, 260, 260, 260, 267, 260, 260,
260, 260, 260, 260, 262, 268, 260, 269, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 262, 260, 260, 260, 260, 260, 268, 264, 265,
261, 266, 254, 267, 268, 256, 262, 254, 262, 262,
263, 254, 254, 254, 254, 254, 254, 254, 254, 256,
254, 187, 187, 187, 254, 254, 254, 264, 198, 265,
254, 266, 254, 267, 268, 254, 254, 254, 269, 254,
254, 221, 221, 264, 198, 269, 264, 270, 264, 270,
271, 270, 271, 0, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254
266, 201, 270, 201, 204, 271, 267, 272, 260, 273,
274, 262, 268, 260, 268, 268, 269, 260, 260, 260,
260, 260, 260, 260, 260, 262, 260, 193, 193, 193,
260, 260, 260, 270, 204, 271, 260, 272, 260, 273,
274, 260, 260, 260, 275, 260, 260, 227, 227, 270,
204, 275, 270, 276, 270, 276, 277, 276, 277, 0,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260
} ;
static const flex_int16_t yy_nxt[663] =
static const flex_int16_t yy_nxt[669] =
{ 0,
18, 18, 18, 18, 19, 20, 21, 22, 23, 24,
25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
35, 36, 37, 38, 39, 40, 41, 42, 43, 43,
43, 43, 43, 43, 44, 45, 46, 47, 48, 43,
43, 43, 43, 43, 43, 49, 50, 51, 52, 53,
94, 53, 71, 54, 55, 56, 57, 254, 58, 82,
208, 83, 254, 78, 59, 60, 61, 79, 84, 125,
62, 71, 72, 73, 208, 83, 73, 95, 124, 88,
89, 90, 83, 83, 85, 91, 126, 83, 127, 97,
80, 72, 254, 99, 92, 130, 74, 83, 98, 99,
20, 20, 20, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40, 41, 42, 43, 44, 45, 45,
45, 45, 45, 45, 46, 47, 48, 49, 50, 45,
45, 45, 45, 45, 45, 51, 52, 53, 54, 55,
97, 55, 74, 56, 57, 58, 59, 260, 60, 85,
214, 86, 260, 81, 61, 62, 63, 82, 87, 129,
64, 74, 75, 76, 214, 86, 76, 98, 128, 91,
92, 93, 86, 86, 88, 94, 130, 86, 131, 100,
83, 75, 260, 100, 95, 134, 77, 86, 101, 102,
100, 81, 102, 101, 254, 86, 178, 103, 121, 119,
121, 110, 103, 83, 103, 104, 111, 105, 105, 83,
73, 123, 254, 132, 133, 97, 77, 106, 107, 77,
107, 108, 83, 74, 98, 122, 254, 106, 77, 77,
77, 77, 109, 113, 103, 114, 86, 114, 115, 120,
119, 120, 134, 245, 135, 116, 138, 117, 88, 89,
180, 83, 118, 119, 120, 114, 114, 146, 147, 181,
75, 148, 149, 150, 254, 151, 156, 159, 156, 159,
240, 153, 154, 136, 139, 87, 139, 155, 140, 141,
100, 142, 163, 164, 121, 119, 121, 240, 158, 143,
103, 84, 101, 104, 105, 89, 125, 106, 125, 107,
125, 123, 125, 114, 107, 86, 107, 108, 115, 109,
109, 86, 76, 127, 103, 136, 137, 260, 80, 110,
111, 80, 111, 112, 86, 77, 184, 126, 260, 110,
80, 80, 80, 80, 113, 117, 107, 118, 89, 118,
119, 124, 123, 124, 138, 234, 139, 120, 142, 121,
91, 92, 185, 86, 122, 123, 124, 118, 118, 150,
151, 186, 78, 152, 153, 154, 250, 155, 160, 163,
160, 163, 260, 157, 158, 140, 143, 90, 143, 159,
144, 145, 260, 146, 260, 167, 168, 102, 150, 151,
186, 144, 179, 83, 73, 145, 83, 98, 102, 103,
157, 160, 165, 162, 103, 254, 103, 161, 103, 83,
103, 104, 254, 104, 104, 121, 126, 121, 127, 254,
185, 254, 185, 182, 183, 103, 183, 184, 228, 107,
103, 107, 103, 182, 187, 187, 134, 254, 135, 116,
185, 103, 185, 187, 187, 187, 118, 119, 120, 244,
103, 146, 147, 206, 187, 120, 119, 120, 148, 149,
153, 154, 141, 163, 164, 254, 142, 195, 241, 195,
148, 149, 153, 154, 151, 156, 240, 156, 155, 203,
204, 163, 164, 145, 196, 197, 197, 159, 208, 159,
162, 147, 191, 148, 191, 86, 76, 149, 86, 101,
183, 105, 161, 164, 106, 169, 166, 260, 192, 165,
107, 107, 86, 107, 260, 107, 260, 107, 108, 260,
108, 108, 125, 123, 125, 152, 153, 193, 193, 145,
188, 189, 251, 189, 190, 246, 193, 193, 193, 107,
188, 212, 226, 111, 107, 111, 107, 193, 120, 130,
107, 131, 138, 107, 139, 122, 123, 124, 124, 123,
124, 157, 158, 167, 168, 152, 153, 146, 201, 155,
201, 157, 158, 160, 163, 160, 163, 159, 209, 210,
167, 168, 214, 149, 215, 202, 203, 203, 191, 189,
208, 220, 209, 97, 183, 236, 183, 219, 219, 203,
204, 165, 98, 237, 233, 199, 210, 254, 183, 157,
183, 210, 208, 210, 209, 254, 254, 254, 254, 238,
239, 160, 221, 231, 222, 222, 185, 228, 185, 219,
219, 229, 228, 222, 223, 224, 77, 107, 108, 195,
183, 195, 183, 208, 223, 77, 77, 77, 77, 77,
185, 247, 185, 248, 198, 248, 196, 197, 197, 210,
228, 254, 227, 254, 210, 248, 210, 248, 248, 248,
248, 248, 226, 225, 218, 217, 216, 199, 221, 174,
221, 221, 228, 215, 251, 249, 214, 213, 212, 221,
191, 189, 100, 209, 210, 214, 246, 215, 216, 234,
169, 101, 260, 216, 260, 216, 205, 161, 164, 225,
225, 244, 245, 201, 260, 201, 260, 243, 260, 247,
189, 260, 189, 227, 255, 228, 228, 191, 246, 191,
202, 203, 203, 214, 228, 229, 230, 80, 111, 112,
214, 234, 225, 225, 242, 229, 80, 80, 80, 80,
80, 205, 204, 189, 239, 189, 216, 191, 254, 191,
254, 216, 237, 216, 227, 253, 227, 227, 254, 260,
254, 260, 254, 235, 254, 227, 248, 249, 234, 189,
190, 254, 233, 254, 232, 234, 248, 254, 231, 254,
242, 243, 205, 183, 184, 202, 200, 192, 191, 138,
242, 248, 137, 248, 136, 135, 190, 131, 130, 129,
128, 127, 189, 124, 123, 116, 122, 188, 112, 111,
77, 101, 98, 96, 251, 95, 94, 93, 252, 252,
252, 252, 252, 252, 177, 176, 89, 175, 122, 252,
252, 252, 252, 252, 252, 254, 84, 254, 83, 82,
81, 174, 79, 78, 254, 76, 173, 72, 172, 171,
170, 169, 168, 167, 166, 152, 137, 131, 251, 129,
128, 254, 252, 252, 252, 252, 252, 252, 112, 77,
96, 93, 87, 252, 252, 252, 252, 252, 252, 63,
224, 223, 257, 222, 179, 221, 220, 219, 218, 211,
208, 206, 198, 197, 142, 141, 140, 139, 196, 135,
257, 134, 133, 132, 258, 258, 258, 258, 258, 258,
131, 195, 128, 127, 120, 258, 258, 258, 258, 258,
258, 260, 126, 260, 194, 116, 115, 80, 187, 104,
101, 99, 98, 97, 96, 182, 181, 92, 180, 126,
87, 86, 85, 84, 257, 179, 82, 81, 258, 258,
258, 258, 258, 258, 260, 79, 178, 75, 177, 258,
258, 258, 258, 258, 258, 65, 65, 65, 65, 65,
65, 65, 80, 176, 80, 118, 118, 118, 199, 199,
63, 63, 63, 63, 63, 63, 77, 76, 77, 114,
114, 114, 193, 193, 75, 193, 193, 193, 193, 194,
194, 254, 194, 194, 194, 194, 198, 70, 70, 69,
198, 198, 201, 201, 69, 201, 201, 201, 201, 207,
207, 207, 207, 207, 207, 207, 211, 211, 68, 211,
211, 211, 211, 197, 197, 230, 230, 230, 230, 230,
230, 230, 232, 232, 232, 232, 232, 232, 232, 234,
234, 68, 234, 234, 234, 234, 235, 235, 235, 235,
235, 235, 235, 246, 246, 246, 246, 246, 246, 246,
250, 250, 250, 250, 250, 250, 250, 253, 253, 253,
175, 199, 199, 199, 199, 200, 200, 174, 200, 200,
200, 200, 204, 173, 172, 171, 204, 204, 207, 207,
170, 207, 207, 207, 207, 213, 213, 213, 213, 213,
213, 213, 217, 217, 156, 217, 217, 217, 217, 203,
203, 236, 236, 236, 236, 236, 236, 236, 238, 238,
238, 238, 238, 238, 238, 240, 240, 141, 240, 240,
240, 240, 241, 241, 241, 241, 241, 241, 241, 252,
252, 252, 252, 252, 252, 252, 256, 256, 256, 256,
256, 256, 256, 259, 259, 259, 259, 259, 259, 259,
135, 133, 132, 260, 116, 80, 99, 96, 90, 79,
253, 253, 253, 253, 67, 67, 66, 66, 65, 65,
64, 64, 17, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254
78, 260, 73, 73, 72, 72, 71, 71, 70, 70,
69, 69, 68, 68, 67, 67, 66, 66, 19, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260
} ;
static const flex_int16_t yy_chk[663] =
static const flex_int16_t yy_chk[669] =
{ 0,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
28, 2, 19, 2, 2, 2, 2, 42, 2, 23,
246, 28, 22, 22, 2, 2, 2, 22, 24, 44,
2, 71, 19, 19, 239, 23, 23, 29, 42, 26,
26, 26, 47, 24, 24, 26, 44, 29, 44, 31,
22, 71, 32, 99, 26, 47, 19, 31, 31, 32,
30, 2, 21, 2, 2, 2, 2, 44, 2, 25,
252, 30, 24, 24, 2, 2, 2, 24, 26, 46,
2, 74, 21, 21, 245, 25, 25, 31, 44, 28,
28, 28, 49, 26, 26, 28, 46, 31, 46, 33,
24, 74, 34, 100, 28, 49, 21, 33, 33, 34,
32, 22, 33, 32, 235, 24, 99, 33, 39, 39,
39, 36, 33, 33, 33, 34, 36, 34, 34, 36,
41, 41, 232, 49, 49, 97, 34, 34, 34, 34,
34, 34, 52, 39, 97, 40, 230, 34, 34, 34,
34, 34, 34, 38, 40, 38, 50, 38, 38, 40,
40, 40, 49, 229, 49, 38, 52, 38, 88, 88,
102, 50, 38, 38, 38, 38, 38, 54, 54, 102,
54, 55, 55, 55, 108, 55, 58, 60, 58, 60,
227, 57, 57, 50, 53, 58, 53, 57, 53, 53,
100, 53, 62, 62, 119, 119, 119, 226, 59, 53,
34, 24, 100, 34, 35, 26, 125, 35, 125, 35,
41, 41, 41, 38, 35, 35, 35, 36, 38, 36,
36, 38, 43, 43, 103, 51, 51, 241, 36, 36,
36, 36, 36, 36, 54, 41, 103, 42, 238, 36,
36, 36, 36, 36, 36, 40, 42, 40, 52, 40,
40, 42, 42, 42, 51, 234, 51, 40, 54, 40,
91, 91, 105, 52, 40, 40, 40, 40, 40, 56,
56, 105, 56, 57, 57, 57, 234, 57, 60, 62,
60, 62, 236, 59, 59, 52, 55, 60, 55, 59,
55, 55, 112, 55, 110, 64, 64, 102, 144, 144,
108, 53, 100, 57, 57, 53, 59, 59, 61, 103,
58, 60, 62, 61, 103, 107, 103, 60, 61, 61,
61, 104, 106, 104, 104, 121, 126, 121, 126, 161,
106, 109, 106, 104, 104, 117, 104, 104, 228, 107,
117, 107, 117, 104, 109, 109, 134, 220, 134, 118,
182, 118, 182, 109, 109, 109, 118, 118, 118, 228,
120, 140, 140, 161, 109, 120, 120, 120, 141, 141,
142, 142, 141, 145, 145, 186, 142, 150, 218, 150,
151, 151, 155, 155, 151, 156, 216, 156, 155, 158,
158, 165, 165, 145, 150, 150, 150, 159, 207, 159,
61, 55, 110, 55, 110, 59, 59, 55, 61, 61,
102, 63, 60, 62, 63, 64, 63, 165, 112, 62,
107, 63, 63, 63, 113, 107, 192, 107, 108, 111,
108, 108, 123, 123, 123, 145, 145, 113, 113, 145,
108, 108, 235, 108, 108, 233, 113, 113, 113, 121,
108, 165, 192, 111, 121, 111, 121, 113, 122, 130,
122, 130, 138, 124, 138, 122, 122, 122, 124, 124,
124, 146, 146, 149, 149, 155, 155, 146, 154, 155,
154, 159, 159, 160, 163, 160, 163, 159, 162, 162,
169, 169, 166, 149, 166, 154, 154, 154, 188, 189,
162, 186, 162, 158, 183, 205, 183, 185, 185, 191,
191, 165, 158, 206, 202, 150, 162, 206, 185, 156,
185, 162, 192, 162, 192, 196, 198, 196, 198, 209,
209, 159, 187, 200, 187, 187, 223, 244, 223, 219,
219, 199, 197, 187, 187, 187, 187, 187, 187, 195,
219, 195, 219, 210, 187, 187, 187, 187, 187, 187,
242, 244, 242, 245, 198, 245, 195, 195, 195, 210,
247, 250, 190, 250, 210, 249, 210, 249, 251, 253,
251, 253, 189, 188, 184, 181, 176, 195, 221, 174,
221, 221, 249, 172, 250, 247, 171, 170, 167, 221,
188, 189, 162, 197, 197, 198, 232, 198, 166, 253,
169, 162, 202, 166, 202, 166, 154, 160, 163, 191,
191, 215, 215, 201, 204, 201, 204, 212, 226, 224,
191, 212, 191, 193, 253, 193, 193, 229, 222, 229,
201, 201, 201, 213, 193, 193, 193, 193, 193, 193,
216, 250, 225, 225, 211, 193, 193, 193, 193, 193,
193, 201, 204, 225, 208, 225, 216, 248, 251, 248,
251, 216, 206, 216, 227, 250, 227, 227, 255, 256,
255, 256, 257, 205, 257, 227, 227, 227, 203, 227,
227, 259, 196, 259, 195, 255, 227, 254, 194, 254,
221, 221, 160, 221, 221, 157, 152, 144, 143, 138,
221, 248, 137, 248, 136, 135, 133, 131, 130, 129,
128, 127, 125, 124, 123, 116, 115, 113, 112, 111,
105, 101, 98, 96, 248, 95, 94, 93, 248, 248,
248, 248, 248, 248, 92, 91, 89, 87, 85, 248,
248, 248, 248, 248, 248, 252, 84, 252, 83, 82,
81, 80, 79, 78, 77, 76, 75, 72, 70, 69,
68, 67, 66, 65, 64, 56, 51, 48, 252, 46,
45, 43, 252, 252, 252, 252, 252, 252, 37, 35,
30, 27, 25, 252, 252, 252, 252, 252, 252, 255,
190, 186, 256, 181, 179, 176, 175, 174, 171, 164,
161, 156, 148, 147, 142, 141, 140, 139, 137, 135,
254, 134, 133, 132, 254, 254, 254, 254, 254, 254,
131, 129, 128, 127, 120, 254, 254, 254, 254, 254,
254, 258, 119, 258, 117, 116, 115, 109, 106, 104,
101, 99, 98, 97, 96, 95, 94, 92, 90, 88,
87, 86, 85, 84, 258, 83, 82, 81, 258, 258,
258, 258, 258, 258, 80, 79, 78, 75, 73, 258,
258, 258, 258, 258, 258, 261, 261, 261, 261, 261,
261, 261, 262, 72, 262, 263, 263, 263, 264, 264,
255, 255, 255, 255, 255, 255, 256, 21, 256, 257,
257, 257, 258, 258, 20, 258, 258, 258, 258, 259,
259, 17, 259, 259, 259, 259, 260, 16, 15, 14,
260, 260, 261, 261, 13, 261, 261, 261, 261, 262,
262, 262, 262, 262, 262, 262, 263, 263, 12, 263,
263, 263, 263, 264, 264, 265, 265, 265, 265, 265,
265, 265, 266, 266, 266, 266, 266, 266, 266, 267,
267, 11, 267, 267, 267, 267, 268, 268, 268, 268,
268, 268, 268, 269, 269, 269, 269, 269, 269, 269,
270, 270, 270, 270, 270, 270, 270, 271, 271, 271,
71, 264, 264, 264, 264, 265, 265, 70, 265, 265,
265, 265, 266, 69, 68, 67, 266, 266, 267, 267,
66, 267, 267, 267, 267, 268, 268, 268, 268, 268,
268, 268, 269, 269, 58, 269, 269, 269, 269, 270,
270, 271, 271, 271, 271, 271, 271, 271, 272, 272,
272, 272, 272, 272, 272, 273, 273, 53, 273, 273,
273, 273, 274, 274, 274, 274, 274, 274, 274, 275,
275, 275, 275, 275, 275, 275, 276, 276, 276, 276,
276, 276, 276, 277, 277, 277, 277, 277, 277, 277,
50, 48, 47, 45, 39, 37, 32, 29, 27, 23,
271, 271, 271, 271, 10, 9, 8, 7, 6, 5,
4, 3, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
254, 254
22, 19, 18, 17, 16, 15, 14, 13, 12, 11,
10, 9, 8, 7, 6, 5, 4, 3, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260, 260, 260,
260, 260, 260, 260, 260, 260, 260, 260
} ;
/* The intent behind this definition is that it'll catch
@ -919,6 +919,7 @@ static const flex_int16_t yy_chk[663] =
#define python_dcomment 5
#define python_scomment 6
#define roff_comment 7
#define lean_comment 8
#ifndef YY_NO_UNISTD_H
/* Special case for "unistd.h", since it is non-ANSI. We include it way
@ -1016,7 +1017,7 @@ extern int yywrap ( yyscan_t yyscanner );
#endif
#ifndef YY_NO_UNPUT
#endif
#ifndef yytext_ptr
@ -1178,7 +1179,7 @@ YY_DECL
#line 56 "tokenizer.l"
#line 1182 "lex.linguist_yy.c"
#line 1183 "lex.linguist_yy.c"
while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */
{
@ -1206,13 +1207,13 @@ yy_match:
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
if ( yy_current_state >= 255 )
if ( yy_current_state >= 261 )
yy_c = yy_meta[yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
++yy_cp;
}
while ( yy_current_state != 254 );
while ( yy_current_state != 260 );
yy_cp = yyg->yy_last_accepting_cpos;
yy_current_state = yyg->yy_last_accepting_state;
@ -1377,20 +1378,20 @@ YY_RULE_SETUP
{ FEED_STATIC("COMMENT.ig"); BEGIN(roff_comment); return 1; }
YY_BREAK
case 23:
/* rule 23 can match eol */
YY_RULE_SETUP
#line 106 "tokenizer.l"
{ /* nothing */ }
#line 105 "tokenizer.l"
{ FEED_STATIC("COMMENT/-"); BEGIN(lean_comment); return 1; }
YY_BREAK
case 24:
YY_RULE_SETUP
#line 107 "tokenizer.l"
{ BEGIN(INITIAL); }
#line 106 "tokenizer.l"
{ FEED_STATIC("COMMENT/-"); BEGIN(lean_comment); return 1; }
YY_BREAK
case 25:
/* rule 25 can match eol */
YY_RULE_SETUP
#line 108 "tokenizer.l"
{ BEGIN(INITIAL); }
{ /* nothing */ }
YY_BREAK
case 26:
YY_RULE_SETUP
@ -1413,25 +1414,25 @@ YY_RULE_SETUP
{ BEGIN(INITIAL); }
YY_BREAK
case 30:
/* rule 30 can match eol */
YY_RULE_SETUP
#line 113 "tokenizer.l"
{ BEGIN(INITIAL); }
YY_BREAK
case 31:
YY_RULE_SETUP
#line 115 "tokenizer.l"
{ /* nothing */ }
#line 114 "tokenizer.l"
{ BEGIN(INITIAL); }
YY_BREAK
case 32:
/* rule 32 can match eol */
YY_RULE_SETUP
#line 116 "tokenizer.l"
{ eat_until_unescaped('"'); }
#line 115 "tokenizer.l"
{ BEGIN(INITIAL); }
YY_BREAK
case 33:
YY_RULE_SETUP
#line 117 "tokenizer.l"
{ eat_until_unescaped('\''); }
#line 116 "tokenizer.l"
{ BEGIN(INITIAL); }
YY_BREAK
case 34:
YY_RULE_SETUP
@ -1440,22 +1441,22 @@ YY_RULE_SETUP
YY_BREAK
case 35:
YY_RULE_SETUP
#line 120 "tokenizer.l"
{ FEED(); return 1; }
#line 119 "tokenizer.l"
{ eat_until_unescaped('"'); }
YY_BREAK
case 36:
YY_RULE_SETUP
#line 122 "tokenizer.l"
{ FEED(); return 1; }
#line 120 "tokenizer.l"
{ eat_until_unescaped('\''); }
YY_BREAK
case 37:
YY_RULE_SETUP
#line 123 "tokenizer.l"
{ FEED(); return 1; }
#line 121 "tokenizer.l"
{ /* nothing */ }
YY_BREAK
case 38:
YY_RULE_SETUP
#line 124 "tokenizer.l"
#line 123 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 39:
@ -1480,42 +1481,42 @@ YY_RULE_SETUP
YY_BREAK
case 43:
YY_RULE_SETUP
#line 130 "tokenizer.l"
#line 129 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 44:
YY_RULE_SETUP
#line 132 "tokenizer.l"
#line 130 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 45:
YY_RULE_SETUP
#line 133 "tokenizer.l"
#line 131 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 46:
YY_RULE_SETUP
#line 135 "tokenizer.l"
#line 133 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 47:
YY_RULE_SETUP
#line 136 "tokenizer.l"
#line 135 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 48:
YY_RULE_SETUP
#line 138 "tokenizer.l"
#line 136 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 49:
YY_RULE_SETUP
#line 139 "tokenizer.l"
#line 138 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 50:
YY_RULE_SETUP
#line 140 "tokenizer.l"
#line 139 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 51:
@ -1545,32 +1546,32 @@ YY_RULE_SETUP
YY_BREAK
case 56:
YY_RULE_SETUP
#line 147 "tokenizer.l"
#line 146 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 57:
YY_RULE_SETUP
#line 148 "tokenizer.l"
#line 147 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 58:
YY_RULE_SETUP
#line 149 "tokenizer.l"
#line 148 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 59:
YY_RULE_SETUP
#line 151 "tokenizer.l"
#line 150 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 60:
YY_RULE_SETUP
#line 152 "tokenizer.l"
#line 151 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 61:
YY_RULE_SETUP
#line 153 "tokenizer.l"
#line 152 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 62:
@ -1654,17 +1655,32 @@ YY_RULE_SETUP
{ FEED(); return 1; }
YY_BREAK
case 78:
/* rule 78 can match eol */
YY_RULE_SETUP
#line 171 "tokenizer.l"
{ /* nothing */ }
#line 170 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 79:
YY_RULE_SETUP
#line 173 "tokenizer.l"
#line 171 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 80:
YY_RULE_SETUP
#line 172 "tokenizer.l"
{ FEED(); return 1; }
YY_BREAK
case 81:
/* rule 81 can match eol */
YY_RULE_SETUP
#line 174 "tokenizer.l"
{ /* nothing */ }
YY_BREAK
case 82:
YY_RULE_SETUP
#line 176 "tokenizer.l"
YY_FATAL_ERROR( "flex scanner jammed" );
YY_BREAK
#line 1668 "lex.linguist_yy.c"
#line 1684 "lex.linguist_yy.c"
case YY_STATE_EOF(INITIAL):
case YY_STATE_EOF(c_comment):
case YY_STATE_EOF(xml_comment):
@ -1673,6 +1689,7 @@ case YY_STATE_EOF(ocaml_comment):
case YY_STATE_EOF(python_dcomment):
case YY_STATE_EOF(python_scomment):
case YY_STATE_EOF(roff_comment):
case YY_STATE_EOF(lean_comment):
yyterminate();
case YY_END_OF_BUFFER:
@ -1972,7 +1989,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
if ( yy_current_state >= 255 )
if ( yy_current_state >= 261 )
yy_c = yy_meta[yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
@ -2001,11 +2018,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
{
yy_current_state = (int) yy_def[yy_current_state];
if ( yy_current_state >= 255 )
if ( yy_current_state >= 261 )
yy_c = yy_meta[yy_c];
}
yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];
yy_is_jam = (yy_current_state == 254);
yy_is_jam = (yy_current_state == 260);
(void)yyg;
return yy_is_jam ? 0 : yy_current_state;
@ -2165,7 +2182,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner)
YY_BUFFER_STATE yy_create_buffer (FILE * file, int size , yyscan_t yyscanner)
{
YY_BUFFER_STATE b;
b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) , yyscanner );
if ( ! b )
YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" );
@ -2231,7 +2248,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner)
}
b->yy_is_interactive = 0;
errno = oerrno;
}
@ -2373,7 +2390,7 @@ static void yyensure_buffer_stack (yyscan_t yyscanner)
YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscanner)
{
YY_BUFFER_STATE b;
if ( size < 2 ||
base[size-2] != YY_END_OF_BUFFER_CHAR ||
base[size-1] != YY_END_OF_BUFFER_CHAR )
@ -2409,7 +2426,7 @@ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscann
*/
YY_BUFFER_STATE yy_scan_string (const char * yystr , yyscan_t yyscanner)
{
return yy_scan_bytes( yystr, (int) strlen(yystr) , yyscanner);
}
@ -2426,7 +2443,7 @@ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan
char *buf;
yy_size_t n;
int i;
/* Get memory for full buffer, including space for trailing EOB's. */
n = (yy_size_t) (_yybytes_len + 2);
buf = (char *) yyalloc( n , yyscanner );
@ -2499,7 +2516,7 @@ int yyget_lineno (yyscan_t yyscanner)
if (! YY_CURRENT_BUFFER)
return 0;
return yylineno;
}
@ -2512,7 +2529,7 @@ int yyget_column (yyscan_t yyscanner)
if (! YY_CURRENT_BUFFER)
return 0;
return yycolumn;
}
@ -2574,7 +2591,7 @@ void yyset_lineno (int _line_number , yyscan_t yyscanner)
/* lineno is only valid if an input buffer exists. */
if (! YY_CURRENT_BUFFER )
YY_FATAL_ERROR( "yyset_lineno called with no buffer" );
yylineno = _line_number;
}
@ -2589,7 +2606,7 @@ void yyset_column (int _column_no , yyscan_t yyscanner)
/* column is only valid if an input buffer exists. */
if (! YY_CURRENT_BUFFER )
YY_FATAL_ERROR( "yyset_column called with no buffer" );
yycolumn = _column_no;
}
@ -2806,4 +2823,6 @@ void yyfree (void * ptr , yyscan_t yyscanner)
#define YYTABLES_NAME "yytables"
#line 173 "tokenizer.l"
#line 176 "tokenizer.l"

9
ext/linguist/lex.linguist_yy.h
Просмотреть файл

@ -246,7 +246,7 @@
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
/* C99 says to define __STDC_LIMIT_MACROS before including stdint.h,
* if you want the limit (max/min) macros for int types.
* if you want the limit (max/min) macros for int types.
*/
#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS 1
@ -263,7 +263,7 @@ typedef uint32_t flex_uint32_t;
typedef signed char flex_int8_t;
typedef short int flex_int16_t;
typedef int flex_int32_t;
typedef unsigned char flex_uint8_t;
typedef unsigned char flex_uint8_t;
typedef unsigned short int flex_uint16_t;
typedef unsigned int flex_uint32_t;
@ -435,6 +435,7 @@ void yyfree ( void * , yyscan_t yyscanner );
#define python_dcomment 5
#define python_scomment 6
#define roff_comment 7
#define lean_comment 8
#endif
@ -694,9 +695,9 @@ extern int yylex (yyscan_t yyscanner);
#undef yyTABLES_NAME
#endif
#line 173 "tokenizer.l"
#line 176 "tokenizer.l"
#line 701 "lex.linguist_yy.h"
#line 702 "lex.linguist_yy.h"
#undef linguist_yyIN_HEADER
#endif /* linguist_yyHEADER_H */

7
ext/linguist/tokenizer.l
Просмотреть файл

@ -51,7 +51,7 @@
%}
%option never-interactive yywrap reentrant nounput warn nodefault header-file="lex.linguist_yy.h" extra-type="VALUE*" prefix="linguist_yy"
%x c_comment xml_comment haskell_comment ocaml_comment python_dcomment python_scomment roff_comment
%x c_comment xml_comment haskell_comment ocaml_comment python_dcomment python_scomment roff_comment lean_comment
%%
@ -102,8 +102,10 @@
"\"\"\"" { FEED_STATIC("COMMENT\"\"\""); BEGIN(python_dcomment); return 1; }
"'''" { FEED_STATIC("COMMENT'''"); BEGIN(python_scomment); return 1; }
^".ig"\n { FEED_STATIC("COMMENT.ig"); BEGIN(roff_comment); return 1; }
"/--" { FEED_STATIC("COMMENT/-"); BEGIN(lean_comment); return 1; }
"/-" { FEED_STATIC("COMMENT/-"); BEGIN(lean_comment); return 1; }
<c_comment,xml_comment,haskell_comment,ocaml_comment,python_dcomment,python_scomment,roff_comment>.|\n { /* nothing */ }
<c_comment,xml_comment,haskell_comment,ocaml_comment,python_dcomment,python_scomment,roff_comment,lean_comment>.|\n { /* nothing */ }
<c_comment>"*/" { BEGIN(INITIAL); }
<xml_comment>"-->" { BEGIN(INITIAL); }
<haskell_comment>"-}" { BEGIN(INITIAL); }
@ -111,6 +113,7 @@
<python_dcomment>"\"\"\"" { BEGIN(INITIAL); }
<python_scomment>"'''" { BEGIN(INITIAL); }
<roff_comment>".."\n { BEGIN(INITIAL); }
<lean_comment>"-/" { BEGIN(INITIAL); }
\"\"|'' { /* nothing */ }
\" { eat_until_unescaped('"'); }

25
test/test_tokenizer.rb
Просмотреть файл

@ -84,6 +84,29 @@ class TestTokenizer < Minitest::Test
assert_equal %w(foo COMMENT""" bar), tokenize("foo\n\"\"\"\nComment\n\"\"\"\nbar")
assert_equal %w(foo COMMENT''' bar), tokenize("foo\n'''\nComment\n'''\nbar")
# Lean comments
assert_equal %w(foo COMMENT/-), tokenize("foo /- Comment -/")
assert_equal %w(foo COMMENT/-), tokenize("foo /-Comment-/")
assert_equal %w(foo COMMENT/-), tokenize("foo /- \nComment\n -/")
assert_equal %w(foo COMMENT/-), tokenize("foo /-\nComment\n-/")
assert_equal %w(foo COMMENT/-), tokenize("foo /-- Comment -/")
assert_equal %w(foo COMMENT/-), tokenize("foo /--Comment-/")
assert_equal %w(foo COMMENT/-), tokenize("foo /--\nComment\n-/")
assert_equal %w(foo COMMENT/-), tokenize("foo /-! Comment -/")
assert_equal %w(foo COMMENT/-), tokenize("foo /-!Comment-/")
assert_equal %w(foo COMMENT/-), tokenize("foo /-!\nComment\n-/")
assert_equal %w(foo COMMENT/- bar), tokenize("foo /-\nComment\n-/ bar")
assert_equal %w(foo COMMENT/- bar), tokenize("foo /-\nComment\n-/\nbar")
assert_equal %w(foo COMMENT/- bar), tokenize("foo\n/-\nComment\n-/\nbar")
# Nested comments are not processed correctly as it's rarely used and adds unnecessary complexity
assert_equal %w(foo COMMENT/- comment), tokenize("foo /- Comment /- Still Comment /- And Still Comment -/ comment")
assert_equal %w(foo COMMENT/- comment3 - / bar), tokenize("foo /- comment1 /- comment2 -/ comment3 -/ bar")
assert_equal %w(COMMENT/-), tokenize("/-\n*\n-/")
assert_equal %w(COMMENT/-), tokenize("/-*-/")
assert_equal %w(COMMENT/-), tokenize("/-**-/")
assert_equal %w(COMMENT/-), tokenize("/-!-/")
assert_equal %w(COMMENT/-), tokenize("/--/ -/")
# Roff comments
assert_equal %w(COMMENT.\\" bar), tokenize(".\\\" foo\nbar")
assert_equal %w(COMMENT.\\" bar), tokenize(". \\\" foo\nbar")
@ -210,7 +233,7 @@ class TestTokenizer < Minitest::Test
assert_equal %w(- ! # $ % & * + , . : ; <=>), tokenize("-!#$%&*+,.:;<=>")
assert_equal %w(- ! # $ % & ? @ \\ ^ _ ` | ~), tokenize("-!#$%&?@\\^_`|~")
assert_equal %w(- ! # $ % & * + , . : ; <=>), tokenize("-!#$%&*+,.:;<=>")
assert_equal %w(/ - ! # $ % & * + , . : ; <>), tokenize("/-!#$%&*+,.:;<>")
assert_equal %w(- / ! # $ % & * + , . : ; <>), tokenize("-/!#$%&*+,.:;<>")
end
def test_c_tokens