diff --git a/ext/linguist/lex.linguist_yy.c b/ext/linguist/lex.linguist_yy.c index b7a88d84b7..444ea43e6e 100644 --- a/ext/linguist/lex.linguist_yy.c +++ b/ext/linguist/lex.linguist_yy.c @@ -243,7 +243,7 @@ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, - * if you want the limit (max/min) macros for int types. + * if you want the limit (max/min) macros for int types. */ #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS 1 @@ -260,7 +260,7 @@ typedef uint32_t flex_uint32_t; typedef signed char flex_int8_t; typedef short int flex_int16_t; typedef int flex_int32_t; -typedef unsigned char flex_uint8_t; +typedef unsigned char flex_uint8_t; typedef unsigned short int flex_uint16_t; typedef unsigned int flex_uint32_t; @@ -384,10 +384,10 @@ typedef size_t yy_size_t; #define EOB_ACT_CONTINUE_SCAN 0 #define EOB_ACT_END_OF_FILE 1 #define EOB_ACT_LAST_MATCH 2 - + #define YY_LESS_LINENO(n) #define YY_LINENO_REWIND_TO(ptr) - + /* Return all but the first "n" matched characters back to the input stream. */ #define yyless(n) \ do \ @@ -545,8 +545,8 @@ static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner ); yyg->yy_hold_char = *yy_cp; \ *yy_cp = '\0'; \ yyg->yy_c_buf_p = yy_cp; -#define YY_NUM_RULES 79 -#define YY_END_OF_BUFFER 80 +#define YY_NUM_RULES 82 +#define YY_END_OF_BUFFER 83 /* This struct is not used in this scanner, but its presence is necessary. */ struct yy_trans_info @@ -554,36 +554,36 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[255] = +static const flex_int16_t yy_accept[261] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 80, 78, 60, 32, - 61, 62, 63, 64, 33, 39, 39, 65, 66, 67, - 59, 68, 52, 34, 34, 69, 70, 52, 49, 52, - 71, 72, 35, 41, 73, 41, 74, 75, 40, 76, - 40, 77, 78, 32, 61, 62, 63, 33, 59, 68, - 52, 70, 23, 23, 23, 23, 23, 23, 23, 23, - 60, 48, 51, 54, 31, 61, 35, 62, 42, 0, - 42, 63, 53, 64, 44, 45, 31, 39, 36, 19, - 0, 0, 39, 65, 66, 67, 59, 46, 56, 57, - - 68, 16, 52, 34, 34, 35, 34, 34, 35, 55, - 69, 70, 50, 50, 44, 47, 52, 52, 49, 52, - 49, 44, 71, 72, 0, 41, 38, 73, 41, 74, - 75, 18, 0, 40, 37, 76, 40, 77, 0, 0, - 0, 0, 0, 0, 0, 8, 8, 3, 3, 0, - 61, 0, 7, 7, 63, 0, 0, 59, 0, 0, - 35, 5, 9, 9, 70, 24, 0, 26, 27, 0, - 0, 0, 20, 42, 21, 0, 58, 56, 57, 15, - 14, 0, 34, 34, 34, 34, 34, 0, 0, 0, - 0, 5, 8, 3, 0, 2, 0, 2, 2, 12, - - 7, 11, 6, 6, 10, 35, 5, 5, 5, 5, - 9, 25, 28, 29, 30, 0, 13, 34, 34, 34, - 34, 34, 34, 34, 17, 0, 0, 0, 2, 12, - 12, 11, 11, 6, 10, 10, 22, 4, 4, 43, - 34, 34, 34, 0, 2, 4, 0, 0, 0, 0, - 0, 1, 0, 0 + 0, 0, 0, 0, 0, 0, 0, 0, 83, 81, + 63, 35, 64, 65, 66, 67, 36, 42, 42, 68, + 69, 70, 62, 71, 55, 37, 37, 72, 73, 55, + 52, 55, 74, 75, 38, 44, 76, 44, 77, 78, + 43, 79, 43, 80, 81, 35, 64, 65, 66, 36, + 62, 71, 55, 73, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 63, 51, 54, 57, 34, 64, 38, + 65, 45, 0, 45, 66, 56, 67, 47, 48, 34, + 42, 39, 19, 0, 0, 42, 68, 69, 70, 62, + + 49, 59, 60, 71, 16, 24, 55, 37, 37, 38, + 37, 37, 38, 58, 72, 73, 53, 53, 47, 50, + 55, 55, 52, 55, 52, 47, 74, 75, 0, 44, + 41, 76, 44, 77, 78, 18, 0, 43, 40, 79, + 43, 80, 0, 0, 0, 0, 0, 0, 0, 8, + 8, 3, 3, 0, 64, 0, 7, 7, 66, 0, + 0, 62, 0, 0, 38, 5, 9, 9, 73, 26, + 0, 28, 29, 0, 0, 0, 33, 20, 45, 21, + 0, 61, 59, 60, 15, 14, 23, 0, 37, 37, + 37, 37, 37, 0, 0, 0, 0, 5, 8, 3, + + 0, 2, 0, 2, 2, 12, 7, 11, 6, 6, + 10, 38, 5, 5, 5, 5, 9, 27, 30, 31, + 32, 0, 13, 37, 37, 37, 37, 37, 37, 37, + 17, 0, 0, 0, 2, 12, 12, 11, 11, 6, + 10, 10, 22, 4, 4, 46, 37, 37, 37, 0, + 2, 4, 0, 0, 0, 0, 0, 1, 0, 0 } ; static const YY_CHAR yy_ec[256] = @@ -627,226 +627,226 @@ static const YY_CHAR yy_meta[50] = 7, 7, 7, 7, 7, 1, 1, 1, 1 } ; -static const flex_int16_t yy_base[272] = +static const flex_int16_t yy_base[278] = { 0, - 0, 48, 598, 597, 593, 592, 591, 590, 592, 591, - 566, 543, 524, 519, 511, 510, 522, 613, 48, 509, - 501, 56, 51, 59, 482, 68, 479, 37, 63, 475, - 73, 86, 89, 98, 445, 95, 466, 139, 85, 126, - 95, 51, 475, 52, 445, 443, 58, 439, 107, 137, - 429, 108, 183, 165, 169, 471, 179, 175, 182, 176, - 195, 190, 613, 456, 457, 425, 459, 465, 459, 451, - 67, 443, 613, 613, 461, 459, 458, 456, 451, 425, - 415, 451, 434, 447, 423, 613, 437, 147, 434, 613, - 428, 423, 425, 423, 421, 418, 109, 407, 80, 176, - - 414, 156, 191, 204, 386, 216, 209, 168, 225, 613, - 408, 406, 411, 613, 410, 409, 217, 233, 171, 242, - 202, 613, 398, 396, 405, 192, 385, 385, 383, 381, - 379, 613, 399, 201, 368, 368, 365, 361, 0, 259, - 266, 268, 392, 389, 271, 613, 0, 613, 0, 276, - 278, 403, 613, 0, 280, 284, 400, 287, 296, 397, - 223, 298, 613, 0, 289, 613, 373, 613, 613, 392, - 386, 391, 613, 353, 613, 369, 613, 613, 613, 613, - 367, 236, 274, 352, 288, 269, 315, 367, 365, 355, - 307, 320, 0, 0, 348, 324, 324, 325, 299, 331, - - 0, 312, 613, 0, 303, 311, 296, 613, 327, 351, - 0, 613, 613, 613, 613, 274, 613, 246, 320, 241, - 371, 0, 322, 0, 613, 161, 133, 220, 110, 134, - 613, 120, 613, 0, 102, 613, 613, 613, 72, 613, - 613, 346, 0, 319, 362, 58, 352, 410, 374, 370, - 377, 454, 378, 613, 499, 502, 506, 512, 519, 525, - 532, 539, 546, 548, 555, 562, 569, 576, 583, 590, - 597 + 0, 48, 604, 603, 599, 598, 597, 596, 598, 597, + 604, 603, 597, 596, 588, 587, 587, 586, 602, 619, + 48, 595, 593, 56, 51, 59, 588, 68, 585, 37, + 63, 581, 73, 86, 91, 100, 551, 97, 572, 141, + 87, 128, 97, 51, 587, 52, 557, 555, 58, 552, + 109, 139, 510, 110, 185, 167, 171, 530, 181, 177, + 184, 178, 198, 193, 619, 502, 499, 467, 501, 502, + 490, 476, 460, 67, 453, 619, 619, 471, 469, 468, + 460, 455, 429, 418, 454, 437, 451, 434, 619, 448, + 149, 445, 619, 439, 434, 442, 440, 438, 436, 77, + + 425, 184, 110, 432, 158, 432, 202, 211, 403, 188, + 223, 186, 218, 619, 425, 423, 428, 619, 426, 418, + 231, 242, 209, 245, 83, 619, 407, 405, 414, 225, + 394, 388, 386, 384, 381, 619, 401, 217, 370, 370, + 368, 366, 0, 196, 233, 269, 397, 394, 271, 619, + 0, 619, 0, 277, 273, 408, 619, 0, 279, 282, + 405, 286, 283, 404, 211, 290, 619, 0, 288, 619, + 383, 619, 619, 402, 396, 403, 619, 619, 368, 619, + 386, 619, 619, 619, 619, 383, 619, 284, 269, 368, + 300, 220, 316, 382, 377, 375, 301, 303, 0, 0, + + 322, 311, 370, 323, 341, 370, 0, 362, 619, 0, + 352, 325, 341, 619, 319, 348, 0, 619, 619, 619, + 619, 326, 619, 297, 333, 322, 357, 0, 323, 0, + 619, 270, 198, 137, 199, 180, 619, 136, 619, 0, + 125, 619, 619, 619, 72, 619, 619, 353, 0, 333, + 367, 58, 291, 396, 377, 378, 381, 440, 390, 619, + 485, 488, 492, 498, 505, 511, 518, 525, 532, 534, + 541, 548, 555, 562, 569, 576, 583 } ; -static const flex_int16_t yy_def[272] = +static const flex_int16_t yy_def[278] = { 0, - 254, 1, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 254, 254, 254, 254, - 256, 256, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 256, 254, 254, 34, 254, 254, 254, 254, 254, - 254, 256, 256, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 256, 22, 254, 254, 254, 32, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 256, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - - 254, 254, 254, 254, 34, 256, 256, 256, 256, 254, - 254, 254, 254, 254, 254, 254, 257, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 53, 254, - 254, 254, 254, 254, 254, 254, 258, 254, 259, 260, - 254, 254, 254, 261, 254, 254, 254, 254, 254, 254, - 256, 262, 254, 263, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 256, 254, 254, 254, 254, - 254, 262, 258, 259, 260, 195, 264, 195, 198, 265, - - 261, 266, 254, 267, 268, 256, 262, 254, 262, 262, - 263, 254, 254, 254, 254, 254, 254, 254, 254, 256, - 254, 187, 187, 187, 254, 254, 254, 264, 198, 265, - 254, 266, 254, 267, 268, 254, 254, 254, 269, 254, - 254, 221, 221, 264, 198, 269, 264, 270, 264, 270, - 271, 270, 271, 0, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254 + 260, 1, 261, 261, 261, 261, 261, 261, 261, 261, + 261, 261, 261, 261, 261, 261, 261, 261, 260, 260, + 260, 260, 262, 262, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 262, 260, 260, 36, 260, 260, 260, + 260, 260, 260, 262, 262, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 262, 24, 260, 260, + 260, 34, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 262, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + + 260, 260, 260, 260, 260, 260, 260, 260, 36, 262, + 262, 262, 262, 260, 260, 260, 260, 260, 260, 260, + 263, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 55, 260, 260, 260, 260, 260, 260, 260, + 264, 260, 265, 266, 260, 260, 260, 267, 260, 260, + 260, 260, 260, 260, 262, 268, 260, 269, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 262, 260, 260, 260, 260, 260, 268, 264, 265, + + 266, 201, 270, 201, 204, 271, 267, 272, 260, 273, + 274, 262, 268, 260, 268, 268, 269, 260, 260, 260, + 260, 260, 260, 260, 260, 262, 260, 193, 193, 193, + 260, 260, 260, 270, 204, 271, 260, 272, 260, 273, + 274, 260, 260, 260, 275, 260, 260, 227, 227, 270, + 204, 275, 270, 276, 270, 276, 277, 276, 277, 0, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260 } ; -static const flex_int16_t yy_nxt[663] = +static const flex_int16_t yy_nxt[669] = { 0, - 18, 18, 18, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 42, 43, 43, - 43, 43, 43, 43, 44, 45, 46, 47, 48, 43, - 43, 43, 43, 43, 43, 49, 50, 51, 52, 53, - 94, 53, 71, 54, 55, 56, 57, 254, 58, 82, - 208, 83, 254, 78, 59, 60, 61, 79, 84, 125, - 62, 71, 72, 73, 208, 83, 73, 95, 124, 88, - 89, 90, 83, 83, 85, 91, 126, 83, 127, 97, - 80, 72, 254, 99, 92, 130, 74, 83, 98, 99, - - 100, 81, 102, 101, 254, 86, 178, 103, 121, 119, - 121, 110, 103, 83, 103, 104, 111, 105, 105, 83, - 73, 123, 254, 132, 133, 97, 77, 106, 107, 77, - 107, 108, 83, 74, 98, 122, 254, 106, 77, 77, - 77, 77, 109, 113, 103, 114, 86, 114, 115, 120, - 119, 120, 134, 245, 135, 116, 138, 117, 88, 89, - 180, 83, 118, 119, 120, 114, 114, 146, 147, 181, - 75, 148, 149, 150, 254, 151, 156, 159, 156, 159, - 240, 153, 154, 136, 139, 87, 139, 155, 140, 141, - 100, 142, 163, 164, 121, 119, 121, 240, 158, 143, - - 186, 144, 179, 83, 73, 145, 83, 98, 102, 103, - 157, 160, 165, 162, 103, 254, 103, 161, 103, 83, - 103, 104, 254, 104, 104, 121, 126, 121, 127, 254, - 185, 254, 185, 182, 183, 103, 183, 184, 228, 107, - 103, 107, 103, 182, 187, 187, 134, 254, 135, 116, - 185, 103, 185, 187, 187, 187, 118, 119, 120, 244, - 103, 146, 147, 206, 187, 120, 119, 120, 148, 149, - 153, 154, 141, 163, 164, 254, 142, 195, 241, 195, - 148, 149, 153, 154, 151, 156, 240, 156, 155, 203, - 204, 163, 164, 145, 196, 197, 197, 159, 208, 159, - - 208, 220, 209, 97, 183, 236, 183, 219, 219, 203, - 204, 165, 98, 237, 233, 199, 210, 254, 183, 157, - 183, 210, 208, 210, 209, 254, 254, 254, 254, 238, - 239, 160, 221, 231, 222, 222, 185, 228, 185, 219, - 219, 229, 228, 222, 223, 224, 77, 107, 108, 195, - 183, 195, 183, 208, 223, 77, 77, 77, 77, 77, - 185, 247, 185, 248, 198, 248, 196, 197, 197, 210, - 228, 254, 227, 254, 210, 248, 210, 248, 248, 248, - 248, 248, 226, 225, 218, 217, 216, 199, 221, 174, - 221, 221, 228, 215, 251, 249, 214, 213, 212, 221, - - 242, 243, 205, 183, 184, 202, 200, 192, 191, 138, - 242, 248, 137, 248, 136, 135, 190, 131, 130, 129, - 128, 127, 189, 124, 123, 116, 122, 188, 112, 111, - 77, 101, 98, 96, 251, 95, 94, 93, 252, 252, - 252, 252, 252, 252, 177, 176, 89, 175, 122, 252, - 252, 252, 252, 252, 252, 254, 84, 254, 83, 82, - 81, 174, 79, 78, 254, 76, 173, 72, 172, 171, - 170, 169, 168, 167, 166, 152, 137, 131, 251, 129, - 128, 254, 252, 252, 252, 252, 252, 252, 112, 77, - 96, 93, 87, 252, 252, 252, 252, 252, 252, 63, - - 63, 63, 63, 63, 63, 63, 77, 76, 77, 114, - 114, 114, 193, 193, 75, 193, 193, 193, 193, 194, - 194, 254, 194, 194, 194, 194, 198, 70, 70, 69, - 198, 198, 201, 201, 69, 201, 201, 201, 201, 207, - 207, 207, 207, 207, 207, 207, 211, 211, 68, 211, - 211, 211, 211, 197, 197, 230, 230, 230, 230, 230, - 230, 230, 232, 232, 232, 232, 232, 232, 232, 234, - 234, 68, 234, 234, 234, 234, 235, 235, 235, 235, - 235, 235, 235, 246, 246, 246, 246, 246, 246, 246, - 250, 250, 250, 250, 250, 250, 250, 253, 253, 253, - - 253, 253, 253, 253, 67, 67, 66, 66, 65, 65, - 64, 64, 17, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254 + 20, 20, 20, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 45, + 45, 45, 45, 45, 46, 47, 48, 49, 50, 45, + 45, 45, 45, 45, 45, 51, 52, 53, 54, 55, + 97, 55, 74, 56, 57, 58, 59, 260, 60, 85, + 214, 86, 260, 81, 61, 62, 63, 82, 87, 129, + 64, 74, 75, 76, 214, 86, 76, 98, 128, 91, + 92, 93, 86, 86, 88, 94, 130, 86, 131, 100, + 83, 75, 260, 100, 95, 134, 77, 86, 101, 102, + + 103, 84, 101, 104, 105, 89, 125, 106, 125, 107, + 125, 123, 125, 114, 107, 86, 107, 108, 115, 109, + 109, 86, 76, 127, 103, 136, 137, 260, 80, 110, + 111, 80, 111, 112, 86, 77, 184, 126, 260, 110, + 80, 80, 80, 80, 113, 117, 107, 118, 89, 118, + 119, 124, 123, 124, 138, 234, 139, 120, 142, 121, + 91, 92, 185, 86, 122, 123, 124, 118, 118, 150, + 151, 186, 78, 152, 153, 154, 250, 155, 160, 163, + 160, 163, 260, 157, 158, 140, 143, 90, 143, 159, + 144, 145, 260, 146, 260, 167, 168, 102, 150, 151, + + 162, 147, 191, 148, 191, 86, 76, 149, 86, 101, + 183, 105, 161, 164, 106, 169, 166, 260, 192, 165, + 107, 107, 86, 107, 260, 107, 260, 107, 108, 260, + 108, 108, 125, 123, 125, 152, 153, 193, 193, 145, + 188, 189, 251, 189, 190, 246, 193, 193, 193, 107, + 188, 212, 226, 111, 107, 111, 107, 193, 120, 130, + 107, 131, 138, 107, 139, 122, 123, 124, 124, 123, + 124, 157, 158, 167, 168, 152, 153, 146, 201, 155, + 201, 157, 158, 160, 163, 160, 163, 159, 209, 210, + 167, 168, 214, 149, 215, 202, 203, 203, 191, 189, + + 191, 189, 100, 209, 210, 214, 246, 215, 216, 234, + 169, 101, 260, 216, 260, 216, 205, 161, 164, 225, + 225, 244, 245, 201, 260, 201, 260, 243, 260, 247, + 189, 260, 189, 227, 255, 228, 228, 191, 246, 191, + 202, 203, 203, 214, 228, 229, 230, 80, 111, 112, + 214, 234, 225, 225, 242, 229, 80, 80, 80, 80, + 80, 205, 204, 189, 239, 189, 216, 191, 254, 191, + 254, 216, 237, 216, 227, 253, 227, 227, 254, 260, + 254, 260, 254, 235, 254, 227, 248, 249, 234, 189, + 190, 254, 233, 254, 232, 234, 248, 254, 231, 254, + + 224, 223, 257, 222, 179, 221, 220, 219, 218, 211, + 208, 206, 198, 197, 142, 141, 140, 139, 196, 135, + 257, 134, 133, 132, 258, 258, 258, 258, 258, 258, + 131, 195, 128, 127, 120, 258, 258, 258, 258, 258, + 258, 260, 126, 260, 194, 116, 115, 80, 187, 104, + 101, 99, 98, 97, 96, 182, 181, 92, 180, 126, + 87, 86, 85, 84, 257, 179, 82, 81, 258, 258, + 258, 258, 258, 258, 260, 79, 178, 75, 177, 258, + 258, 258, 258, 258, 258, 65, 65, 65, 65, 65, + 65, 65, 80, 176, 80, 118, 118, 118, 199, 199, + + 175, 199, 199, 199, 199, 200, 200, 174, 200, 200, + 200, 200, 204, 173, 172, 171, 204, 204, 207, 207, + 170, 207, 207, 207, 207, 213, 213, 213, 213, 213, + 213, 213, 217, 217, 156, 217, 217, 217, 217, 203, + 203, 236, 236, 236, 236, 236, 236, 236, 238, 238, + 238, 238, 238, 238, 238, 240, 240, 141, 240, 240, + 240, 240, 241, 241, 241, 241, 241, 241, 241, 252, + 252, 252, 252, 252, 252, 252, 256, 256, 256, 256, + 256, 256, 256, 259, 259, 259, 259, 259, 259, 259, + 135, 133, 132, 260, 116, 80, 99, 96, 90, 79, + + 78, 260, 73, 73, 72, 72, 71, 71, 70, 70, + 69, 69, 68, 68, 67, 67, 66, 66, 19, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260 } ; -static const flex_int16_t yy_chk[663] = +static const flex_int16_t yy_chk[669] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, - 28, 2, 19, 2, 2, 2, 2, 42, 2, 23, - 246, 28, 22, 22, 2, 2, 2, 22, 24, 44, - 2, 71, 19, 19, 239, 23, 23, 29, 42, 26, - 26, 26, 47, 24, 24, 26, 44, 29, 44, 31, - 22, 71, 32, 99, 26, 47, 19, 31, 31, 32, - - 32, 22, 33, 32, 235, 24, 99, 33, 39, 39, - 39, 36, 33, 33, 33, 34, 36, 34, 34, 36, - 41, 41, 232, 49, 49, 97, 34, 34, 34, 34, - 34, 34, 52, 39, 97, 40, 230, 34, 34, 34, - 34, 34, 34, 38, 40, 38, 50, 38, 38, 40, - 40, 40, 49, 229, 49, 38, 52, 38, 88, 88, - 102, 50, 38, 38, 38, 38, 38, 54, 54, 102, - 54, 55, 55, 55, 108, 55, 58, 60, 58, 60, - 227, 57, 57, 50, 53, 58, 53, 57, 53, 53, - 100, 53, 62, 62, 119, 119, 119, 226, 59, 53, - - 108, 53, 100, 57, 57, 53, 59, 59, 61, 103, - 58, 60, 62, 61, 103, 107, 103, 60, 61, 61, - 61, 104, 106, 104, 104, 121, 126, 121, 126, 161, - 106, 109, 106, 104, 104, 117, 104, 104, 228, 107, - 117, 107, 117, 104, 109, 109, 134, 220, 134, 118, - 182, 118, 182, 109, 109, 109, 118, 118, 118, 228, - 120, 140, 140, 161, 109, 120, 120, 120, 141, 141, - 142, 142, 141, 145, 145, 186, 142, 150, 218, 150, - 151, 151, 155, 155, 151, 156, 216, 156, 155, 158, - 158, 165, 165, 145, 150, 150, 150, 159, 207, 159, - - 162, 186, 162, 158, 183, 205, 183, 185, 185, 191, - 191, 165, 158, 206, 202, 150, 162, 206, 185, 156, - 185, 162, 192, 162, 192, 196, 198, 196, 198, 209, - 209, 159, 187, 200, 187, 187, 223, 244, 223, 219, - 219, 199, 197, 187, 187, 187, 187, 187, 187, 195, - 219, 195, 219, 210, 187, 187, 187, 187, 187, 187, - 242, 244, 242, 245, 198, 245, 195, 195, 195, 210, - 247, 250, 190, 250, 210, 249, 210, 249, 251, 253, - 251, 253, 189, 188, 184, 181, 176, 195, 221, 174, - 221, 221, 249, 172, 250, 247, 171, 170, 167, 221, - - 221, 221, 160, 221, 221, 157, 152, 144, 143, 138, - 221, 248, 137, 248, 136, 135, 133, 131, 130, 129, - 128, 127, 125, 124, 123, 116, 115, 113, 112, 111, - 105, 101, 98, 96, 248, 95, 94, 93, 248, 248, - 248, 248, 248, 248, 92, 91, 89, 87, 85, 248, - 248, 248, 248, 248, 248, 252, 84, 252, 83, 82, - 81, 80, 79, 78, 77, 76, 75, 72, 70, 69, - 68, 67, 66, 65, 64, 56, 51, 48, 252, 46, - 45, 43, 252, 252, 252, 252, 252, 252, 37, 35, - 30, 27, 25, 252, 252, 252, 252, 252, 252, 255, - - 255, 255, 255, 255, 255, 255, 256, 21, 256, 257, - 257, 257, 258, 258, 20, 258, 258, 258, 258, 259, - 259, 17, 259, 259, 259, 259, 260, 16, 15, 14, - 260, 260, 261, 261, 13, 261, 261, 261, 261, 262, - 262, 262, 262, 262, 262, 262, 263, 263, 12, 263, - 263, 263, 263, 264, 264, 265, 265, 265, 265, 265, - 265, 265, 266, 266, 266, 266, 266, 266, 266, 267, - 267, 11, 267, 267, 267, 267, 268, 268, 268, 268, - 268, 268, 268, 269, 269, 269, 269, 269, 269, 269, - 270, 270, 270, 270, 270, 270, 270, 271, 271, 271, - - 271, 271, 271, 271, 10, 9, 8, 7, 6, 5, - 4, 3, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, - 254, 254 + 30, 2, 21, 2, 2, 2, 2, 44, 2, 25, + 252, 30, 24, 24, 2, 2, 2, 24, 26, 46, + 2, 74, 21, 21, 245, 25, 25, 31, 44, 28, + 28, 28, 49, 26, 26, 28, 46, 31, 46, 33, + 24, 74, 34, 100, 28, 49, 21, 33, 33, 34, + + 34, 24, 100, 34, 35, 26, 125, 35, 125, 35, + 41, 41, 41, 38, 35, 35, 35, 36, 38, 36, + 36, 38, 43, 43, 103, 51, 51, 241, 36, 36, + 36, 36, 36, 36, 54, 41, 103, 42, 238, 36, + 36, 36, 36, 36, 36, 40, 42, 40, 52, 40, + 40, 42, 42, 42, 51, 234, 51, 40, 54, 40, + 91, 91, 105, 52, 40, 40, 40, 40, 40, 56, + 56, 105, 56, 57, 57, 57, 234, 57, 60, 62, + 60, 62, 236, 59, 59, 52, 55, 60, 55, 59, + 55, 55, 112, 55, 110, 64, 64, 102, 144, 144, + + 61, 55, 110, 55, 110, 59, 59, 55, 61, 61, + 102, 63, 60, 62, 63, 64, 63, 165, 112, 62, + 107, 63, 63, 63, 113, 107, 192, 107, 108, 111, + 108, 108, 123, 123, 123, 145, 145, 113, 113, 145, + 108, 108, 235, 108, 108, 233, 113, 113, 113, 121, + 108, 165, 192, 111, 121, 111, 121, 113, 122, 130, + 122, 130, 138, 124, 138, 122, 122, 122, 124, 124, + 124, 146, 146, 149, 149, 155, 155, 146, 154, 155, + 154, 159, 159, 160, 163, 160, 163, 159, 162, 162, + 169, 169, 166, 149, 166, 154, 154, 154, 188, 189, + + 188, 189, 162, 197, 197, 198, 232, 198, 166, 253, + 169, 162, 202, 166, 202, 166, 154, 160, 163, 191, + 191, 215, 215, 201, 204, 201, 204, 212, 226, 224, + 191, 212, 191, 193, 253, 193, 193, 229, 222, 229, + 201, 201, 201, 213, 193, 193, 193, 193, 193, 193, + 216, 250, 225, 225, 211, 193, 193, 193, 193, 193, + 193, 201, 204, 225, 208, 225, 216, 248, 251, 248, + 251, 216, 206, 216, 227, 250, 227, 227, 255, 256, + 255, 256, 257, 205, 257, 227, 227, 227, 203, 227, + 227, 259, 196, 259, 195, 255, 227, 254, 194, 254, + + 190, 186, 256, 181, 179, 176, 175, 174, 171, 164, + 161, 156, 148, 147, 142, 141, 140, 139, 137, 135, + 254, 134, 133, 132, 254, 254, 254, 254, 254, 254, + 131, 129, 128, 127, 120, 254, 254, 254, 254, 254, + 254, 258, 119, 258, 117, 116, 115, 109, 106, 104, + 101, 99, 98, 97, 96, 95, 94, 92, 90, 88, + 87, 86, 85, 84, 258, 83, 82, 81, 258, 258, + 258, 258, 258, 258, 80, 79, 78, 75, 73, 258, + 258, 258, 258, 258, 258, 261, 261, 261, 261, 261, + 261, 261, 262, 72, 262, 263, 263, 263, 264, 264, + + 71, 264, 264, 264, 264, 265, 265, 70, 265, 265, + 265, 265, 266, 69, 68, 67, 266, 266, 267, 267, + 66, 267, 267, 267, 267, 268, 268, 268, 268, 268, + 268, 268, 269, 269, 58, 269, 269, 269, 269, 270, + 270, 271, 271, 271, 271, 271, 271, 271, 272, 272, + 272, 272, 272, 272, 272, 273, 273, 53, 273, 273, + 273, 273, 274, 274, 274, 274, 274, 274, 274, 275, + 275, 275, 275, 275, 275, 275, 276, 276, 276, 276, + 276, 276, 276, 277, 277, 277, 277, 277, 277, 277, + 50, 48, 47, 45, 39, 37, 32, 29, 27, 23, + + 22, 19, 18, 17, 16, 15, 14, 13, 12, 11, + 10, 9, 8, 7, 6, 5, 4, 3, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260 } ; /* The intent behind this definition is that it'll catch @@ -919,6 +919,7 @@ static const flex_int16_t yy_chk[663] = #define python_dcomment 5 #define python_scomment 6 #define roff_comment 7 +#define lean_comment 8 #ifndef YY_NO_UNISTD_H /* Special case for "unistd.h", since it is non-ANSI. We include it way @@ -1016,7 +1017,7 @@ extern int yywrap ( yyscan_t yyscanner ); #endif #ifndef YY_NO_UNPUT - + #endif #ifndef yytext_ptr @@ -1178,7 +1179,7 @@ YY_DECL #line 56 "tokenizer.l" -#line 1182 "lex.linguist_yy.c" +#line 1183 "lex.linguist_yy.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { @@ -1206,13 +1207,13 @@ YY_DECL while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 255 ) + if ( yy_current_state >= 261 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; ++yy_cp; } - while ( yy_current_state != 254 ); + while ( yy_current_state != 260 ); yy_cp = yyg->yy_last_accepting_cpos; yy_current_state = yyg->yy_last_accepting_state; @@ -1377,20 +1378,20 @@ YY_RULE_SETUP { FEED_STATIC("COMMENT.ig"); BEGIN(roff_comment); return 1; } YY_BREAK case 23: -/* rule 23 can match eol */ YY_RULE_SETUP -#line 106 "tokenizer.l" -{ /* nothing */ } +#line 105 "tokenizer.l" +{ FEED_STATIC("COMMENT/-"); BEGIN(lean_comment); return 1; } YY_BREAK case 24: YY_RULE_SETUP -#line 107 "tokenizer.l" -{ BEGIN(INITIAL); } +#line 106 "tokenizer.l" +{ FEED_STATIC("COMMENT/-"); BEGIN(lean_comment); return 1; } YY_BREAK case 25: +/* rule 25 can match eol */ YY_RULE_SETUP #line 108 "tokenizer.l" -{ BEGIN(INITIAL); } +{ /* nothing */ } YY_BREAK case 26: YY_RULE_SETUP @@ -1413,25 +1414,25 @@ YY_RULE_SETUP { BEGIN(INITIAL); } YY_BREAK case 30: -/* rule 30 can match eol */ YY_RULE_SETUP #line 113 "tokenizer.l" { BEGIN(INITIAL); } YY_BREAK case 31: YY_RULE_SETUP -#line 115 "tokenizer.l" -{ /* nothing */ } +#line 114 "tokenizer.l" +{ BEGIN(INITIAL); } YY_BREAK case 32: +/* rule 32 can match eol */ YY_RULE_SETUP -#line 116 "tokenizer.l" -{ eat_until_unescaped('"'); } +#line 115 "tokenizer.l" +{ BEGIN(INITIAL); } YY_BREAK case 33: YY_RULE_SETUP -#line 117 "tokenizer.l" -{ eat_until_unescaped('\''); } +#line 116 "tokenizer.l" +{ BEGIN(INITIAL); } YY_BREAK case 34: YY_RULE_SETUP @@ -1440,22 +1441,22 @@ YY_RULE_SETUP YY_BREAK case 35: YY_RULE_SETUP -#line 120 "tokenizer.l" -{ FEED(); return 1; } +#line 119 "tokenizer.l" +{ eat_until_unescaped('"'); } YY_BREAK case 36: YY_RULE_SETUP -#line 122 "tokenizer.l" -{ FEED(); return 1; } +#line 120 "tokenizer.l" +{ eat_until_unescaped('\''); } YY_BREAK case 37: YY_RULE_SETUP -#line 123 "tokenizer.l" -{ FEED(); return 1; } +#line 121 "tokenizer.l" +{ /* nothing */ } YY_BREAK case 38: YY_RULE_SETUP -#line 124 "tokenizer.l" +#line 123 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 39: @@ -1480,42 +1481,42 @@ YY_RULE_SETUP YY_BREAK case 43: YY_RULE_SETUP -#line 130 "tokenizer.l" +#line 129 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 44: YY_RULE_SETUP -#line 132 "tokenizer.l" +#line 130 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 45: YY_RULE_SETUP -#line 133 "tokenizer.l" +#line 131 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 46: YY_RULE_SETUP -#line 135 "tokenizer.l" +#line 133 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 47: YY_RULE_SETUP -#line 136 "tokenizer.l" +#line 135 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 48: YY_RULE_SETUP -#line 138 "tokenizer.l" +#line 136 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 49: YY_RULE_SETUP -#line 139 "tokenizer.l" +#line 138 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 50: YY_RULE_SETUP -#line 140 "tokenizer.l" +#line 139 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 51: @@ -1545,32 +1546,32 @@ YY_RULE_SETUP YY_BREAK case 56: YY_RULE_SETUP -#line 147 "tokenizer.l" +#line 146 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 57: YY_RULE_SETUP -#line 148 "tokenizer.l" +#line 147 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 58: YY_RULE_SETUP -#line 149 "tokenizer.l" +#line 148 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 59: YY_RULE_SETUP -#line 151 "tokenizer.l" +#line 150 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 60: YY_RULE_SETUP -#line 152 "tokenizer.l" +#line 151 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 61: YY_RULE_SETUP -#line 153 "tokenizer.l" +#line 152 "tokenizer.l" { FEED(); return 1; } YY_BREAK case 62: @@ -1654,17 +1655,32 @@ YY_RULE_SETUP { FEED(); return 1; } YY_BREAK case 78: -/* rule 78 can match eol */ +YY_RULE_SETUP +#line 170 "tokenizer.l" +{ FEED(); return 1; } + YY_BREAK +case 79: YY_RULE_SETUP #line 171 "tokenizer.l" +{ FEED(); return 1; } + YY_BREAK +case 80: +YY_RULE_SETUP +#line 172 "tokenizer.l" +{ FEED(); return 1; } + YY_BREAK +case 81: +/* rule 81 can match eol */ +YY_RULE_SETUP +#line 174 "tokenizer.l" { /* nothing */ } YY_BREAK -case 79: +case 82: YY_RULE_SETUP -#line 173 "tokenizer.l" +#line 176 "tokenizer.l" YY_FATAL_ERROR( "flex scanner jammed" ); YY_BREAK -#line 1668 "lex.linguist_yy.c" +#line 1684 "lex.linguist_yy.c" case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(c_comment): case YY_STATE_EOF(xml_comment): @@ -1673,6 +1689,7 @@ case YY_STATE_EOF(ocaml_comment): case YY_STATE_EOF(python_dcomment): case YY_STATE_EOF(python_scomment): case YY_STATE_EOF(roff_comment): +case YY_STATE_EOF(lean_comment): yyterminate(); case YY_END_OF_BUFFER: @@ -1972,7 +1989,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 255 ) + if ( yy_current_state >= 261 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; @@ -2001,11 +2018,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 255 ) + if ( yy_current_state >= 261 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - yy_is_jam = (yy_current_state == 254); + yy_is_jam = (yy_current_state == 260); (void)yyg; return yy_is_jam ? 0 : yy_current_state; @@ -2165,7 +2182,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner) YY_BUFFER_STATE yy_create_buffer (FILE * file, int size , yyscan_t yyscanner) { YY_BUFFER_STATE b; - + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) , yyscanner ); if ( ! b ) YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); @@ -2231,7 +2248,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner) } b->yy_is_interactive = 0; - + errno = oerrno; } @@ -2373,7 +2390,7 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscanner) { YY_BUFFER_STATE b; - + if ( size < 2 || base[size-2] != YY_END_OF_BUFFER_CHAR || base[size-1] != YY_END_OF_BUFFER_CHAR ) @@ -2409,7 +2426,7 @@ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscann */ YY_BUFFER_STATE yy_scan_string (const char * yystr , yyscan_t yyscanner) { - + return yy_scan_bytes( yystr, (int) strlen(yystr) , yyscanner); } @@ -2426,7 +2443,7 @@ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan char *buf; yy_size_t n; int i; - + /* Get memory for full buffer, including space for trailing EOB's. */ n = (yy_size_t) (_yybytes_len + 2); buf = (char *) yyalloc( n , yyscanner ); @@ -2499,7 +2516,7 @@ int yyget_lineno (yyscan_t yyscanner) if (! YY_CURRENT_BUFFER) return 0; - + return yylineno; } @@ -2512,7 +2529,7 @@ int yyget_column (yyscan_t yyscanner) if (! YY_CURRENT_BUFFER) return 0; - + return yycolumn; } @@ -2574,7 +2591,7 @@ void yyset_lineno (int _line_number , yyscan_t yyscanner) /* lineno is only valid if an input buffer exists. */ if (! YY_CURRENT_BUFFER ) YY_FATAL_ERROR( "yyset_lineno called with no buffer" ); - + yylineno = _line_number; } @@ -2589,7 +2606,7 @@ void yyset_column (int _column_no , yyscan_t yyscanner) /* column is only valid if an input buffer exists. */ if (! YY_CURRENT_BUFFER ) YY_FATAL_ERROR( "yyset_column called with no buffer" ); - + yycolumn = _column_no; } @@ -2806,4 +2823,6 @@ void yyfree (void * ptr , yyscan_t yyscanner) #define YYTABLES_NAME "yytables" -#line 173 "tokenizer.l" +#line 176 "tokenizer.l" + + diff --git a/ext/linguist/lex.linguist_yy.h b/ext/linguist/lex.linguist_yy.h index 9e02bc01df..f82ca5065b 100644 --- a/ext/linguist/lex.linguist_yy.h +++ b/ext/linguist/lex.linguist_yy.h @@ -246,7 +246,7 @@ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, - * if you want the limit (max/min) macros for int types. + * if you want the limit (max/min) macros for int types. */ #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS 1 @@ -263,7 +263,7 @@ typedef uint32_t flex_uint32_t; typedef signed char flex_int8_t; typedef short int flex_int16_t; typedef int flex_int32_t; -typedef unsigned char flex_uint8_t; +typedef unsigned char flex_uint8_t; typedef unsigned short int flex_uint16_t; typedef unsigned int flex_uint32_t; @@ -435,6 +435,7 @@ void yyfree ( void * , yyscan_t yyscanner ); #define python_dcomment 5 #define python_scomment 6 #define roff_comment 7 +#define lean_comment 8 #endif @@ -694,9 +695,9 @@ extern int yylex (yyscan_t yyscanner); #undef yyTABLES_NAME #endif -#line 173 "tokenizer.l" +#line 176 "tokenizer.l" -#line 701 "lex.linguist_yy.h" +#line 702 "lex.linguist_yy.h" #undef linguist_yyIN_HEADER #endif /* linguist_yyHEADER_H */ diff --git a/ext/linguist/tokenizer.l b/ext/linguist/tokenizer.l index 25762c0bc3..33e15c276e 100644 --- a/ext/linguist/tokenizer.l +++ b/ext/linguist/tokenizer.l @@ -51,7 +51,7 @@ %} %option never-interactive yywrap reentrant nounput warn nodefault header-file="lex.linguist_yy.h" extra-type="VALUE*" prefix="linguist_yy" -%x c_comment xml_comment haskell_comment ocaml_comment python_dcomment python_scomment roff_comment +%x c_comment xml_comment haskell_comment ocaml_comment python_dcomment python_scomment roff_comment lean_comment %% @@ -102,8 +102,10 @@ "\"\"\"" { FEED_STATIC("COMMENT\"\"\""); BEGIN(python_dcomment); return 1; } "'''" { FEED_STATIC("COMMENT'''"); BEGIN(python_scomment); return 1; } ^".ig"\n { FEED_STATIC("COMMENT.ig"); BEGIN(roff_comment); return 1; } +"/--" { FEED_STATIC("COMMENT/-"); BEGIN(lean_comment); return 1; } +"/-" { FEED_STATIC("COMMENT/-"); BEGIN(lean_comment); return 1; } -.|\n { /* nothing */ } +.|\n { /* nothing */ } "*/" { BEGIN(INITIAL); } "-->" { BEGIN(INITIAL); } "-}" { BEGIN(INITIAL); } @@ -111,6 +113,7 @@ "\"\"\"" { BEGIN(INITIAL); } "'''" { BEGIN(INITIAL); } ".."\n { BEGIN(INITIAL); } +"-/" { BEGIN(INITIAL); } \"\"|'' { /* nothing */ } \" { eat_until_unescaped('"'); } diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 647640544c..c214f5d2d2 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -84,6 +84,29 @@ def test_comments assert_equal %w(foo COMMENT""" bar), tokenize("foo\n\"\"\"\nComment\n\"\"\"\nbar") assert_equal %w(foo COMMENT''' bar), tokenize("foo\n'''\nComment\n'''\nbar") + # Lean comments + assert_equal %w(foo COMMENT/-), tokenize("foo /- Comment -/") + assert_equal %w(foo COMMENT/-), tokenize("foo /-Comment-/") + assert_equal %w(foo COMMENT/-), tokenize("foo /- \nComment\n -/") + assert_equal %w(foo COMMENT/-), tokenize("foo /-\nComment\n-/") + assert_equal %w(foo COMMENT/-), tokenize("foo /-- Comment -/") + assert_equal %w(foo COMMENT/-), tokenize("foo /--Comment-/") + assert_equal %w(foo COMMENT/-), tokenize("foo /--\nComment\n-/") + assert_equal %w(foo COMMENT/-), tokenize("foo /-! Comment -/") + assert_equal %w(foo COMMENT/-), tokenize("foo /-!Comment-/") + assert_equal %w(foo COMMENT/-), tokenize("foo /-!\nComment\n-/") + assert_equal %w(foo COMMENT/- bar), tokenize("foo /-\nComment\n-/ bar") + assert_equal %w(foo COMMENT/- bar), tokenize("foo /-\nComment\n-/\nbar") + assert_equal %w(foo COMMENT/- bar), tokenize("foo\n/-\nComment\n-/\nbar") + # Nested comments are not processed correctly as it's rarely used and adds unnecessary complexity + assert_equal %w(foo COMMENT/- comment), tokenize("foo /- Comment /- Still Comment /- And Still Comment -/ comment") + assert_equal %w(foo COMMENT/- comment3 - / bar), tokenize("foo /- comment1 /- comment2 -/ comment3 -/ bar") + assert_equal %w(COMMENT/-), tokenize("/-\n*\n-/") + assert_equal %w(COMMENT/-), tokenize("/-*-/") + assert_equal %w(COMMENT/-), tokenize("/-**-/") + assert_equal %w(COMMENT/-), tokenize("/-!-/") + assert_equal %w(COMMENT/-), tokenize("/--/ -/") + # Roff comments assert_equal %w(COMMENT.\\" bar), tokenize(".\\\" foo\nbar") assert_equal %w(COMMENT.\\" bar), tokenize(". \\\" foo\nbar") @@ -210,7 +233,7 @@ def test_operators assert_equal %w(- ! # $ % & * + , . : ; <=>), tokenize("-!#$%&*+,.:;<=>") assert_equal %w(- ! # $ % & ? @ \\ ^ _ ` | ~), tokenize("-!#$%&?@\\^_`|~") assert_equal %w(- ! # $ % & * + , . : ; <=>), tokenize("-!#$%&*+,.:;<=>") - assert_equal %w(/ - ! # $ % & * + , . : ; <>), tokenize("/-!#$%&*+,.:;<>") + assert_equal %w(- / ! # $ % & * + , . : ; <>), tokenize("-/!#$%&*+,.:;<>") end def test_c_tokens