From cfc9ecf91517eb021cb04570e420e4fb1818b9c8 Mon Sep 17 00:00:00 2001 From: Junichi Kobayashi Date: Mon, 16 Oct 2023 01:18:48 +0900 Subject: [PATCH] Replace NewParser with Parser #3 - Enable to parse colon-less rules --- lib/lrama/new_lexer.rb | 9 +- lib/lrama/new_parser.rb | 386 +++++++++++++++++----------------- parser.y | 2 +- spec/lrama/new_parser_spec.rb | 2 +- 4 files changed, 204 insertions(+), 195 deletions(-) diff --git a/lib/lrama/new_lexer.rb b/lib/lrama/new_lexer.rb index 0588dc43..02c0614c 100644 --- a/lib/lrama/new_lexer.rb +++ b/lib/lrama/new_lexer.rb @@ -92,7 +92,14 @@ def lex_token when @scanner.scan(/\d+/) return [:INTEGER, Integer(@scanner.matched)] when @scanner.scan(/([a-zA-Z_.][-a-zA-Z0-9_.]*)/) - return [:IDENTIFIER, build_token(type: Token::Ident, s_value: @scanner.matched)] + token = build_token(type: Token::Ident, s_value: @scanner.matched) + type = + if @scanner.check(/\s*(\[\s*[a-zA-Z_.][-a-zA-Z0-9_.]*\s*\])?\s*:/) + :IDENT_COLON + else + :IDENTIFIER + end + return [type, token] else raise end diff --git a/lib/lrama/new_parser.rb b/lib/lrama/new_parser.rb index 25041b89..49797dd3 100644 --- a/lib/lrama/new_parser.rb +++ b/lib/lrama/new_parser.rb @@ -33,74 +33,74 @@ def next_token ##### State transition tables begin ### racc_action_table = [ - 81, 64, 131, 64, 6, 33, 34, 7, 3, 133, - 55, 33, 34, 75, 33, 34, 130, 67, 33, 34, - 64, 80, 33, 34, 33, 34, 134, 64, 82, 20, - 22, 23, 24, 25, 26, 27, 28, 29, 64, 33, - 34, 33, 34, 67, 33, 34, 133, 60, 33, 34, - 64, 67, 33, 34, 33, 34, 33, 34, 9, 33, - 34, 8, 14, 134, 12, 64, 13, 15, 16, 17, - 18, 33, 34, 19, 20, 22, 23, 24, 25, 26, - 27, 28, 29, 64, 31, 64, 33, 34, 64, 33, - 34, 64, 33, 34, 64, 33, 34, 64, 33, 34, - 64, 33, 34, 64, 33, 34, 33, 34, 152, 33, - 34, 152, 33, 34, 152, 33, 34, 152, 33, 34, - 152, 33, 34, 152, 33, 34, 20, 22, 23, 24, - 25, 26, 27, 28, 29, 40, 33, 34, 33, 34, - 33, 34, 33, 34, 33, 34, 43, 45, 45, 45, - 48, 49, 50, 51, 52, 72, 76, 78, 83, 83, - 83, 90, 94, 95, 103, 104, 106, 107, 108, 109, - 110, 111, 115, 117, 118, 121, 122, 123, 137, 138, - 139, 140, 141, 142, 121, 78, 147, 148, 155, 159, - 160, 78, 78 ] + 75, 131, 40, 41, 6, 81, 64, 7, 55, 40, + 41, 67, 40, 41, 133, 130, 20, 22, 23, 24, + 25, 26, 27, 28, 29, 64, 80, 40, 41, 64, + 3, 134, 8, 37, 82, 40, 41, 133, 67, 40, + 41, 64, 67, 40, 41, 60, 40, 41, 31, 9, + 40, 41, 38, 14, 134, 12, 64, 13, 15, 16, + 17, 18, 40, 41, 19, 20, 22, 23, 24, 25, + 26, 27, 28, 29, 64, 43, 64, 40, 41, 64, + 45, 64, 40, 41, 64, 40, 41, 64, 40, 41, + 64, 40, 41, 64, 45, 40, 41, 40, 41, 64, + 40, 41, 40, 41, 152, 40, 41, 152, 40, 41, + 152, 40, 41, 152, 40, 41, 64, 40, 41, 152, + 40, 41, 20, 22, 23, 24, 25, 26, 27, 28, + 29, 40, 41, 40, 41, 45, 152, 40, 41, 37, + 40, 41, 40, 41, 40, 41, 40, 41, 48, 49, + 50, 51, 52, 72, 76, 78, 83, 83, 83, 90, + 94, 95, 103, 104, 106, 107, 108, 109, 110, 111, + 115, 117, 118, 121, 122, 123, 137, 138, 139, 140, + 141, 142, 121, 78, 147, 148, 155, 159, 160, 78, + 78 ] racc_action_check = [ - 41, 120, 119, 59, 2, 13, 13, 2, 1, 120, - 25, 25, 25, 32, 54, 54, 119, 27, 27, 27, - 60, 41, 120, 120, 59, 59, 120, 26, 41, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 143, 32, - 32, 60, 60, 28, 28, 28, 143, 26, 26, 26, - 95, 29, 29, 29, 55, 55, 65, 65, 4, 143, - 143, 3, 4, 143, 4, 97, 4, 4, 4, 4, - 4, 95, 95, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 116, 7, 134, 97, 97, 140, 66, - 66, 141, 67, 67, 142, 70, 70, 149, 71, 71, - 153, 90, 90, 154, 116, 116, 134, 134, 140, 140, - 140, 141, 141, 141, 142, 142, 142, 149, 149, 149, - 153, 153, 153, 154, 154, 154, 9, 9, 9, 9, - 9, 9, 9, 9, 9, 12, 9, 9, 92, 92, - 98, 98, 100, 100, 112, 112, 14, 15, 16, 17, - 18, 20, 22, 23, 24, 30, 37, 38, 44, 46, - 47, 53, 57, 58, 77, 78, 84, 85, 86, 87, - 88, 89, 93, 101, 102, 103, 104, 105, 124, 125, - 126, 127, 128, 129, 130, 132, 135, 136, 145, 156, - 158, 159, 160 ] + 32, 119, 13, 13, 2, 39, 120, 2, 25, 25, + 25, 27, 27, 27, 120, 119, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 26, 39, 120, 120, 143, + 1, 120, 3, 32, 39, 54, 54, 143, 28, 28, + 28, 59, 29, 29, 29, 26, 26, 26, 7, 4, + 143, 143, 12, 4, 143, 4, 60, 4, 4, 4, + 4, 4, 59, 59, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 95, 14, 97, 60, 60, 116, + 15, 134, 55, 55, 140, 65, 65, 141, 66, 66, + 142, 67, 67, 149, 16, 95, 95, 97, 97, 153, + 116, 116, 134, 134, 140, 140, 140, 141, 141, 141, + 142, 142, 142, 149, 149, 149, 154, 70, 70, 153, + 153, 153, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 71, 71, 90, 90, 17, 154, 154, 154, 9, + 92, 92, 98, 98, 100, 100, 112, 112, 18, 20, + 22, 23, 24, 30, 35, 36, 44, 46, 47, 53, + 57, 58, 77, 78, 84, 85, 86, 87, 88, 89, + 93, 101, 102, 103, 104, 105, 124, 125, 126, 127, + 128, 129, 130, 132, 135, 136, 145, 156, 158, 159, + 160 ] racc_action_pointer = [ - nil, 8, 1, 61, 56, nil, nil, 77, nil, 108, - nil, nil, 126, -23, 139, 132, 133, 134, 135, nil, - 136, nil, 137, 138, 139, -17, 20, -10, 16, 24, - 151, nil, 11, nil, nil, nil, nil, 139, 124, nil, - nil, -7, nil, nil, 143, nil, 144, 145, nil, nil, - nil, nil, nil, 134, -14, 26, nil, 153, 136, -4, - 13, nil, nil, nil, nil, 28, 61, 64, nil, nil, - 67, 70, nil, nil, nil, nil, nil, 134, 137, nil, - nil, nil, nil, nil, 162, 163, 164, 165, 166, 167, - 73, nil, 110, 165, nil, 43, nil, 58, 112, nil, - 114, 168, 170, 160, 142, 173, nil, nil, nil, nil, - nil, nil, 116, nil, nil, nil, 76, nil, nil, -15, - -6, nil, nil, nil, 162, 163, 164, 165, 166, 167, - 169, nil, 152, nil, 78, 182, 171, nil, nil, nil, - 81, 84, 87, 31, nil, 184, nil, nil, nil, 90, - nil, nil, nil, 93, 96, nil, 173, nil, 174, 158, - 159, nil, nil ] + nil, 30, 1, 32, 47, nil, nil, 41, nil, 104, + nil, nil, 43, -26, 68, 65, 79, 120, 133, nil, + 134, nil, 135, 136, 137, -19, 18, -16, 11, 15, + 149, nil, -2, nil, nil, 137, 122, nil, nil, -2, + nil, nil, nil, nil, 141, nil, 142, 143, nil, nil, + nil, nil, nil, 132, 7, 54, nil, 151, 134, 34, + 49, nil, nil, nil, nil, 57, 60, 63, nil, nil, + 89, 103, nil, nil, nil, nil, nil, 132, 135, nil, + nil, nil, nil, nil, 160, 161, 162, 163, 164, 165, + 105, nil, 112, 163, nil, 67, nil, 69, 114, nil, + 116, 166, 168, 158, 140, 171, nil, nil, nil, nil, + nil, nil, 118, nil, nil, nil, 72, nil, nil, -16, + -1, nil, nil, nil, 160, 161, 162, 163, 164, 165, + 167, nil, 150, nil, 74, 180, 169, nil, nil, nil, + 77, 80, 83, 22, nil, 182, nil, nil, nil, 86, + nil, nil, nil, 92, 109, nil, 171, nil, 172, 156, + 157, nil, nil ] racc_action_default = [ -2, -104, -8, -104, -104, -3, -4, -104, 163, -104, -9, -10, -104, -104, -104, -104, -104, -104, -104, -20, -104, -24, -104, -104, -104, -104, -104, -104, -104, -104, - -104, -7, -91, -69, -70, -71, -73, -104, -88, -90, - -11, -95, -94, -13, -14, -60, -15, -16, -17, -21, + -104, -7, -91, -71, -73, -104, -88, -90, -11, -95, + -69, -70, -94, -13, -14, -60, -15, -16, -17, -21, -25, -28, -31, -34, -39, -104, -42, -45, -35, -50, -104, -53, -55, -56, -103, -36, -63, -104, -66, -68, -37, -38, -5, -1, -72, -92, -74, -104, -104, -12, @@ -115,156 +115,156 @@ def next_token -88, -86, -83 ] racc_goto_table = [ - 77, 61, 59, 91, 120, 99, 35, 39, 54, 157, - 98, 42, 100, 157, 157, 98, 98, 44, 46, 47, - 149, 153, 154, 57, 1, 69, 69, 69, 2, 74, - 39, 143, 4, 32, 96, 61, 97, 99, 92, 99, - 73, 91, 65, 70, 71, 5, 30, 101, 10, 11, - 41, 79, 57, 57, 85, 125, 86, 126, 87, 127, - 88, 91, 128, 69, 69, 69, 89, 129, 69, 69, - 61, 116, 96, 112, 53, 58, 93, 113, 114, 105, - 136, 84, 124, 119, 145, 158, 135, 156, 57, 102, - 57, 96, nil, nil, 144, 132, 69, nil, 69, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, 146, - 57, nil, nil, nil, nil, nil, nil, nil, 132, nil, - nil, 161, 162 ] + 77, 61, 42, 54, 59, 99, 33, 91, 157, 120, + 1, 2, 157, 157, 57, 4, 69, 69, 69, 98, + 32, 100, 73, 5, 98, 98, 44, 46, 47, 74, + 149, 153, 154, 92, 96, 61, 143, 99, 97, 99, + 65, 70, 71, 57, 57, 91, 30, 101, 10, 11, + 39, 79, 85, 125, 69, 69, 69, 86, 126, 69, + 69, 87, 127, 88, 128, 91, 89, 129, 112, 53, + 61, 58, 96, 116, 93, 113, 114, 105, 136, 57, + 84, 57, 124, 119, 145, 158, 135, 69, 156, 69, + 102, 96, nil, nil, nil, 132, 144, nil, nil, nil, + nil, 57, nil, nil, nil, nil, nil, nil, nil, 146, + nil, nil, nil, nil, nil, nil, nil, nil, 132, nil, + nil, nil, nil, 161, 162 ] racc_goto_check = [ - 46, 36, 35, 30, 48, 42, 43, 31, 29, 54, - 41, 31, 41, 54, 54, 41, 41, 13, 13, 13, - 17, 17, 17, 31, 1, 31, 31, 31, 2, 43, - 31, 48, 3, 4, 36, 36, 35, 42, 29, 42, - 5, 30, 28, 28, 28, 6, 7, 8, 9, 10, - 11, 12, 31, 31, 14, 15, 18, 19, 20, 21, - 22, 30, 23, 31, 31, 31, 24, 25, 31, 31, - 36, 35, 36, 29, 26, 27, 32, 33, 34, 37, - 38, 39, 40, 47, 49, 50, 51, 52, 31, 53, - 31, 36, nil, nil, 46, 36, 31, nil, 31, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, 36, - 31, nil, nil, nil, nil, nil, nil, nil, 36, nil, - nil, 46, 46 ] + 46, 36, 31, 29, 35, 42, 43, 30, 54, 48, + 1, 2, 54, 54, 31, 3, 31, 31, 31, 41, + 4, 41, 5, 6, 41, 41, 13, 13, 13, 43, + 17, 17, 17, 29, 36, 36, 48, 42, 35, 42, + 28, 28, 28, 31, 31, 30, 7, 8, 9, 10, + 11, 12, 14, 15, 31, 31, 31, 18, 19, 31, + 31, 20, 21, 22, 23, 30, 24, 25, 29, 26, + 36, 27, 36, 35, 32, 33, 34, 37, 38, 31, + 39, 31, 40, 47, 49, 50, 51, 31, 52, 31, + 53, 36, nil, nil, nil, 36, 46, nil, nil, nil, + nil, 31, nil, nil, nil, nil, nil, nil, nil, 36, + nil, nil, nil, nil, nil, nil, nil, nil, 36, nil, + nil, nil, nil, 46, 46 ] racc_goto_pointer = [ - nil, 24, 28, 30, 24, 8, 43, 40, -25, 44, - 45, 37, 10, 2, 6, -52, nil, -120, 7, -51, - 8, -50, 9, -48, 14, -44, 49, 49, 15, -17, - -51, -2, 19, -16, -15, -24, -25, -4, -43, 36, - -24, -55, -61, -3, nil, nil, -38, -20, -99, -49, - -70, -35, -60, 14, -140 ] + nil, 10, 11, 13, 11, -10, 21, 40, -25, 44, + 45, 37, 12, 11, 4, -54, nil, -110, 8, -50, + 11, -47, 12, -46, 14, -44, 44, 45, 13, -22, + -47, -11, 17, -18, -17, -22, -25, -6, -45, 35, + -24, -46, -61, -3, nil, nil, -36, -20, -94, -49, + -70, -35, -59, 15, -141 ] racc_goto_default = [ nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - 37, nil, nil, nil, nil, nil, 21, nil, nil, nil, + 35, nil, nil, nil, nil, nil, 21, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 56, 62, nil, nil, 63, nil, 151, nil, nil, nil, - nil, 66, 68, nil, 36, 38, nil, nil, nil, nil, + nil, 66, 68, nil, 34, 36, nil, nil, nil, nil, nil, nil, nil, nil, 150 ] racc_reduce_table = [ 0, 0, :racc_error, - 5, 37, :_reduce_none, - 0, 38, :_reduce_none, - 2, 38, :_reduce_none, - 0, 43, :_reduce_4, - 0, 44, :_reduce_5, - 5, 42, :_reduce_6, - 2, 42, :_reduce_none, - 0, 39, :_reduce_8, + 5, 38, :_reduce_none, + 0, 39, :_reduce_none, 2, 39, :_reduce_none, - 1, 45, :_reduce_none, - 2, 45, :_reduce_11, - 3, 45, :_reduce_none, - 2, 45, :_reduce_none, - 2, 45, :_reduce_none, - 2, 45, :_reduce_15, - 2, 45, :_reduce_16, - 0, 50, :_reduce_17, - 0, 51, :_reduce_18, - 6, 45, :_reduce_19, - 1, 45, :_reduce_none, - 0, 54, :_reduce_21, - 0, 55, :_reduce_22, - 6, 46, :_reduce_23, + 0, 44, :_reduce_4, + 0, 45, :_reduce_5, + 5, 43, :_reduce_6, + 2, 43, :_reduce_none, + 0, 40, :_reduce_8, + 2, 40, :_reduce_none, 1, 46, :_reduce_none, - 0, 56, :_reduce_25, - 0, 57, :_reduce_26, - 7, 46, :_reduce_none, - 0, 58, :_reduce_28, - 0, 59, :_reduce_29, - 7, 46, :_reduce_30, - 0, 60, :_reduce_31, - 0, 61, :_reduce_32, - 7, 46, :_reduce_33, - 2, 52, :_reduce_none, - 2, 52, :_reduce_35, - 2, 52, :_reduce_36, - 2, 52, :_reduce_37, - 2, 52, :_reduce_38, - 1, 62, :_reduce_39, - 2, 62, :_reduce_40, - 3, 62, :_reduce_41, - 1, 65, :_reduce_42, - 2, 65, :_reduce_43, - 3, 66, :_reduce_44, - 0, 68, :_reduce_none, - 1, 68, :_reduce_none, + 2, 46, :_reduce_11, + 3, 46, :_reduce_none, + 2, 46, :_reduce_none, + 2, 46, :_reduce_none, + 2, 46, :_reduce_15, + 2, 46, :_reduce_16, + 0, 51, :_reduce_17, + 0, 52, :_reduce_18, + 6, 46, :_reduce_19, + 1, 46, :_reduce_none, + 0, 55, :_reduce_21, + 0, 56, :_reduce_22, + 6, 47, :_reduce_23, + 1, 47, :_reduce_none, + 0, 57, :_reduce_25, + 0, 58, :_reduce_26, + 7, 47, :_reduce_none, + 0, 59, :_reduce_28, + 0, 60, :_reduce_29, + 7, 47, :_reduce_30, + 0, 61, :_reduce_31, + 0, 62, :_reduce_32, + 7, 47, :_reduce_33, + 2, 53, :_reduce_none, + 2, 53, :_reduce_35, + 2, 53, :_reduce_36, + 2, 53, :_reduce_37, + 2, 53, :_reduce_38, + 1, 63, :_reduce_39, + 2, 63, :_reduce_40, + 3, 63, :_reduce_41, + 1, 66, :_reduce_42, + 2, 66, :_reduce_43, + 3, 67, :_reduce_44, 0, 69, :_reduce_none, 1, 69, :_reduce_none, - 1, 69, :_reduce_none, - 1, 63, :_reduce_50, - 2, 63, :_reduce_51, - 3, 63, :_reduce_52, - 1, 71, :_reduce_53, - 2, 71, :_reduce_54, - 1, 72, :_reduce_none, - 1, 72, :_reduce_none, - 0, 73, :_reduce_57, - 0, 74, :_reduce_58, - 6, 49, :_reduce_59, - 0, 75, :_reduce_60, - 0, 76, :_reduce_61, - 5, 49, :_reduce_62, - 1, 64, :_reduce_63, - 2, 64, :_reduce_64, - 2, 64, :_reduce_65, - 1, 77, :_reduce_66, - 2, 77, :_reduce_67, - 1, 78, :_reduce_none, - 1, 67, :_reduce_none, - 1, 67, :_reduce_none, - 1, 40, :_reduce_none, - 2, 40, :_reduce_none, + 0, 70, :_reduce_none, + 1, 70, :_reduce_none, + 1, 70, :_reduce_none, + 1, 64, :_reduce_50, + 2, 64, :_reduce_51, + 3, 64, :_reduce_52, + 1, 72, :_reduce_53, + 2, 72, :_reduce_54, + 1, 73, :_reduce_none, + 1, 73, :_reduce_none, + 0, 74, :_reduce_57, + 0, 75, :_reduce_58, + 6, 50, :_reduce_59, + 0, 76, :_reduce_60, + 0, 77, :_reduce_61, + 5, 50, :_reduce_62, + 1, 65, :_reduce_63, + 2, 65, :_reduce_64, + 2, 65, :_reduce_65, + 1, 78, :_reduce_66, + 2, 78, :_reduce_67, 1, 79, :_reduce_none, - 2, 79, :_reduce_none, - 4, 80, :_reduce_75, - 1, 83, :_reduce_76, - 3, 83, :_reduce_77, - 2, 83, :_reduce_none, - 0, 84, :_reduce_79, - 3, 84, :_reduce_80, - 0, 85, :_reduce_81, - 0, 86, :_reduce_82, - 7, 84, :_reduce_83, - 0, 87, :_reduce_84, - 0, 88, :_reduce_85, - 6, 84, :_reduce_86, - 3, 84, :_reduce_87, - 0, 82, :_reduce_none, - 3, 82, :_reduce_89, - 1, 81, :_reduce_none, - 0, 41, :_reduce_none, - 0, 89, :_reduce_92, - 3, 41, :_reduce_93, - 1, 47, :_reduce_none, - 0, 48, :_reduce_none, - 1, 48, :_reduce_none, - 1, 48, :_reduce_none, + 1, 68, :_reduce_none, + 1, 68, :_reduce_none, + 1, 41, :_reduce_none, + 2, 41, :_reduce_none, + 1, 80, :_reduce_none, + 2, 80, :_reduce_none, + 4, 81, :_reduce_75, + 1, 84, :_reduce_76, + 3, 84, :_reduce_77, + 2, 84, :_reduce_none, + 0, 85, :_reduce_79, + 3, 85, :_reduce_80, + 0, 86, :_reduce_81, + 0, 87, :_reduce_82, + 7, 85, :_reduce_83, + 0, 88, :_reduce_84, + 0, 89, :_reduce_85, + 6, 85, :_reduce_86, + 3, 85, :_reduce_87, + 0, 83, :_reduce_none, + 3, 83, :_reduce_89, + 1, 82, :_reduce_none, + 0, 42, :_reduce_none, + 0, 90, :_reduce_92, + 3, 42, :_reduce_93, 1, 48, :_reduce_none, - 1, 53, :_reduce_99, - 2, 53, :_reduce_100, - 1, 90, :_reduce_none, - 1, 90, :_reduce_none, - 1, 70, :_reduce_103 ] + 0, 49, :_reduce_none, + 1, 49, :_reduce_none, + 1, 49, :_reduce_none, + 1, 49, :_reduce_none, + 1, 54, :_reduce_99, + 2, 54, :_reduce_100, + 1, 91, :_reduce_none, + 1, 91, :_reduce_none, + 1, 71, :_reduce_103 ] racc_reduce_n = 104 @@ -306,9 +306,10 @@ def next_token "%prec" => 32, "[" => 33, "]" => 34, - "{...}" => 35 } + :IDENT_COLON => 35, + "{...}" => 36 } -racc_nt_base = 36 +racc_nt_base = 37 racc_use_result_var = true @@ -365,6 +366,7 @@ def next_token "\"%prec\"", "\"[\"", "\"]\"", + "IDENT_COLON", "\"{...}\"", "$start", "input", diff --git a/parser.y b/parser.y index 10096e6b..b79b8645 100644 --- a/parser.y +++ b/parser.y @@ -328,7 +328,7 @@ rule named_ref_opt: # empty | '[' IDENTIFIER ']' { result = val[1].s_value } - id_colon: id + id_colon: IDENT_COLON epilogue_opt: # empty | "%%" diff --git a/spec/lrama/new_parser_spec.rb b/spec/lrama/new_parser_spec.rb index dfde938f..2a372f54 100644 --- a/spec/lrama/new_parser_spec.rb +++ b/spec/lrama/new_parser_spec.rb @@ -550,7 +550,7 @@ ]) end - xit "; for rules is optional" do + it "; for rules is optional" do y = header + <<~INPUT %%