Normalize the checked key to lowercase when looking for unsupported r…

…ules. PiperOrigin-RevId: 582285639
google · May 22, 2024 · 19feaac · 19feaac
1 parent de56676
commit 19feaac
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 5 deletions.
diff --git a/reporting_robots.cc b/reporting_robots.cc
@@ -4,6 +4,7 @@
 #include <string>
 #include <vector>
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/string_view.h"
 
 namespace googlebot {
@@ -69,7 +70,8 @@ void RobotsParsingReporter::HandleUnknownAction(int line_num,
                                                 absl::string_view action,
                                                 absl::string_view line_value) {
   RobotsParsedLine::RobotsTagName rtn =
-      std::count(kUnsupportedTags.begin(), kUnsupportedTags.end(), action) > 0
+      std::count(kUnsupportedTags.begin(), kUnsupportedTags.end(),
+                 absl::AsciiStrToLower(action)) > 0
           ? RobotsParsedLine::kUnused
           : RobotsParsedLine::kUnknown;
   unused_directives_++;

diff --git a/reporting_robots_test.cc b/reporting_robots_test.cc
@@ -111,11 +111,12 @@ TEST(RobotsUnittest, LinesNumbersAreCountedCorrectly) {
       "useragent: baz\n"                      // 11
       "disallaw: /some\n"                     // 12
       "site-map: https://e/s.xml #comment\n"  // 13
-      "sitemap: https://e/t.xml\n";           // 14
-                                              // 15 (from \n)
+      "sitemap: https://e/t.xml\n"            // 14
+      "Noarchive: /someCapital\n";            // 15
+                                              // 16 (from \n)
   googlebot::ParseRobotsTxt(kSimpleFile, &report);
   EXPECT_EQ(8, report.valid_directives());
-  EXPECT_EQ(15, report.last_line_seen());
+  EXPECT_EQ(16, report.last_line_seen());
   EXPECT_EQ(report.parse_results().size(), report.last_line_seen());
   std::vector<absl::string_view> lines = absl::StrSplit(kSimpleFile, '\n');
 
@@ -295,10 +296,22 @@ TEST(RobotsUnittest, LinesNumbersAreCountedCorrectly) {
                            .has_directive = true,
                            .is_acceptable_typo = false,
                        }});
-  // For line 15 (which is empty and comes from the last \n)
+  // For line "Noarchive: /someCapital\n"        // 15
   expectLineToParseTo(
       lines, report.parse_results(),
       RobotsParsedLine{.line_num = 15,
+                       .tag_name = RobotsParsedLine::RobotsTagName::kUnused,
+                       .is_typo = false,
+                       .metadata = RobotsParseHandler::LineMetadata{
+                           .is_empty = false,
+                           .has_comment = false,
+                           .is_comment = false,
+                           .has_directive = true,
+                       }});
+  // For line 16 (which is empty and comes from the last \n)
+  expectLineToParseTo(
+      lines, report.parse_results(),
+      RobotsParsedLine{.line_num = 16,
                        .tag_name = RobotsParsedLine::RobotsTagName::kUnknown,
                        .is_typo = false,
                        .metadata = RobotsParseHandler::LineMetadata{