Skip to content
This repository has been archived by the owner on Jul 30, 2024. It is now read-only.
/ NuGet.Jobs Public archive

ChinaStatsCollector should properly reformat scstatus of log lines #713

Merged
merged 4 commits into from
Feb 5, 2019
Merged

ChinaStatsCollector should properly reformat scstatus of log lines #713

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f0e36d8
chinastatscollector should properly reformat scstatus
Jan 31, 2019
b192719
fix unit tests
Jan 31, 2019
3045315
add parsing of china format to cdnlogentryparser
Jan 31, 2019
3c70d77
get hit-miss column from china logs
Feb 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions src/Stats.AzureCdnLogs.Common/CdnLogEntryParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,14 @@ public static CdnLogEntry ParseLogEntryFromLine(int lineNumber, string line, Act
// small margin of error caused by non-200 HTTP status codes.
if (entry.CacheStatusCode != null)
{
// Format: cache status + "/" + HTTP status code
// Example: "TCP_MISS/504"
// Previously, we were not correctly converting logs from China CDN to the format used by Global CDN, so we must support both formats.
// Global format: cache status + "/" + HTTP status code
// Global example: "TCP_MISS/504"
// China format: HTTP status code
// China example: "504"
var slashIndex = entry.CacheStatusCode.LastIndexOf('/');
uint httpStatusCode;
if (slashIndex >= 0
&& slashIndex + 1 < entry.CacheStatusCode.Length
if (slashIndex + 1 < entry.CacheStatusCode.Length
&& uint.TryParse(entry.CacheStatusCode.Substring(slashIndex + 1), out httpStatusCode)
&& (httpStatusCode < 200 || httpStatusCode >= 300))
{
Expand Down
12 changes: 5 additions & 7 deletions src/Stats.CollectAzureChinaCDNLogs/ChinaStatsCollector.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public override OutputLogLine TransformRawLogLine(string line)
string.IsNullOrEmpty(line) ||
line.Trim().StartsWith("c-ip", ignoreCase: true, culture: System.Globalization.CultureInfo.InvariantCulture))
{
//is the header
// Ignore empty lines or the header
return null;
}

Expand All @@ -59,19 +59,17 @@ public override OutputLogLine TransformRawLogLine(string line)
DateTime dt = DateTime.Parse(timestamp, CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal);
string timeStamp2 = ToUnixTimeStamp(dt);

//ignore 400 error codes
if(segments[5] == "400")
{
return null;
}
// Global status code format: cache status + "/" + HTTP status code
// China status code format: HTTP status code
var scstatus = segments[(int)ChinaLogHeaderFields.hitmiss] + "/" + segments[(int)ChinaLogHeaderFields.scstatus];

return new OutputLogLine(timestamp: timeStamp2,
timetaken: notAvailableInt,
cip:segments[(int)ChinaLogHeaderFields.cip],
filesize: notAvailableInt,
sip: segments[(int)ChinaLogHeaderFields.sip],
sport: notAvailableInt,
scstatus: segments[(int)ChinaLogHeaderFields.scstatus],
scstatus: scstatus,
scbytes: segments[(int)ChinaLogHeaderFields.scbytes],
csmethod: segments[(int)ChinaLogHeaderFields.csmethod],
csuristem: segments[(int)ChinaLogHeaderFields.csuristem],
Expand Down
7 changes: 3 additions & 4 deletions tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,10 @@ namespace Tests.Stats.CollectAzureChinaCDNLogs
public class ChinaCollectorTests
{
[Theory]
[InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,200,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", "1501174209 0 40.125.202.231 0 118.180.6.168 0 200 1196 GET /v3-flatcontainer/system.net.primitives/index.json - 133 0 - NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0) na na")]
[InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,400,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", null)]
[InlineData("40.125.202.231,7/27/2017 4:50:09 PM +00:00,GET,\"/v3-flatcontainer/system.net.primitives/index.json\",HTTP/1.1,200,1196,\"-\",\"NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)\",133,TCP_MISS,118.180.6.168", "1501174209 0 40.125.202.231 0 118.180.6.168 0 TCP_MISS/200 1196 GET /v3-flatcontainer/system.net.primitives/index.json - 133 0 - NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0) na na")]
[InlineData("c-ip, timestamp, cs-method, cs-uri-stem, http-ver, sc-status, sc-bytes, c-referer, c-user-agent, rs-duration(ms), hit-miss, s-ip", null)]
[InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon na na")]
[InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon na na")]
[InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google+Favicon na na")]
[InlineData("66.102.6.172,7/27/2017 4:50:09 PM +00:00,GET,\"/favicon.ico\",HTTP/1.1,200,726,\"-\",\"Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon\",216,TCP_MISS,150.138.143.19", "1501174209 0 66.102.6.172 0 150.138.143.19 0 TCP_MISS/200 726 GET /favicon.ico - 216 0 - Mozilla/5.0+ X11;+Linux+x86_64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/49.0.2623.75+Safari/537.36+Google,Favicon na na")]
public void TransformRawLogLine(string input, string expectedOutput)
{
var collector = new ChinaStatsCollector();
Expand Down
6 changes: 5 additions & 1 deletion tests/Tests.Stats.ImportAzureCdnStatistics/CdnLogEntryParserFacts.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ public class TheParseLogEntryFromLineMethod
[InlineData("SOMETHING_ELSE/404")]
[InlineData("TCP_MISS/504")]
[InlineData("TCP_MISS/604")]
[InlineData("0")]
[InlineData("304")]
[InlineData("400")]
[InlineData("404")]
[InlineData("500")]
public void IgnoresNon200HttpStatusCodes(string status)
{
// Arrange
Expand All @@ -43,7 +48,6 @@ public void IgnoresNon200HttpStatusCodes(string status)
[InlineData("TCP_MISS/")]
[InlineData("TCP_MISS")]
[InlineData("200")]
[InlineData("500")]
public void DoesNotIgnore200LevelAndUnrecognizedHttpStatusCodes(string status)
{
// Arrange
Expand Down