diff --git a/_config.yml b/_config.yml new file mode 100644 index 000000000..8797e37e0 --- /dev/null +++ b/_config.yml @@ -0,0 +1,8 @@ +title: Community notes explorer + +defaults: + - + scope: + path: "" + values: + layout: "default" diff --git a/_data/.gitkeep b/_data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/_data/meta.json b/_data/meta.json new file mode 100644 index 000000000..9d6782055 --- /dev/null +++ b/_data/meta.json @@ -0,0 +1 @@ +{"scraped_at": "2024-12-01T16:19:42.919953+00:00", "most_recent": "2024-11-22T01:02:09+00:00", "total_notes": 10310, "total_fetched": 0, "locked_until": null, "total_tweets": 0} \ No newline at end of file diff --git a/_data/mps.json b/_data/mps.json new file mode 100644 index 000000000..f9efd8277 --- /dev/null +++ b/_data/mps.json @@ -0,0 +1 @@ +[10955042, 14077382, 14395178, 14561015, 14590758, 14728535, 14845210, 15197262, 15500138, 15580900, 15726425, 15747950, 15765684, 15792947, 15865570, 15887422, 15957319, 16348332, 17390474, 17752924, 17755776, 18020612, 18099795, 18632946, 18951643, 19017675, 19126349, 19177609, 19245478, 19295262, 19407599, 19581817, 19589280, 19698191, 19859690, 19915728, 20000725, 20035457, 20132840, 20141120, 20142835, 20148039, 20226550, 20325923, 20362684, 20379080, 20392734, 20424362, 20427568, 20443985, 20508666, 20763907, 20813135, 20815162, 20924620, 20995648, 21084719, 21134627, 21189360, 21608406, 21713090, 21769986, 21868093, 22359874, 22398060, 22474050, 22645918, 22904581, 23089237, 23100225, 23105852, 23330953, 23357503, 23424533, 23749162, 23930261, 24045870, 24228290, 24727170, 24781358, 24919478, 25091518, 25587142, 26054047, 26985345, 27212119, 27286910, 27844589, 28079313, 28329646, 29746583, 29757382, 30221653, 30385952, 30978439, 31230434, 31756041, 32369044, 32439211, 32965648, 33027054, 33031863, 33038062, 33546827, 33921661, 34230896, 34374472, 36023636, 36664451, 36924726, 40297217, 40336185, 40444755, 40680478, 41195676, 41614566, 41783821, 41905997, 43312141, 45227179, 45314159, 45884485, 47561197, 49619469, 49720593, 51062977, 51174420, 52124856, 53358128, 54351001, 56338737, 56657187, 58821256, 61046587, 61660254, 61733092, 61781260, 61862447, 62093763, 63111042, 64679759, 65324089, 66078636, 67289397, 68659708, 69733004, 70803659, 71777646, 72341341, 72883916, 73923338, 77234984, 77808680, 77981490, 78126370, 78702900, 78996834, 80021045, 80151292, 80247465, 80812413, 81367969, 82093305, 87647640, 89307992, 90681210, 91786287, 93860818, 94701778, 95074968, 97932789, 98358195, 98390335, 101278778, 101712079, 102023574, 102105819, 102107018, 102696964, 103879974, 105800463, 107722321, 108055233, 108654496, 109845429, 110417782, 110550982, 111973662, 112398730, 113476721, 113491007, 114505454, 115643685, 117777690, 120720108, 124270074, 125287190, 129569947, 129768260, 130092487, 130133607, 130254822, 130280613, 130717174, 134196574, 134511026, 135019823, 136398656, 140107265, 143433549, 143508762, 144180351, 146162768, 148446462, 153810216, 153817010, 154661114, 155289056, 155698530, 164226176, 165387465, 168260255, 169241887, 169864622, 169898513, 172090468, 174386076, 174396155, 175512020, 175786071, 176721099, 176818616, 179970314, 188412761, 189280488, 191807697, 192935794, 194809611, 195752750, 198236824, 200538258, 200700960, 202558774, 202610289, 204873579, 207707329, 208516133, 209029473, 209746549, 212019800, 213243048, 215782428, 216516145, 219330249, 219976700, 222748037, 222762589, 225857392, 234089713, 235526305, 236588290, 237341484, 240202308, 240799464, 243135658, 243306810, 245849058, 247595473, 248186795, 249845912, 250601094, 255277792, 260760660, 261886643, 263691967, 264149246, 269788397, 271868768, 274033876, 276868661, 284147164, 288259199, 293147252, 293693143, 295948042, 296942850, 297967419, 298536072, 300872531, 301713110, 302302898, 302691378, 303321306, 304023718, 315283542, 316349960, 328634628, 329863756, 336529567, 342187925, 347347870, 349075382, 350223904, 350331171, 351077887, 351600865, 351657777, 351812038, 355623405, 368314502, 370700221, 376161322, 382845430, 383238347, 385306338, 389434876, 393436815, 397566701, 398132018, 398627964, 401568431, 404115169, 406646373, 407741330, 408454349, 409432990, 412213474, 415910519, 426809037, 431730103, 432396682, 443166451, 449148784, 450711709, 454933267, 456537189, 468621844, 474180693, 475138726, 477802917, 481120639, 485146783, 487655647, 493494710, 494082207, 496060628, 504523786, 515520108, 517998217, 518660800, 522273829, 532351874, 536479831, 538352656, 545081356, 546021288, 549492065, 562866860, 568249574, 570352145, 574843120, 576682647, 579804819, 580749365, 587742302, 592855211, 609096352, 620007630, 622426905, 706747004, 722347303, 728039018, 737066858, 748846627, 759067032, 794109804, 804178058, 804596460, 805185025, 808662330, 813678139, 818278134, 831250927, 860471443, 869651114, 883173469, 927480949, 948015937, 950737508, 952069440, 989608254, 990891818, 994816520, 997293914, 1086696404, 1087324843, 1114681081, 1123375357, 1151672852, 1170457166, 1179455215, 1198816026, 1302003805, 1328897358, 1337725045, 1342945016, 1364153984, 1381684742, 1444709809, 1466783923, 1513382018, 1562872584, 1573638926, 1585889305, 1608370866, 1633543440, 1678239019, 1704239983, 1704469674, 1722829178, 1858780626, 1864159236, 1865540413, 2157036506, 2159153839, 2173779986, 2174657848, 2183385501, 2187566068, 2205165174, 2209331701, 2262623615, 2278921206, 2323710210, 2350624098, 2350948635, 2382227424, 2406093795, 2425571623, 2469794059, 2471414497, 2483456906, 2491491242, 2522525965, 2528020694, 2540941937, 2583099904, 2601366250, 2609201187, 2616215901, 2718096642, 2718417153, 2750166746, 2776689260, 2785648892, 2788637049, 2797497890, 2797521996, 2805519629, 2834640796, 2855602085, 2860522038, 2872987265, 2873822342, 2933530091, 2942626846, 2979449745, 2979630241, 2979642791, 2999868527, 3005893073, 3012159189, 3021495911, 3034284694, 3056307455, 3092429597, 3128468829, 3183943337, 3298553870, 3307960979, 3323245259, 3792653536, 4010545737, 4104745522, 4397367323, 4453006828, 4897196079, 4898067429, 700732124299714561, 704931810216644609, 725725130186444800, 727494797490049024, 728595635713282048, 732224740820193280, 748521119691464705, 757652483472752640, 761499948890329088, 796764331266473984, 797021745563766785, 802599212194131968, 807297775176138752, 809357945725190144, 835388009533030401, 839428390201470976, 849799226523570177, 857641376628043778, 858747670562734081, 859085802218893312, 859339993810161664, 859433240452583424, 860188105982320644, 860807798480941056, 861333913695924225, 862712954336292864, 868787843510013953, 869291187630428160, 873636843627196416, 887252498054033408, 893178169380962304, 912263373752020993, 914410646548205568, 917084834874445824, 918729143159853056, 967847140105052165, 968505100372267009, 983085276766621696, 990637064696758272, 999264330179891200, 1043825118819110912, 1064912412736917504, 1079489837097455616, 1093899576329801729, 1111190141736046592, 1113783447561043972, 1117104681002721288, 1121375429884039175, 1133305785910550528, 1133792320040701953, 1148340474534666240, 1156498325056696320, 1158490894028460032, 1168968080690749441, 1172535926209298432, 1178829710186467328, 1179516894862364673, 1184537212303630336, 1191274965783531520, 1192899400936374272, 1192911732823482370, 1202243093552148482, 1214108358292041728, 1233337992670072837, 1244645536973676552, 1258352022132346880, 1260896506363641856, 1298994924101095425, 1303073868240875522, 1344403235742879745, 1355609944528191491, 1355938448016887808, 1393142778142527491, 1439528766368268292, 1459907434890375170, 1475756824787918851, 1483044185297068034, 1499421123594924034, 1511355740367298562, 1512840908064108545, 1518319401380024325, 1521111364399468545, 1526871150965440513, 1530711181169672194, 1530991036725608453, 1533495425524809731, 1536784150971260928, 1543007983361314821, 1544382687921799169, 1545003717388378112, 1547647624978739204, 1554589610881286145, 1568570575727075329, 1572344860052492288, 1581393252829302785, 1582194024445726720, 1582714278632689664, 1591728223292989441, 1596061977004097536, 1599753147861422087, 1602226274495303682, 1604889595056988160, 1610060002848276481, 1620056193300299778, 1620114766906773504, 1621807711510740993, 1636111129192611840, 1641121619442556928, 1655535691214778369, 1660431957849784320, 1664208792383373315, 1669648544020545538, 1692175731324047360, 1713959748284854272, 1715777573488111616, 1724049395685969920, 1727693587091996672, 1733600668412186624, 1740014998749913088, 1751336830723555328, 1754451616294408193, 1754991198886895616, 1760942562687225856, 1763897247857803264, 1768365178922409984, 1770463440437411840, 1781431480624754688, 1790468410909839360, 1795697527011864576, 1797169888831721472, 1797389916638212097, 1797567996258357248, 1797607388746297344, 1798364617116102656, 1798665613151178752, 1805309466302132224, 1810349008688422912, 1812997362262089728, 1813998834512506880] diff --git a/_includes/commify.html b/_includes/commify.html new file mode 100644 index 000000000..6f7554ab7 --- /dev/null +++ b/_includes/commify.html @@ -0,0 +1,21 @@ +{%- comment -%} +This is a ludicrously complicated way of adding thousands separators to large +numbers. + +I couldn’t figure out a better way of doing it without requiring either Ruby or +Javascript. +{%- endcomment -%} + +{%- assign number_arr = include.number | split: '' -%} +{%- assign commified_number = '' -%} + +{%- for digit in number_arr reversed -%} + {%- assign remainder = forloop.index0 | modulo: 3 -%} + {%- if remainder == 0 and forloop.index0 != 0 -%} + {%- assign commified_number = digit | append: ',' | append: commified_number -%} + {%- else -%} + {%- assign commified_number = digit | append: commified_number -%} + {%- endif -%} +{%- endfor -%} + +{{- commified_number -}} diff --git a/_layouts/default.html b/_layouts/default.html new file mode 100644 index 000000000..e4272fce7 --- /dev/null +++ b/_layouts/default.html @@ -0,0 +1,67 @@ + + + + + + {{ page.title }}{% if page.url != '/' %} | {{ site.title }}{% endif %} + + + + + + + + + + +
+
+ +
+ + {% if site.data.meta.total_tweets == 0 %} + + {% elsif site.data.meta.total_fetched < site.data.meta.total_tweets %} + {% assign perc_unfetched = site.data.meta.total_fetched | times: -100 | divided_by: site.data.meta.total_tweets | plus:100 %} + {% if perc_unfetched > 0 %} + + {% endif %} + {% endif %} + +
+

{{ page.title }}

+ + {{ content }} +
+
+ + diff --git a/about/index.md b/about/index.md new file mode 100644 index 000000000..63fbdd201 --- /dev/null +++ b/about/index.md @@ -0,0 +1,73 @@ +--- +title: How it works +--- + +Community note data is fetched regularly [from Twitter (X)](https://x.com/i/communitynotes/download-data). + +This data is always a couple of days old (**most recent data is from , scraped **). + +Notes are excluded if they meet any of the following criteria: + +* Created more than a week ago +* Classifying the post as ‘not misleading’ (i.e. in support of the post) +* Currently rated ‘unhelpful’ + +We also attempt to filter out notes for deleted tweets and non-English tweets. + +--- + +### Filter by author group + +With thanks to [@leobenedictus](https://x.com/leobenedictus) for the suggestion, community notes can be filtered by current UK MPs. + +--- + +### Special Twitter (X) language codes + +When Twitter (X) can’t determine the language of a tweet, it uses one of several reserved language codes. For the purpose of language filtering, we’ve grouped these all together. But this is the breakdown: + +|---------------|---------------------------------------------| +| Language code | Description | +|---------------|---------------------------------------------| +| `art` | Tweet contains emojis only | +| `qam` | Tweet contains mentions only | +| `qct` | Tweet contains cashtags only | +| `qht` | Tweet contains hashtags only | +| `qme` | Tweet contains media only | +| `qst` | Tweet text is very short | +| `und` | Undefined (couldn’t determine the language) | +| `zxx` | Tweet contains media or twitter card only | +{: .table .table-striped .w-inherit } + +--- + +### Tweet indexing status + +After fetching new proposed community notes, the text of the tweets that the notes reference is not immediately searchable. In order to make it searchable, we need to fetch these tweets – a process that can take several hours. You can see the current status below. + +{% if site.data.meta.total_tweets > 0 %} + {% assign perc_fetched = site.data.meta.total_fetched | times: 100 | divided_by: site.data.meta.total_tweets %} +{% else %} + {% assign perc_fetched = 0 %} +{% endif %} + +
+
+
+{{ perc_fetched }}% of tweets ({% include commify.html number=site.data.meta.total_fetched %} / {% include commify.html number=site.data.meta.total_tweets %}) are currently searchable. + + + +--- + +### Why is the language unknown for some tweets? + +Until we’ve fetched a tweet, we don’t know its language. So ‘unknown language’ may mean we haven’t yet fetched that tweet. Once we’ve fetched it (in the next hour or so) we should know the tweet author, language and text. + +‘Unknown language’ may also mean the tweet has been deleted. In this case, we have no way of determining the tweet author, language or text. diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/data/notes.json b/data/notes.json new file mode 100644 index 000000000..0637a088a --- /dev/null +++ b/data/notes.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/index.md b/index.md new file mode 100644 index 000000000..b89f9df7b --- /dev/null +++ b/index.md @@ -0,0 +1,260 @@ +--- +title: Community notes explorer +--- + +Proposed [Twitter (X) community notes](https://x.com/i/communitynotes/download-data) from the last week, updated regularly. _[More…]({{ '/about/' | relative_url }})_ + +
+ + + + + + + + + + + + + + + + +
CreatedTweetNote(s)Reason for noteTweet languageTweet statusTweet authorTweet contentMax score
+
+ +