Deploying to gh-pages from @ 71cae10 🚀

_config.yml

@@ -0,0 +1,8 @@
+title: Twitter community notes
+
+defaults:
+  -
+    scope:
+      path: ""
+    values:
+      layout: "default"

_data/meta.json

@@ -0,0 +1 @@
+{"scraped_at": "2024-06-10T04:19:42.680383+00:00", "most_recent": "2024-06-08T02:30:16+00:00", "total_notes": 10310, "total_fetched": 8554, "locked_until": "2024-06-10T08:00:00+00:00", "total_tweets": 9819}

_data/mps.json

@@ -0,0 +1 @@
+["sarahathertonmp", "andrewmitchmp", "bob4beckenham", "ofkawczynski", "richardfoordld", "jillmortimer4hp", "uk_forbenvoy", "sarahgreenld", "kimleadbeater", "mpglosoffice", "allandoransmp", "adamhollowaymp", "matt_vickersmp", "gibbo4darlo", "twocitiesnickie", "sarbritcliffemp", "dines4dales", "robinmillarmp", "david4wantage", "sallyann1066", "kateosbornemp", "glindonmary", "dean4watford", "bren4bassetlaw", "apsanabegummp", "jamessunderl", "marcolonghi4dn", "oliverdowden", "rishisunak", "_robbiemoore", "christian4burys", "anthonybrownemp", "charleswalkermp", "drneilhudson", "mpiainds", "paulabarkermp", "ruthnewportwest", "maggie_erewash", "mickwhitleymp", "sirrogergale", "shaileshvara", "kimjohnsonmp", "bbradley_mans", "jane_stevenson_", "teamranil", "andrewbowie_mp", "yasinforbedford", "jacob_rees_mogg", "brineminister", "navpmishra", "jamie4north", "lia_nici", "robertlargan", "anthonymangnal1", "redditchrachel", "dancardenmp", "fabianleedsne", "julialopezmp", "eddiehughes4wn", "liztwistmp", "david_duguid", "ajrichardsonmp", "paulscullymp", "taiwoowatemi", "drcarolinej", "esthermcvey1", "michaelgove", "theresa_may", "scottbentonmp", "_andersonstuart", "iowbobseely", "kwasikwarteng", "nickfletchermp", "gillfurnissmp", "garethbaconmp", "garystreeterswd", "dominicraab", "lukehall", "sarahjolney1", "asollowayuk", "helenmorganmp", "jamiewallismp", "jimshannonmp", "kemibadenoch", "lucyfrazermp", "baynes_simon", "antony_hig", "nadams", "stevenbonnarsnp", "alewermbe", "edwardtimpson", "derekthomasuk", "laura__farris", "meljstride", "borisjohnson", "chris_evansmp", "nicktorfaen", "zarahsultana", "johnmcnallysnp", "nickgibbuk", "eleanor4epping", "khalid4pb", "pgibsonsnp", "petergrantmp", "simon4ndorset", "alanbrownsnp", "albertocostamp", "margaretferrier", "gradysnp", "justinmadders", "gilliankeegan", "mickeybradysf", "hannahb4livimp", "johncryermp", "imran_hussainmp", "eastantrimmp", "brendanoharamp", "dr_philippaw", "cmonaghansnp", "huwmerriman", "douglas4moray", "bethwintermp", "rehman_chishti", "victoriaprentis", "jochurchill_mp", "toniaantoniazzi", "kieranmullanuk", "drlisacameronmp", "daviddavismp", "lrobertsontewks", "drbenspencer", "jonathanlord", "wendychambld", "jcartlidgemp", "kirstenoswald", "kirstysnp", "alancampbellmp", "joannaccherry", "putneyfleur", "rosieduffield1", "darreng_henry", "keir_starmer", "carolynharris24", "jacobyoungmp", "drrosena", "marierimmer", "lsrplaid", "anumsnp", "mrjohnnicolson", "robertjenrick", "k_fletcher_mp", "marktamimp", "neill_bob", "derektwiggmp", "richolden", "chrislawsnp", "robertsyms", "tomtugendhat", "rthomsonmp", "alanmakmp", "walkerworcester", "mpgeorgeeustice", "tandhesi", "marykfoy", "aloksharma_rdg", "suellabraverman", "simonclarkemp", "marcusfysh", "dunne4ludlow", "desmondswayne", "william_wragg", "nus_ghani", "lynbrownmp", "marcus4nuneaton", "q66suzi", "mgreenwoodww", "simonhartmp", "jamesdalymp", "anna_firth", "votepursglove", "alexchalkchelt", "paulgirvanmp", "rees4neath", "andreajenkyns", "craig4monty", "laboursj", "nicolafrichards", "jostevenslabour", "ianblackford_mp", "cgreenuk", "mphendrick", "julianknight15", "lee4ned", "louie_french", "edwardjdavey", "dsimmonds_rnp", "mark4dewsbury", "natalieelphicke", "janetdaby", "jacklopresti", "maryrobinson01", "franciemolloy", "nigelmills", "_oliviablake", "rach_hopkins", "sarahchampionmp", "andymcdonaldmp", "peter_dowd", "henrysmithuk", "peterboneuk", "martinvickers", "mariamilleruk", "kevanjonesmp", "morton_wendy", "scrabbpembs", "ajonesmp", "bobblackman", "mpritcharduk", "julieelliottmp", "awmurrison", "emmalewellbuck", "juliansmithuk", "skinnock", "rupahuq", "ianmearnsmp", "sdoughtymp", "preetkgillmp", "lindsayhoyle_mp", "caroline_ansell", "davidrutley", "cwhittaker_mp", "stephenfarrymp", "siobhain_mc", "paulhowellmp", "nadiawhittomemp", "ianbyrnemp", "richardbaconmp", "huddlestonnigel", "y_fovarguemp", "leodochertyuk", "davidtcdavies", "richardburgon", "joymorrissey", "blaenaugwentmp", "liamfox", "anneliesedodds", "rebeccaharrismp", "joncruddas_1", "billwigginmp", "geraldjoneslab", "stephenflynnsnp", "karinsmyth", "tobias_ellwood", "niagriffithmp", "drlukeevans", "edwardleighmp", "shaun4wbw", "pennymordaunt", "nadinedorries", "benmlake", "danielzeichner", "jsheappey", "cskidmoreuk", "metcalfe_sbet", "karlturnermp", "bhatti_saqib", "mark_j_harper", "deidrebrock", "damiangreen", "kateosamor", "waynedavid_mp", "hilarybennmp", "shabanamahmood", "orfhlaithbegley", "judithcummins", "amycallaghansnp", "davedoogansnp", "munirawilson", "smcpartland", "cliveefford", "alanwhiteheadmp", "angelacrawley30", "conorburnsuk", "norwichchloe", "rlong_bailey", "cj_dinenage", "hollylynch5", "amandamilling", "paulblomfieldmp", "johnhealey_mp", "garysambrook89", "pow_rebecca", "jeromemayhew", "tomhunt1988", "gavinwilliamson", "thisischerilyn", "nazshahbfd", "catherinewest1", "garethdavies_mp", "abenaopp", "chrishazzardsf", "charlotte2153", "clairecoutinho", "keeleymp", "yvettecoopermp", "valerie_vazmp", "bimafolami", "abridgen", "tahiralimp", "ianlaverymp", "robbaylesbury", "meg_hilliermp", "danny__kruger", "tommysheppard", "elliotcolburn", "j_donaldson_mp", "petewishart", "andrewhwestern", "jamesowild", "stevebarclay", "selainesaxby", "thangammp", "helenhayes_", "karenpbuckmp", "peter_aldous", "johnfinucane", "drewhendrysnp", "dougchapmansnp", "sarahowen_", "justintomlinson", "barrysheerman", "grobinsondup", "coyleneil", "rushanaraali", "chrisstephens", "mattwestern_", "aaronbell4nul", "kennymacaskill", "danjarvismp", "bellribeiroaddy", "mimsdavies", "offord4hendon", "debbie_abrahams", "gregclarkmp", "bernardjenkin", "angelarayner", "john4carlisle", "alex_stafford", "gareththomasmp", "stewarthosiesnp", "kellytolhurst", "jakeberry", "mattrodda", "simonjamesjupp", "stuart_mcdonald", "jnhanvey", "carolinenokes", "mariacaulfield", "oliverhealduk", "afzal4gorton", "sharonhodgsonmp", "michael4mdnp", "chrisloder", "gregsmith_uk", "meaglemp", "jon_trickett", "kate_hollernmp", "marshadecordova", "davidmorrisml", "cmackinlay", "jamesmurray_ldn", "vickyfoxcroft", "pauljholmes", "cpjelmore", "samanthadixonmp", "chrispincher", "emilythornberry", "johnpenrosenews", "jamesmorris", "mark_spencer", "jimfromoldham", "hackneyabbott", "geraintdaviesmp", "acunninghammp", "louhaigh", "bwallacemp", "guyopperman", "jonashworth", "gavnewlandssnp", "craig4nwarks", "griffitha", "jonesyfay", "paulettehamilto", "ronniecowan", "andrew4pendle", "tom_randall", "jdjanogly", "helen_whately", "john2win", "alecshelbrooke", "siobhan_baillie", "jwhittingdale", "gildernewm", "philipdaviesuk", "greghands", "angelaeagle", "jamesduddridge", "nadhimzahawi", "bill_esterson", "mhairiblack", "ben_everitt", "jeremycorbyn", "michelledonelan", "alisonthewliss", "markjenkinsonmp", "dawnbutlerbrent", "tulipsiddiq", "libdemdaisy", "scottmann4nc", "jeremy_hunt", "margarethodge", "pbottomleymp", "ruthcadbury", "timloughton", "barrygardiner", "leicesterliz", "geoffrey_cox", "stephenctimms", "bambosmp", "georgefreemanmp", "billcashmp", "iainastewart", "johnnymerceruk", "chrisclarksonmp", "lisanandy", "johnredwood", "martyndaysnp", "amcarmichaelmp", "andrewselous", "cphilpofficial", "cajardinemp", "jgray", "chionwurah", "annemariemorris", "chhcalling", "paulmaskeymp", "carolinelucas", "rachaelmaskell", "timfarron", "marionfellows", "jesse_norman", "ianmurraymp", "catmckinnell", "johnmcdonnellmp", "aluncairns", "crispinblunt", "royston_smith", "yasminqureshimp", "mikekanemp", "dianajohnsonmp", "paulbristow79", "jogideon", "feryalclark", "hammersmithandy", "trussliz", "neildotobrien", "tracey_crouch", "ed_miliband", "tobyperkinsmp", "pritipatel", "theodoraclarke", "grahamemorris", "gwynnemp", "markpawsey", "elliereeves", "steph_peacock", "claudiawebbe", "theresecoffey", "kevinhollinrake", "s_hammond", "angusmacneilsnp", "kevin_j_foster", "jessicamordenmp", "aliciakearns", "ashleydalton_mp", "lloyd_rm", "jackiedp", "andrealeadsom", "gileswatling", "labourlewis", "johnglenuk", "conormcginn", "janemhunt", "rachelreevesmp", "alexdaviesjones", "lucympowell", "seemamalhotra1", "felicitybuchan", "catsmithmp", "simonlightwood", "owenthompson", "gaganmohindra", "dehennadavison", "liambyrnemp", "karlmccartney", "mrandy_carter", "samtarry", "patmcfaddenmp", "duncancbaker", "martinjdocherty", "helengrantmp", "benpbradshaw", "stephenmorganmp", "stevereedmp", "tony4rochdale", "virendrasharma", "laylamoran", "annemclaughlin", "alynsmith", "carlalockhart", "davidlinden", "alexburghart", "jamesdavies", "adamafriyie", "peterkyle", "kevinbrennanmp", "heatherwheeler", "spellar", "alison_mcgovern", "steve_mccabe", "hywelplaidcymru", "stevedouble", "mtpennycook", "juliemarsonmp", "rhonddabryant", "columeastwood", "floeshalomi", "clairehanna", "willquince", "stuartandrew", "darrenpjones", "wesstreeting", "mikeamesburymp", "robertcourts", "annietrev", "mark4wyreforest", "liliangreenwood", "alexnorrisnn", "andrewrosindell", "sajidjavid", "jessphillips", "harrietharman", "wera_hobhouse", "michael_ellis1", "matthancock", "halfon4harlowmp", "jasonmccartney", "stewartmcdonald", "davidmundelldct", "kitmalthouse", "bphillipsonmp", "hbaldwin", "pauline_latham", "mike_fabricant", "jeffsmithetc", "jreynoldsmp", "sheryllmurray", "kerrymp", "davidlammy", "mattwarman", "scullyp", "davidevennettmp", "davidjonesmp", "brandonlewis", "simonfell", "lucyallan", "alexsobel", "stellacreasy", "gregknight", "stevebakerhw", "paulmaynarduk", "damiancollins", "mikejwood", "vickyford", "annamcmorrin", "damianhinds", "grahamstuart", "robertbuckland", "grantshapps", "jamescleverly", "brinemp", "lukepollard", "nigelmp", "juliansturdy"]

_includes/commify.html

@@ -0,0 +1,26 @@
+{%- comment -%}
+This is a ludicrously complicated way of thousands separators to large numbers.
+
+I couldn’t figure out a better way of doing it without requiring either Ruby or
+Javascript.
+{%- endcomment -%}
+
+{%- assign number_string = include.number | split: '' -%}
+{%- assign total_digits = number_string.size -%}
+{%- assign commify_number = '' -%}
+
+{%- for i in (1..total_digits) -%}
+  {%- assign index = total_digits | minus: i -%}
+  {%- assign digit = number_string[index] -%}
+
+  {%- assign position = index -%}
+  {%- assign remainder = position | modulo: 3 -%}
+
+  {%- if remainder == 0 and i != 1 -%}
+    {%- assign commify_number = digit | append: ',' | append: commify_number -%}
+  {%- else -%}
+    {%- assign commify_number = digit | append: commify_number -%}
+  {%- endif -%}
+{%- endfor -%}
+
+{{- commify_number -}}

_layouts/default.html

@@ -0,0 +1,51 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>{{ page.title }}{% if page.url != '/' %} | {{ site.title }}{% endif %}</title>
+  <link href="//cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
+  <link rel="stylesheet" href="//cdn.datatables.net/v/bs5/jq-3.7.0/dt-2.0.7/fh-4.0.1/sp-2.3.1/sl-2.0.1/datatables.min.css">
+  <style type="text/css">
+    td {
+      word-break: break-word;
+    }
+    time {
+      cursor: help;
+      text-decoration: underline;
+      text-decoration-style: dotted;
+    }
+    .dt-type-numeric {
+      text-align: left !important;
+    }
+    .w-inherit {
+      width: inherit;
+    }
+  </style>
+</head>
+<body>
+  <script src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
+  <script src="//cdnjs.cloudflare.com/ajax/libs/luxon/2.3.1/luxon.min.js"></script>
+  <script src="//cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
+  <script src="//cdn.datatables.net/v/bs5/jq-3.7.0/dt-2.0.7/fh-4.0.1/sp-2.3.1/sl-2.0.1/datatables.min.js"></script>
+
+  <div class="container py-3">
+    <header class="d-flex flex-wrap justify-content-end py-3 mb-4 border-bottom">
+      <ul class="nav nav-pills">
+        <li class="nav-item">
+          <a class="nav-link{% if page.url == '/' %} active{% endif %}" href="{{ '/' | relative_url }}">Home</a>
+        </li>
+        <li class="nav-item">
+          <a class="nav-link{% if page.url == '/about/' %} active{% endif %}" href="{{ '/about/' | relative_url }}">About</a>
+        </li>
+      </ul>
+    </header>
+
+    <main>
+      <h1>{{ page.title }}</h1>
+
+      {{ content }}
+    </main>
+  </div>
+</body>
+</html>

about/index.md

@@ -0,0 +1,79 @@
+---
+title: How it works
+---
+
+Community note data is fetched regularly [from Twitter (X)](https://x.com/i/communitynotes/download-data).
+
+This data is always a couple of days old (**most recent data is from <time class="dt" datetime="{{ site.data.meta.most_recent }}" title="{{ site.data.meta.most_recent | date_to_rfc822 }}">{{ site.data.meta.most_recent }}</time>, scraped <time class="dt" datetime="{{ site.data.meta.scraped_at }}" title="{{ site.data.meta.scraped_at | date_to_rfc822 }}">{{ site.data.meta.scraped_at }}</time>**).
+
+Notes are excluded if they meet any of the following criteria:
+
+* Created more than a week ago
+* Classifying the post as ‘not misleading’ (i.e. in support of the post)
+* Currently rated ‘unhelpful’
+
+We also attempt to filter out notes for deleted tweets and non-English tweets.
+
+---
+
+### Filter by author group
+
+With thanks to [@leobenedictus](https://x.com/leobenedictus) for the suggestion, community notes can be filtered by ~~current UK MPs~~ **UK General Election candidates**.
+
+In order to do this, we need a list of the Twitter (X) handles of election candidates. This data is pulled daily from [Democracy Club candidates](https://candidates.democracyclub.org.uk/). It’s incomplete, but you can help improve it by finding and adding candidates / candidate Twitter (X) handles to their data.
+
+{% assign total_candidate_handles = site.data.ge2024-candidates | size %}
+
+At present, Democracy Club candidates has Twitter (X) handles of **{% include commify.html number=total_candidate_handles %} UK General Election candidates**.
+
+---
+
+### Special Twitter (X) language codes
+
+When Twitter (X) can’t determine the language of a tweet, it uses one of several reserved language codes. For the purpose of language filtering, we’ve grouped these all together. But this is the breakdown:
+
+|---------------|---------------------------------------------|
+| Language code | Description                                 |
+|---------------|---------------------------------------------|
+| `art`         | Tweet contains emojis only                  |
+| `qam`         | Tweet contains mentions only                |
+| `qct`         | Tweet contains cashtags only                |
+| `qht`         | Tweet contains hashtags only                |
+| `qme`         | Tweet contains media only                   |
+| `qst`         | Tweet text is very short                    |
+| `und`         | Undefined (couldn’t determine the language) |
+| `zxx`         | Tweet contains media or twitter card only   |
+{: .table .table-striped .w-inherit }
+
+---
+
+### Tweet indexing status
+
+After fetching new proposed community notes, the text of the tweets that the notes reference is not immediately searchable. In order to make it searchable, we need to fetch these tweets – a process that can take several hours. You can see the current status below.
+
+{% if site.data.meta.total_tweets %}
+  {% assign perc_fetched = site.data.meta.total_fetched | times: 100 | divided_by: site.data.meta.total_tweets %}
+{% else %}
+  {% assign perc_fetched = 0 %}
+{% endif %}
+
+<div class="progress my-2" style="max-width: 500px;" role="progressbar">
+  <div class="progress-bar text-bg-{% if perc_fetched == 100 %}success{% elsif perc_fetched < 50 %}danger{% else %}warning{% endif %}" style="width: {{ perc_fetched }}%">{{ perc_fetched }}% ({% include commify.html number=site.data.meta.total_fetched %} / {% include commify.html number=site.data.meta.total_tweets %})</div>
+</div>
+{{ perc_fetched }}% of tweets are searchable.
+
+<script>
+  const dts = document.getElementsByClassName('dt');
+  for (var i = 0; i < dts.length; i++) {
+    var dt = dts[i];
+    dt.textContent = luxon.DateTime.fromISO(dt.textContent).toRelative();
+  }
+</script>
+
+---
+
+### Why is the language unknown for some tweets?
+
+Until we’ve fetched a tweet, we don’t know its language. So ‘unknown language’ may mean we haven’t yet fetched that tweet. Once we’ve fetched it (in the next hour or so) we should know the tweet author, language and text.
+
+‘Unknown language’ may also mean the tweet has been deleted. In this case, we have no way of determining the tweet author, language or text.