diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 00000000000..fa7adc7ac72 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.3.5 diff --git a/src/explore-education-statistics-api-docs/.gitignore b/src/explore-education-statistics-api-docs/.gitignore new file mode 100644 index 00000000000..313ccf02bd7 --- /dev/null +++ b/src/explore-education-statistics-api-docs/.gitignore @@ -0,0 +1,16 @@ +.idea + +# Ignore bundler config +/.bundle + +# Ignore the build directory +/build + +# Ignore cache +/.sass-cache +/.cache + +# Ignore .DS_store file +.DS_Store + +Staticfile.auth diff --git a/src/explore-education-statistics-api-docs/Gemfile b/src/explore-education-statistics-api-docs/Gemfile new file mode 100644 index 00000000000..c410cd3f01f --- /dev/null +++ b/src/explore-education-statistics-api-docs/Gemfile @@ -0,0 +1,22 @@ +source 'https://rubygems.org' + +ruby '3.3.5' + +# For faster file watcher updates on Windows: +gem 'wdm', '~> 0.1.0', platforms: [:mswin, :mingw] + +# Windows does not come with time zone data +gem 'tzinfo-data', platforms: [:mswin, :mingw, :jruby] + +gem 'govuk_tech_docs' + +gem 'middleman-gh-pages' + +# Include linter to check for dead internal links +gem 'html-proofer' + +gem 'chronic' + +gem 'http' + +gem 'rake' diff --git a/src/explore-education-statistics-api-docs/Gemfile.lock b/src/explore-education-statistics-api-docs/Gemfile.lock new file mode 100644 index 00000000000..706bcc7dd85 --- /dev/null +++ b/src/explore-education-statistics-api-docs/Gemfile.lock @@ -0,0 +1,270 @@ +GEM + remote: https://rubygems.org/ + specs: + Ascii85 (1.1.1) + activesupport (7.0.8.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + afm (0.2.2) + async (2.17.0) + console (~> 1.26) + fiber-annotation + io-event (~> 1.6, >= 1.6.5) + autoprefixer-rails (10.4.19.0) + execjs (~> 2) + backports (3.25.0) + base64 (0.2.0) + bigdecimal (3.1.8) + chronic (0.10.2) + chunky_png (1.4.0) + coffee-script (2.4.1) + coffee-script-source + execjs + coffee-script-source (1.12.2) + commonmarker (0.23.10) + compass (1.0.3) + chunky_png (~> 1.2) + compass-core (~> 1.0.2) + compass-import-once (~> 1.0.5) + rb-fsevent (>= 0.9.3) + rb-inotify (>= 0.9) + sass (>= 3.3.13, < 3.5) + compass-core (1.0.3) + multi_json (~> 1.0) + sass (>= 3.3.0, < 3.5) + compass-import-once (1.0.5) + sass (>= 3.2, < 3.5) + concurrent-ruby (1.3.4) + console (1.27.0) + fiber-annotation + fiber-local (~> 1.1) + json + contracts (0.16.1) + csv (3.3.0) + domain_name (0.6.20240107) + dotenv (3.1.4) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + erubis (2.7.0) + ethon (0.16.0) + ffi (>= 1.15.0) + eventmachine (1.2.7) + execjs (2.10.0) + fast_blank (1.0.1) + fastimage (2.3.1) + ffi (1.17.0) + ffi-compiler (1.3.2) + ffi (>= 1.15.5) + rake + fiber-annotation (0.2.0) + fiber-local (1.1.0) + fiber-storage + fiber-storage (1.0.0) + google-protobuf (4.28.3) + bigdecimal + rake (>= 13) + govuk_tech_docs (4.1.0) + autoprefixer-rails (~> 10.2) + base64 + bigdecimal + chronic (~> 0.10.2) + csv + haml (~> 6.0) + middleman (~> 4.0) + middleman-autoprefixer (~> 2.10) + middleman-compass (~> 4.0) + middleman-livereload + middleman-search-gds + middleman-sprockets (~> 4.0.0) + middleman-syntax (~> 3.4) + mutex_m + nokogiri + openapi3_parser (~> 0.9.0) + redcarpet (~> 3.6) + sassc-embedded (~> 1.78.0) + terser (~> 1.2.3) + haml (6.3.0) + temple (>= 0.8.2) + thor + tilt + hamster (3.0.0) + concurrent-ruby (~> 1.0) + hashery (2.1.2) + hashie (3.6.0) + html-proofer (5.0.9) + addressable (~> 2.3) + async (~> 2.1) + nokogiri (~> 1.13) + pdf-reader (~> 2.11) + rainbow (~> 3.0) + typhoeus (~> 1.3) + yell (~> 2.0) + zeitwerk (~> 2.5) + http (5.2.0) + addressable (~> 2.8) + base64 (~> 0.1) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.5.0) + http-cookie (1.0.7) + domain_name (~> 0.5) + http-form_data (2.3.0) + http_parser.rb (0.8.0) + i18n (1.6.0) + concurrent-ruby (~> 1.0) + io-event (1.7.3) + json (2.7.4) + kramdown (2.4.0) + rexml + listen (3.9.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + llhttp-ffi (0.5.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) + memoist (0.16.2) + middleman (4.5.1) + coffee-script (~> 2.2) + haml (>= 4.0.5) + kramdown (>= 2.3.0) + middleman-cli (= 4.5.1) + middleman-core (= 4.5.1) + middleman-autoprefixer (2.10.0) + autoprefixer-rails (>= 9.1.4) + middleman-core (>= 3.3.3) + middleman-cli (4.5.1) + thor (>= 0.17.0, < 1.3.0) + middleman-compass (4.0.1) + compass (>= 1.0.0, < 2.0.0) + middleman-core (>= 4.0.0) + middleman-core (4.5.1) + activesupport (>= 6.1, < 7.1) + addressable (~> 2.4) + backports (~> 3.6) + bundler (~> 2.0) + contracts (~> 0.13, < 0.17) + dotenv + erubis + execjs (~> 2.0) + fast_blank + fastimage (~> 2.0) + hamster (~> 3.0) + hashie (~> 3.4) + i18n (~> 1.6.0) + listen (~> 3.0) + memoist (~> 0.14) + padrino-helpers (~> 0.15.0) + parallel + rack (>= 1.4.5, < 3) + sassc (~> 2.0) + servolux + tilt (~> 2.0.9) + toml + uglifier (~> 3.0) + webrick + middleman-gh-pages (0.4.1) + rake (> 0.9.3) + middleman-livereload (3.4.7) + em-websocket (~> 0.5.1) + middleman-core (>= 3.3) + rack-livereload (~> 0.3.15) + middleman-search-gds (0.11.2) + execjs (~> 2.6) + middleman-core (>= 3.2) + nokogiri (~> 1.6) + middleman-sprockets (4.0.0) + middleman-core (~> 4.0) + sprockets (>= 3.0) + middleman-syntax (3.4.0) + middleman-core (>= 3.2) + rouge (~> 3.2) + mini_portile2 (2.8.7) + minitest (5.25.1) + multi_json (1.15.0) + mutex_m (0.2.0) + nokogiri (1.16.7) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + openapi3_parser (0.9.2) + commonmarker (~> 0.17) + padrino-helpers (0.15.3) + i18n (>= 0.6.7, < 2) + padrino-support (= 0.15.3) + tilt (>= 1.4.1, < 3) + padrino-support (0.15.3) + parallel (1.26.3) + parslet (2.0.0) + pdf-reader (2.12.0) + Ascii85 (~> 1.0) + afm (~> 0.2.1) + hashery (~> 2.0) + ruby-rc4 + ttfunk + public_suffix (6.0.1) + racc (1.8.1) + rack (2.2.10) + rack-livereload (0.3.17) + rack + rainbow (3.1.1) + rake (13.2.1) + rb-fsevent (0.11.2) + rb-inotify (0.11.1) + ffi (~> 1.0) + redcarpet (3.6.0) + rexml (3.3.9) + rouge (3.30.0) + ruby-rc4 (0.1.5) + sass (3.4.25) + sass-embedded (1.80.4) + google-protobuf (~> 4.28) + rake (>= 13) + sassc (2.4.0) + ffi (~> 1.9) + sassc-embedded (1.78.0) + sass-embedded (~> 1.78) + servolux (0.13.0) + sprockets (4.2.1) + concurrent-ruby (~> 1.0) + rack (>= 2.2.4, < 4) + temple (0.10.3) + terser (1.2.4) + execjs (>= 0.3.0, < 3) + thor (1.2.2) + tilt (2.0.11) + toml (0.3.0) + parslet (>= 1.8.0, < 3.0.0) + ttfunk (1.8.0) + bigdecimal (~> 3.1) + typhoeus (1.4.1) + ethon (>= 0.9.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + uglifier (3.2.0) + execjs (>= 0.3.0, < 3) + webrick (1.8.2) + yell (2.2.2) + zeitwerk (2.7.1) + +PLATFORMS + ruby + +DEPENDENCIES + chronic + govuk_tech_docs + html-proofer + http + middleman-gh-pages + rake + tzinfo-data + wdm (~> 0.1.0) + +RUBY VERSION + ruby 3.3.5p100 + +BUNDLED WITH + 2.3.22 diff --git a/src/explore-education-statistics-api-docs/README.md b/src/explore-education-statistics-api-docs/README.md new file mode 100644 index 00000000000..14a36e983c4 --- /dev/null +++ b/src/explore-education-statistics-api-docs/README.md @@ -0,0 +1,46 @@ +# Explore education statistics API documentation + +This repository is used to generate the documentation website for the Explore education statistics API. +It is based on the GOV.UK [Technical Documentation Template](https://tdt-documentation.london.cloudapps.digital/) +for building + +## Pre-requisites + +The following pre-requisite dependencies are required to get started: + +- [Node.js](https://nodejs.org/en/) v20+ (can be installed with [nvm](https://github.com/nvm-sh/nvm) or [fnm](https://github.com/Schniz/fnm)) +- [Ruby](https://www.ruby-lang.org/en/) v3.3.5 (can be installed with [rbenv](https://github.com/rbenv/rbenv) or [rvm](https://rvm.io/)) + +As always, it's advisable to install any versions using a version manager to make it easier to upgrade +and keep aligned with the project. + +### Ubuntu + +If you are using Ubuntu, you may need to install the following dependencies before you can install +Ruby and its required gems: + +```shell +sudo apt install libssl-dev libyaml-dev +``` + +## Getting started + +Once the pre-requisites have been installed, follow these steps: + +1. Install the project's Ruby dependencies: + + ```shell + bundle install + ``` + +2. Start the development server: + + ```shell + bundle exec middleman + ``` + + This will start the Middleman development server on [https://localhost:4567](https://localhost:4567). + +3. Optional. To automatically refresh the browser upon code changes, install the [LiveReload browser extension](https://chrome.google.com/webstore/detail/livereload/jnihajbhpnppcggbcgedagnkighmdlei?hl=en). + +For further guidance on how to develop this documentation, please visit the [Technical Documentation Template](https://tdt-documentation.london.cloudapps.digital/) website. diff --git a/src/explore-education-statistics-api-docs/Rakefile b/src/explore-education-statistics-api-docs/Rakefile new file mode 100644 index 00000000000..6568b9c0638 --- /dev/null +++ b/src/explore-education-statistics-api-docs/Rakefile @@ -0,0 +1,45 @@ +require_relative './lib/notifier' +require 'chronic' + +task default: ["notify:expired"] + +namespace :notify do + pages_urls = [ + "https://dfe-analytical-services.github.io/explore-education-statistics-api-docs/api/pages.json" + ] + + limits = { + } + + live = ENV.fetch("REALLY_POST_TO_SLACK", 0) == "1" + slack_url = ENV["SLACK_WEBHOOK_URL"] + slack_token = ENV["SLACK_TOKEN"] + + if live && (!slack_url && !slack_token) then + fail "If you want to post to Slack you need to set SLACK_TOKEN or SLACK_WEBHOOK_URL" + end + + desc "Notifies of all pages which have expired" + task :expired do + notification = Notification::Expired.new + + pages_urls.each do |page_url| + puts "== #{page_url}" + + Notifier.new(notification, page_url, slack_url, live, limits.fetch(page_url, -1)).run + end + end + + desc "Notifies of all pages which will expire soon" + task :expires, :timeframe do |_, args| + args.with_defaults(timeframe: "in 1 month") + expire_by = Chronic.parse(args[:timeframe]).to_date + notification = Notification::WillExpireBy.new(expire_by) + + pages_urls.each do |page_url| + puts "== #{page_url}" + + Notifier.new(notification, page_url, slack_url, live).run + end + end +end diff --git a/src/explore-education-statistics-api-docs/Staticfile b/src/explore-education-statistics-api-docs/Staticfile new file mode 100644 index 00000000000..bc014d31169 --- /dev/null +++ b/src/explore-education-statistics-api-docs/Staticfile @@ -0,0 +1,4 @@ +root: build +http_strict_transport_security: true +http_strict_transport_security_include_subdomains: true +location_include: includes/*.conf \ No newline at end of file diff --git a/src/explore-education-statistics-api-docs/config.rb b/src/explore-education-statistics-api-docs/config.rb new file mode 100644 index 00000000000..40b304aed97 --- /dev/null +++ b/src/explore-education-statistics-api-docs/config.rb @@ -0,0 +1,45 @@ +require 'govuk_tech_docs' +require 'lib/api_reference_pages_extension' +require 'lib/helpers' +require 'lib/api_reference_helpers' +require 'lib/govuk_tech_docs/path_helpers' + +# Check for broken links +require 'html-proofer' + +GovukTechDocs.configure(self, livereload: { js_host: "localhost", host: "127.0.0.1" }) + +# Override config from environment variables + +if ENV.has_key?("TECH_DOCS_HOST") + config[:tech_docs][:host] = ENV["TECH_DOCS_HOST"] || config[:tech_docs][:host] +end + +if ENV.has_key?("TECH_DOCS_PREVENT_INDEXING") + config[:tech_docs][:prevent_indexing] = ENV["TECH_DOCS_PREVENT_INDEXING"] +end + +if ENV.has_key?("TECH_DOCS_API_DOCS_PATH") + config[:tech_docs][:api_docs_path] = ENV["TECH_DOCS_API_DOCS_PATH"] +end + +helpers Helpers +helpers ApiReferenceHelpers +activate :api_reference_pages + +activate :relative_assets +set :relative_links, true + +after_build do |builder| + begin + HTMLProofer.check_directory(config[:build_dir], + { + :disable_external => true, + :swap_urls => { + config[:tech_docs][:host] => "", + } + }).run + rescue RuntimeError => e + abort e.to_s + end +end diff --git a/src/explore-education-statistics-api-docs/config/tech-docs.yml b/src/explore-education-statistics-api-docs/config/tech-docs.yml new file mode 100644 index 00000000000..e5ef95923f4 --- /dev/null +++ b/src/explore-education-statistics-api-docs/config/tech-docs.yml @@ -0,0 +1,41 @@ +# Host to use for canonical URL generation (without trailing slash) +host: https://docs.statistics.api.education.gov.uk +contact_email: explore.statistics@education.gov.uk + +# Header-related options +show_govuk_logo: true +service_name: Explore education statistics API +service_link: /index.html +phase: Beta + +# Links to show on right-hand-side of header +header_links: + Support: /support/index.html +footer_links: + Accessibility statement: /accessibility-statement/index.html + +# Table of contents depth – how many levels to include in the table of contents. +# If your ToC is too long, reduce this number and we'll only show higher-level +# headings. +max_toc_heading_level: 1 + +api_docs_path: https://dev.statistics.api.education.gov.uk/docs/1.0/openapi.json + +# Multi-page options +multipage_nav: true +collapsible_nav: true + +# Show links to contribute on GitHub +show_contribution_banner: true +github_repo: dfe-analytical-services/explore-education-statistics-api-docs + +# Enable search +enable_search: true + +# Prevent robots from indexing (for example whilst in development) +prevent_indexing: true + +# Ownership for page reviews +default_owner_slack: '#alerts' +owner_slack_workspace: exploreeducationstats +show_expiry: false diff --git a/src/explore-education-statistics-api-docs/lib/api_reference_helpers.rb b/src/explore-education-statistics-api-docs/lib/api_reference_helpers.rb new file mode 100644 index 00000000000..83b81a48451 --- /dev/null +++ b/src/explore-education-statistics-api-docs/lib/api_reference_helpers.rb @@ -0,0 +1,207 @@ +module ApiReferenceHelpers + # @param [Openapi3Parser::Node::Schema] schema_data + # @param [Array] references + # @return [*] + def schema_example(schema_data, references = []) + if schema_data.example + return schema_data.example + end + + if schema_data.default + return schema_data.default + end + + if is_primitive_schema(schema_data) + return Utils::primitive_schema_example(schema_data) + end + + if schema_data.type == "array" + items = schema_data.items + + if items && items != schema_data + if items.one_of&.any? + return items + .one_of + .reject { |schema| references.include?(schema) } + # Add schema to references to avoid infinite recursion + .map { |schema| schema_example(schema, references + [schema]) } + else + return [schema_example(items)] + end + else + return items ? [schema_example(items)] : [] + end + end + + if schema_data.one_of&.any? + schemas = schema_data + .one_of + .reject { |schema| references.include?(schema) } + + # Add schema to references to avoid infinite recursion + return schema_example(schemas[0], references + [schema_data]) + end + + # Is extending a referenced another schema using allOf + if schema_data.type.nil? && schema_data.all_of&.count == 1 + return schema_example(schema_data.all_of[0]) + end + + properties = get_schema_properties(schema_data) + + properties = properties.each_with_object({}) do |(name, schema), memo| + memo[name] = schema_example(schema, references + [schema]) + end + + if schema_data.additional_properties? + properties["<*>"] = if schema_data.additional_properties_schema + schema_example(schema_data.additional_properties_schema) + else + {} + end + end + + properties + end + + # @param [Openapi3Parser::Node::Schema] schema + # @return [*] + def get_schema_properties(schema) + properties = schema.properties.to_h + + schema.all_of.to_a.each do |all_of_schema| + properties.merge!(all_of_schema.properties.to_h) + end + + properties + end + + # @param [Openapi3Parser::Node::Schema] schema + # @return [String] + def render_schema_type(schema) + if schema.one_of&.any? + schemas = schema.one_of.map { |s| "
  • #{render_schema_type(s)}
    • " } + .join + + return "one of: " + end + + if schema.type == "object" + unless schema.name + if schema.additional_properties? && schema.additional_properties_schema + return "dictionary (#{render_schema_type(schema.additional_properties_schema)})" + end + + return "object" + end + + href = ::Middleman::Util::url_for(@app, "/schemas/#{schema.name}/index.html", { + current_resource: current_resource + }) + + "#{schema.name}" + elsif schema.type == "array" + items = schema.items + + if items.nil? + return "array" + end + + "array (#{render_schema_type(items)})" + else + if schema.name + href = ::Middleman::Util::url_for(@app, "/schemas/#{schema.name}/index.html", { + current_resource: current_resource + }) + + "#{schema.name}" + else + # Make assumption that all oneOf items are same type as + # Swashbuckle theoretically shouldn't allow different types. + if schema.all_of&.any? + return render_schema_type(schema.all_of[0]) + end + + schema.type || "any" + end + end + end + + # @param [Openapi3Parser::Node::Schema] schema + # @param [Openapi3Parser::Node::Schema, String] property + # @return [Boolean] + def is_required_schema_property?(schema, property) + if schema.requires?(property) + return true + end + + if schema.all_of&.any? + return schema.all_of.to_a.reduce(false) do |_, all_of_schema| + all_of_schema.requires?(property) + end + end + + false + end + + # Return true if the schema is a reference (i.e. a $ref). + # + # The OpenAPI parser automatically resolves references for us, but we sometimes + # need to know if the schema was originally a reference. + # + # @param [Openapi3Parser::Node::Schema] schema + # @return [Boolean] + def is_referenced_schema(schema) + schema.node_context.document_location != schema.node_context.source_location + end + + # @param [Openapi3Parser::Node::Schema] schema + # @return [Boolean] + def is_complex_schema(schema) + !!(schema.all_of || schema.any_of || schema.one_of || schema.not) + end + + # @param [Openapi3Parser::Node::Schema] schema + # @return [Boolean] + def is_primitive_schema(schema) + !is_complex_schema(schema) && !schema.type.nil? && schema.type != "object" && schema.type != "array" + end + + # @return [Boolean] + def has_schema_validations(schema) + schema.min_length != 0 || + schema.max_length != nil || + schema.min_items != 0 || + schema.max_items != nil || + schema.min_properties != 0 || + schema.max_properties != nil || + schema.minimum != nil || + schema.maximum != nil || + schema.unique_items? + end + + class Utils + # @param [Openapi3Parser::Node::Schema] schema + # @return [String, Number, Boolean] + def self.primitive_schema_example(schema) + if schema.example + return schema.example + end + + if schema.default + return schema.default + end + + case schema.type + when "string" + schema.format ? "string(#{schema.format})" : "string" + when "number", "integer" + 0 + when "boolean" + true + else + raise "Invalid primitive schema type: #{schema.type}" + end + end + end +end diff --git a/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb b/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb new file mode 100644 index 00000000000..64449f821aa --- /dev/null +++ b/src/explore-education-statistics-api-docs/lib/api_reference_pages_extension.rb @@ -0,0 +1,126 @@ +class ApiReferencePagesExtension < Middleman::Extension + expose_to_template :api_url + + def initialize(app, options_hash = {}, &block) + super + + @sitemap = app.sitemap + @config = app.config[:tech_docs] + @endpoint_template = Middleman::Util.normalize_path("/endpoints/template.html") + @schema_template = Middleman::Util.normalize_path("/schemas/template.html") + + app.ignore @endpoint_template + app.ignore @schema_template + end + + # @param [List] resources + # @return [List] + def manipulate_resource_list(resources) + api_docs_path = @config[:api_docs_path] + + if api_docs_path.nil? + return resources + end + + document = if uri?(api_docs_path) + Openapi3Parser.load_url(api_docs_path) + elsif File.exist?(api_docs_path) + # Load api file and set existence flag. + Openapi3Parser.load_file(api_docs_path) + else + raise "Unable to load `api_docs_path` from config/tech-docs.yml" + end + + new_resources = [] + + @base_url = document.servers[0]&.url || "" + + document.paths.each do |uri, http_methods| + get_operations(http_methods).each do |http_method, operation| + new_resources << create_endpoint_page(uri, http_method, operation) + end + end + + document.components.schemas.each do |_, schema| + new_resources << create_schema_page(schema) + end + + resources + new_resources + end + + private + + # @param [String] uri + # @param [String] http_method + # @param [Openapi3Parser::Node::Operation] operation + # @return [Middleman::Sitemap::ProxyResource] + def create_endpoint_page(uri, http_method, operation) + id = operation.operation_id + + Middleman::Sitemap::ProxyResource.new( + @sitemap, + Middleman::Util.normalize_path("/endpoints/#{id}/index.html"), + @endpoint_template + ).tap do |p| + p.add_metadata locals: { + title: operation.summary, + url: api_url(uri), + http_method: http_method.upcase, + description: operation.description, + parameters: operation.parameters, + request_body: operation.request_body, + responses: operation.responses + } + end + end + + # @param [String] uri + # @return [String] + def api_url(uri = "") + @base_url.chomp("/") + uri + end + + private + + # @param [Openapi3Parser::Node::Schema] schema + # @return [Middleman::Sitemap::ProxyResource] + def create_schema_page(schema) + name = schema.name + + Middleman::Sitemap::ProxyResource.new( + @sitemap, + Middleman::Util.normalize_path("/schemas/#{name}/index.html"), + @schema_template + ).tap do |p| + p.add_metadata locals: { + title: name, + schema: schema, + } + end + end + + # @param [Openapi3Parser::Node::PathItem] path + # @return [Hash] + def get_operations(path) + { + "get" => path.get, + "put" => path.put, + "post" => path.post, + "delete" => path.delete, + "patch" => path.patch, + }.compact + end + + # @param [String] string + # @return [Boolean] + def uri?(string) + uri = URI.parse(string) + %w[http https].include?(uri.scheme || "") + rescue URI::BadURIError + false + rescue URI::InvalidURIError + false + end +end + +Middleman::Extensions.register(:api_reference_pages, ApiReferencePagesExtension) diff --git a/src/explore-education-statistics-api-docs/lib/govuk_tech_docs/path_helpers.rb b/src/explore-education-statistics-api-docs/lib/govuk_tech_docs/path_helpers.rb new file mode 100644 index 00000000000..55047da798e --- /dev/null +++ b/src/explore-education-statistics-api-docs/lib/govuk_tech_docs/path_helpers.rb @@ -0,0 +1,18 @@ +module PatchedPathHelpers + # Monkey patch this function as it doesn't correctly generate + # paths when using relative links. This means that the TOC + # won't open correctly on the current page as the paths are wrong. + def get_path_to_resource(config, resource, current_page) + if defined? app + Middleman::Util::url_for(app, resource, { current_resource: current_page }) + else + super + end + end +end + +module GovukTechDocs + module PathHelpers + prepend PatchedPathHelpers + end +end diff --git a/src/explore-education-statistics-api-docs/lib/helpers.rb b/src/explore-education-statistics-api-docs/lib/helpers.rb new file mode 100644 index 00000000000..a23f252598f --- /dev/null +++ b/src/explore-education-statistics-api-docs/lib/helpers.rb @@ -0,0 +1,30 @@ +module Helpers + + # @param [*] value + # @return [String] + def json_pretty(value) + JSON.pretty_generate(value) + end + + # Render markdown to HTML when working in non-Markdown + # contexts e.g. a HTML fragment. + # + # @param [String] content + # @return [String] + def render_markdown(content) + markdown = Redcarpet::Markdown.new(config[:markdown][:renderer]) + markdown.render(content) + end + + def host_url + config[:tech_docs][:host] + end + + # @param [String] text + # @return [String] + def link_to_contact_md(text = "") + email = config[:tech_docs][:contact_email] + + "[#{text.presence || email}](mailto:#{email})" + end +end diff --git a/src/explore-education-statistics-api-docs/lib/notification/expired.rb b/src/explore-education-statistics-api-docs/lib/notification/expired.rb new file mode 100644 index 00000000000..c94d65398a2 --- /dev/null +++ b/src/explore-education-statistics-api-docs/lib/notification/expired.rb @@ -0,0 +1,29 @@ +require 'date' + +module Notification + class Expired + def include?(page) + page.review_by <= Date.today + end + + def line_for(page) + age = (Date.today - page.review_by).to_i + expired_when = if page.review_by == Date.today + "today" + elsif age == 1 + "yesterday" + else + "#{age} days ago" + end + "- <#{page.url}|#{page.title}> (#{expired_when})" + end + + def singular_message + "I've found a page that is due for review" + end + + def multiple_message + "I've found %s pages that are due for review" + end + end +end diff --git a/src/explore-education-statistics-api-docs/lib/notification/will_expire_by.rb b/src/explore-education-statistics-api-docs/lib/notification/will_expire_by.rb new file mode 100644 index 00000000000..124d08c8521 --- /dev/null +++ b/src/explore-education-statistics-api-docs/lib/notification/will_expire_by.rb @@ -0,0 +1,33 @@ +require 'date' + +module Notification + class WillExpireBy + def initialize(expiry_date) + @expiry_date = expiry_date + end + + def include?(page) + page.review_by > Date.today && page.review_by <= @expiry_date + end + + def line_for(page) + age = (page.review_by - Date.today).to_i + expires_when = if page.review_by == Date.today + "today" + elsif age == 1 + "tomorrow" + else + "in #{age} days" + end + "- <#{page.url}|#{page.title}> (#{expires_when})" + end + + def singular_message + "I've found a page that will expire on or before #{@expiry_date}" + end + + def multiple_message + "I've found %s pages that will expire on or before #{@expiry_date}" + end + end +end diff --git a/src/explore-education-statistics-api-docs/lib/notifier.rb b/src/explore-education-statistics-api-docs/lib/notifier.rb new file mode 100644 index 00000000000..3e177e52d95 --- /dev/null +++ b/src/explore-education-statistics-api-docs/lib/notifier.rb @@ -0,0 +1,140 @@ +require 'http' +require 'json' +require 'date' +require 'uri' + +require_relative './page' +require_relative './notification/expired' +require_relative './notification/will_expire_by' + +class Notifier + def initialize(notification, pages_url, slack_url, live, limit = -1) + @notification = notification + @pages_url = pages_url + @slack_url = slack_url + @live = !!live + @limit = limit + end + + def run + payloads = message_payloads(pages_per_channel) + + return if payloads.empty? + + puts "== JSON Payload:" + puts JSON.pretty_generate(payloads) + + if Date.today.saturday? || Date.today.sunday? + puts "SKIPPING POST: Not posting anything, this is not a working day" + return + end + + unless post_to_slack? + puts "SKIPPING POST: Not posting anything, this is a dry run" + return + end + + payloads.each do |message_payload| + if ENV.has_key? "SLACK_TOKEN" + response = HTTP + .auth("Bearer #{ENV['SLACK_TOKEN']}") + .post("https://slack.com/api/chat.postMessage", json: message_payload) + .parse + + if !response["ok"] then + if response["error"] == "invalid_auth" then + raise "Unable to post to Slack: SLACK_TOKEN is not valid" + else + puts "Unable to post to Slack: #{response['error']}" + end + end + else + HTTP.post(@slack_url, body: JSON.dump(message_payload)) + end + end + end + + def pages + begin + JSON.parse(HTTP.get(@pages_url)).map { |data| + data['url'] = get_absolute_url(data['url']) + Page.new(data) + } + rescue => exception + warn "Notifier: could not get pages for tech docs at #{@pages_url}" + warn exception.message + return [] + end + end + + def pages_per_channel + pages + .reject { |page| page.review_by.nil? } + .select { |page| @notification.include?(page) } + .group_by { |page| page.owner } + .map do |owner, pages| + [owner, pages.sort_by { |page| page.review_by }] + end + end + + def message_payloads(grouped_pages) + grouped_pages.map do |channel, pages| + + page_count = @limit == -1 ? pages.size : [pages.size, @limit].min + notification_message = page_count == 1 ? @notification.singular_message : @notification.multiple_message + number_of = notification_message % [page_count] + + page_lines = pages[0..page_count-1].map do |page| + @notification.line_for(page) + end + + message_prefix = ENV.fetch('OVERRIDE_SLACK_MESSAGE_PREFIX', "Hello :paw_prints:, this is your friendly manual spaniel.") + message = <<~doc + #{message_prefix} #{number_of}: + + #{page_lines.join("\n")} + doc + + channel = ENV.fetch('OVERRIDE_SLACK_CHANNEL', channel) + username = ENV.fetch('OVERRIDE_SLACK_USERNAME', "Daniel the Manual Spaniel") + icon_emoji = ENV.fetch('OVERRIDE_SLACK_ICON_EMOJI', ":daniel-the-manual-spaniel:") + + puts "== Message to #{channel}" + puts message + + { + username: username, + icon_emoji: icon_emoji, + text: message, + mrkdwn: true, + channel: channel, + } + end + end + + def post_to_slack? + @live + end + + private + + def get_absolute_url url + target_uri = URI(url) + target_path = Pathname.new(target_uri.path) + source_uri = URI(@pages_url) + + if target_path.relative? + resulting_path = URI::join(source_uri, target_uri.path).path + else + resulting_path = target_uri.path + end + + if source_uri.scheme == 'https' + URI::HTTPS.build(scheme: source_uri.scheme, port: source_uri.port, host: source_uri.host, path: resulting_path).to_s + else + URI::HTTP.build(scheme: source_uri.scheme, port: source_uri.port, host: source_uri.host, path: resulting_path).to_s +end + end +end + + diff --git a/src/explore-education-statistics-api-docs/lib/page.rb b/src/explore-education-statistics-api-docs/lib/page.rb new file mode 100644 index 00000000000..cba3735fecd --- /dev/null +++ b/src/explore-education-statistics-api-docs/lib/page.rb @@ -0,0 +1,12 @@ +require 'date' + +class Page + attr_reader :url, :title, :review_by, :owner + + def initialize(page_data) + @url = page_data["url"] + @title = page_data["title"] + @review_by = page_data["review_by"].nil? ? nil : Date.parse(page_data["review_by"]) + @owner = page_data["owner_slack"] + end +end diff --git a/src/explore-education-statistics-api-docs/nginx/conf/includes/custom_header.conf b/src/explore-education-statistics-api-docs/nginx/conf/includes/custom_header.conf new file mode 100644 index 00000000000..fad8732e137 --- /dev/null +++ b/src/explore-education-statistics-api-docs/nginx/conf/includes/custom_header.conf @@ -0,0 +1 @@ +add_header Cache-Control "private, max-age=60"; \ No newline at end of file diff --git a/src/explore-education-statistics-api-docs/source/accessibility-statement/index.html.md.erb b/src/explore-education-statistics-api-docs/source/accessibility-statement/index.html.md.erb new file mode 100644 index 00000000000..2ba98e96844 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/accessibility-statement/index.html.md.erb @@ -0,0 +1,64 @@ +--- +title: Accessibility statement for Technical Documentation Template and documentation +last_reviewed_on: 2024-10-29 +review_in: 2 months +hide_in_navigation: true +--- + +# Accessibility statement + +This accessibility statement applies to the <%= link_to host_url, host_url %> website. +This website is run by the [Department for Education (DfE)](https://www.gov.uk/government/organisations/department-for-education). + +This statement does not cover any other services run by the Department for Education (DfE) or GOV.UK. + +## How you should be able to use this website + +We want as many people as possible to be able to use this website. You should be able to: + +- change colours, contrast levels and fonts using browser or device settings +- zoom in up to 400% without the text spilling off the screen +- navigate most of the website using a keyboard or speech recognition software +- listen to most of the website using a screen reader (including the most recent versions of JAWS, NVDA and VoiceOver) + +We've also made the website text as simple as possible to understand. + +[AbilityNet](https://mcmw.abilitynet.org.uk/) has advice on making your device easier to use if you +have a disability. + +## How accessible this website is + +This website is fully compliant with the [Web Content Accessibility Guidelines version 2.1](https://www.w3.org/TR/WCAG21/) +AA standard. + +## Feedback and contact information + +If you find any problems not listed on this page, need information in a different format, or think +we're not meeting accessibility requirements, contact us: + +- email <%= link_to_contact_md %> + +In your message, include: + +- the web address (URL) of the content + +## Enforcement procedure + +The Equality and Human Rights Commission (EHRC) is responsible for enforcing the Public Sector Bodies +(Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018 (the ‘accessibility regulations’). + +If you’re not happy with how we respond to your complaint, [contact the Equality Advisory and Support Service (EASS)](https://www.equalityadvisoryservice.com/). + +## Technical information about this website’s accessibility + +The Department for Education (DfE) is committed to making its website accessible, in accordance with +the Public Sector Bodies (Websites and Mobile Applications) (No. 2) Accessibility Regulations 2018. + +## Compliance status + +This website is fully compliant with the [Web Content Accessibility Guidelines version 2.1](https://www.w3.org/TR/WCAG21/) +AA standard. + +## Preparation of this accessibility statement + +This statement was prepared on 27 September 2024. It was last reviewed on 27 September 2024. diff --git a/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb b/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb new file mode 100644 index 00000000000..bef4b101d9d --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/endpoints/index.html.md.erb @@ -0,0 +1,22 @@ +--- +title: Endpoints +last_reviewed_on: 2024-09-16 +review_in: 24 months +weight: 100 +--- + +# API Endpoints + +This section lists all the endpoints available on the explore education statistics (EES) API. + +The URL for the EES API is: <%= link_to api_url, api_url %> + +The documentation for each endpoint is composed of information about the request and response, with +examples and schemas provided. + +For convenience, request examples in different languages/tools are provided. We currently support: + +- cURL +- JavaScript +- Python +- R diff --git a/src/explore-education-statistics-api-docs/source/endpoints/template.html.md.erb b/src/explore-education-statistics-api-docs/source/endpoints/template.html.md.erb new file mode 100644 index 00000000000..ddba5950b3f --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/endpoints/template.html.md.erb @@ -0,0 +1,187 @@ +# <%= title %> + +<%= description %> + +The URL for this endpoint is: + +``` +<%= http_method %> <%= url %> +``` + +<% if parameters.any? %> +## Parameters + +<% path_parameters = parameters.select { |parameter| parameter.in == "path" } %> + +<% if path_parameters.any? %> +### Path parameters + +The following parameters will need to be substituted into the URL path. + +<%= partial("partials/parameter_table", :locals => { parameters: path_parameters }) %> +<% end %> + +<% query_parameters = parameters.select { |parameter| parameter.in == "query" } %> + +<% if query_parameters.any? %> +### Query parameters + +<%= partial("partials/parameter_table", :locals => { parameters: query_parameters }) %> +<% end %> + +<% headers = parameters.select { |parameter| parameter.in == "header" } %> + +<% if headers.any? %> +### Headers +<%= partial("partials/parameter_table", :locals => { parameters: headers }) %> +<% end %> +<% end %> + +<% if request_body && request_body.content["application/json"] %> +## Request body + +The request body is described by the <%= render_schema_type(request_body.content["application/json"].schema) %> +schema which contains the following parameters: + +<% +schema = request_body.content["application/json"].schema +body_schema = schema.type.nil? && schema.all_of&.count == 1 ? schema.all_of[0] : schema +body_parameters = get_schema_properties(schema) +%> + + + + + + + + + + + + <% body_parameters.each do |name, parameter| %> + + + + + + + <% end %> + +
    NameTypeRequiredDescription
    <%= name %><%= render_schema_type(parameter) %><%= body_schema.requires?(parameter) %> + <%= partial("partials/schema_description", :locals => { + description: parameter.description, + schema: parameter + }) %> +
    + +### Example request body + +<% request_json_content = request_body.content["application/json"] + request_json_example = json_pretty(request_json_content["example"] || + schema_example(request_json_content.schema)) +%> +```json +<%= request_json_example %> +``` +<% end %> + +## Example request + +To illustrate how to use this API endpoint, we have provided some samples below in various +languages. + +<%= partial("partials/tabs", :locals => { + title: 'Languages', + tabs: [ + { + title: "cURL", + id: "example-request-curl", + html: partial("partials/request_example_curl", :locals => { + url: url, + http_method: http_method, + body_json: request_json_example + }) + }, + { + title: "JavaScript", + id: "example-request-js", + html: partial("partials/request_example_js", :locals => { + url: url, + http_method: http_method, + body_json: request_json_example + }) + }, + { + title: 'Python', + id: 'example-request-py', + html: partial("partials/request_example_py", :locals => { + url: url, + http_method: http_method, + body_json: request_json_example + }) + }, + { + title: 'R', + id: 'example-request-r', + html: partial("partials/request_example_r", :locals => { + url: url, + http_method: http_method, + body_json: request_json_example + }) + } + ] +}) %> + +<% if responses.any? %> +## Responses + + + + + + + + + + + +<% responses.each do |status, response| %> +<% response.content.each do |media_type, content| %> + + + + + + +<% end %> +<% end %> + +
    StatusDescriptionMedia TypeSchema
    <%= status %><%= render_markdown(response.description) %> + <%= media_type %> + + <%= render_schema_type(content.schema) %> +
    + +<% if responses[200] %> +### Example successful response + +<% response = responses[200] + response_json = response.content["application/json"] +%> + +<% if response_json %> +<% if response_json["example"] + example = json_pretty(response_json["example"]) + else + example = json_pretty(schema_example(response_json.schema)) + end %> + +<% unless example.blank? %> +```json +<%= example %> +``` +<% end %> +<% end %> +<% end %> +<% end %> diff --git a/src/explore-education-statistics-api-docs/source/getting-started/building-api-integrations/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/building-api-integrations/index.html.md.erb new file mode 100644 index 00000000000..fc7c9911faa --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/getting-started/building-api-integrations/index.html.md.erb @@ -0,0 +1,19 @@ +--- +title: Building API integrations +last_reviewed_on: 2024-10-24 +review_in: 2 months +weight: 3 +--- + +# Building API integrations + +To support building integrations on top of the explore education statistics API, software development +kits (SDKs) are provided to streamline common tasks and communication with the API. + +We are keen to hear from users who are interested in having more dedicated support for their +chosen language or software. If you have any ideas, questions or would like to work with us to +develop support for your chosen language or software, [contact us via email](/support/index.html). + +## Available SDKs + +- for R users - [eesyapi package](https://github.com/dfe-analytical-services/eesyapi) diff --git a/src/explore-education-statistics-api-docs/source/getting-started/creating-advanced-data-set-queries/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/creating-advanced-data-set-queries/index.html.md.erb new file mode 100644 index 00000000000..3106070fd30 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/getting-started/creating-advanced-data-set-queries/index.html.md.erb @@ -0,0 +1,476 @@ +--- +title: Creating advanced data set queries +last_reviewed_on: 2024-09-16 +review_in: 12 months +weight: 4 +--- + +# Creating advanced data set queries + +The explore education statistics (EES) API allows you to query data sets using a syntax that can +express highly complex criteria. + +In this guide, you'll learn about: + +- basic query syntax +- condition clauses +- chaining conditions +- sorting results + +By the end, you should have an understanding of all the possible options and how they can be combined. + +## What you'll need + +You should already be familiar with the EES API. If not, you should read the [Quick start](/getting-started/quick-start/index.html) +first as this guide will presume some prior knowledge. + +To run request examples in this guide, it is a good idea to come prepared with a HTTP or API client +tool. Good recommendations for beginners are [Postman](https://www.postman.com/) +or [Insomnia](https://insomnia.rest/). + +Some prior knowledge of working with your chosen HTTP client will be necessary to work with the +examples. + +## The basic query syntax + +Data set queries can be made using a POST request to the [Query a data set](/endpoints/QueryDataSetPost/index.html) +endpoint. At its most basic, such a request would look like the following: + +``` +POST <%= api_url "/api/v1/data-sets/{dataSetId}/query" %> +{ + "criteria": {}, + "indicators": [] +} +``` + +The request body must contain `facets` and `indicators` properties. These filter the result data that +can be in the response and must be populated using the IDs of facets from the data set metadata +(see the [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) endpoint). + +The `indicators` property controls what data values are shown in the results. This should simply +contain a list of indicator IDs. For example, using the following indicators: + +```json +{ + "indicators": ["7SJdk", "1J57c"] +} +``` + +Each result in the response would look like the following: + +```json +{ + "values": { + "7SJdk": "123", + "1J57c": "245" + } +} +``` + +Note that the values are keyed by their indicator's respective name rather than the indicator's ID. + +### Facet types + +The `criteria` property controls what results will be in the response by filtering them based on the +query criteria specified. + +At its most simple, the `criteria` should contain a **facets object** that can have any of the +following properties: + +| Property | Description | Facet option examples | +|--------------------|--------------------------------------|----------------------------------------------------------------------------------| +| `filters` | Filter by filter option ID | `1oX7c`, `uKR2K` | +| `geographicLevels` | Filter by geographic level code | `LA`, `REG`, `NAT` | +| `locations` | Filter by location option ID or code | `{ "level: "REG", "code": "E12000001" }`,
    `{ "level: "NAT", "id": "2FmYX" }` | +| `timePeriods` | Filter by time period | `{ "period": "2022/2023", "code": "AY" }` | + + +Note that all the facet properties are **optional** so you only need to use the ones relevant to +your query. + +#### Using IDs or codes for locations + +It should be noted that `locations` can accept IDs (e.g. `2FmYX`) and codes (e.g. `E12000001`). +These will match locations in significantly different ways: + +- an ID will only refer to a **single** location +- a code may refer to **multiple** locations + +Location codes are typically unique, but the same code may be used across multiple locations. +This may produce unexpected results in cases where codes are re-used, so it is recommended to use +IDs where possible to only get the exact locations you are interested in. + +### Facet comparators + +For each facet property in the query, its value should be a **comparator object**. This describes +how the facet's values should be compared when filtering the results. + +The full list of comparators permitted is as follows: + +| Comparator | Description | Multiple values? | Example | +|------------|--------------------------|------------------|--------------------------------------------------| +| `eq` | Equal to | No | `"eq": "1oX7c"` | +| `notEq` | Not equal to | No | `"notEq": "1oX7c"` | +| `in` | In a set | Yes | `"in": ["1oX7c", "uKR2K"]` | +| `notIn` | Not in a set | Yes | `"notIn": ["1oX7c", "uKR2K"]` | +| `lte` | Less than or equal to | No | `"lte": { "period": "2022/2023", "code": "AY" }` | +| `lt` | Less than | No | `"lt": { "period": "2022/2023", "code": "AY" }` | +| `gte` | Greater than or equal to | No | `"gte": { "period": "2022/2023", "code": "AY" }` | +| `gt` | Greater than | No | `"gt": { "period": "2022/2023", "code": "AY" }` | + +Note that facet properties may only permit certain comparators to be used. Consult the +[schema documentation](/schemas/DataSetQueryCriteriaFacets/index.html) for each facet property to see if a +comparator is allowed. + +Using all the above information, you could write a query that looks like the following: + +```json +{ + "criteria": { + "filters": { + "in": ["qtY3J", "psUkV"] + }, + "timePeriod": { + "gte": { "period": "2018/2019", "code": "AY" }, + "lte": { "period": "2022/2023", "code": "AY" } + }, + "locations": { + "notIn": [ + { "level": "REG", "id": "0rWTr" }, + { "level": "REG", "id": "pn6kV" } + ] + }, + "geographicLevel": { + "eq": "LA" + } + } +} +``` + +The results will be filtered so they: + +- match either filter options `qtY3J` or `psUkV` +- are between the 2018/19 and 2022/23 academic years +- don't match location options `0rWTr` or `pn6kV` +- only contain local authority level data + +It should be noted that **all parts** of the query criteria must resolve to true for a result to be +included in the response. + +Whilst the above query may be suitable for simpler queries, you may have more complex requirements. +In the next section, you'll explore some of the more advanced query syntax available. + +## Queries with condition clauses + +For more advanced queries, you can specify multiple sets of criteria that results should match. +These sets of criteria need to be combined using condition clauses that express the relationship +between each set. + +In the query syntax, a condition clause is expressed through a **condition object**. Currently, +`and`, `or` and `not` condition objects are supported by the API. + +### The 'and' condition + +The `and` condition can be used when multiple sub-criteria **must all** be true for the overall +condition to be true. The syntax for this looks like the following: + +```json +{ + "criteria": { + "and": [ + { + "filters": { + "in": ["qtY3J", "psUkV"] + } + }, + { + "locations": { + "notEq": { "level": "REG", "id": "0rWTr" } + } + }, + { + "timePeriods": { + "gte": { "period": "2018/2019", "code": "AY" }, + "lte": { "period": "2022/2023", "code": "AY" } + } + } + ] + } +} +``` + +The results will be filtered so that they: + +- match either filter options `qtY3J` or `psUkV` +- don't match location option `0rWTr` +- are between the 2018/19 and 2022/23 academic years + +Note that a facet object is equivalent to a set of `and` conditions as it combines all of its +clauses together using a logical AND. The earlier example would be equivalent to the following +facet object: + +```json +{ + "criteria": { + "filters": { + "in": ["qtY3J", "psUkV"] + }, + "locations": { + "notEq": { "level": "REG", "id": "0rWTr" } + }, + "timePeriods": { + "gte": { "period": "2018/2019", "code": "AY" }, + "lte": { "period": "2022/2023", "code": "AY" } + } + } +} +``` + +It's typically simpler to use facet objects instead of combining multiple clauses using an `and` +condition. Facet objects are more compact and their use should be preferred where possible. + +### The 'or' condition + +The `or` condition can be used when there are multiple sub-criteria and **only one** must be true +for the overall condition to be true. The syntax for this looks like the following: + +```json +{ + "criteria": { + "or": [ + { + "filters": { + "eq": "qtY3J" + }, + "timePeriods": { + "gte": { "period": "2016/2017", "code": "AY" }, + "lte": { "period": "2017/2018", "code": "AY" } + } + }, + { + "filters": { + "eq": "psUkV" + }, + "timePeriods": { + "gte": { "period": "2021/2022", "code": "AY" } + } + } + ] + } +} +``` + +The results will be filtered so that they either match: + +- filter option `qtY3J` and are between the 2016/17 and 2017/18 academic years +- filter option `psUkV` and are after the 2021/22 academic year + +The `or` condition is particularly useful for expressing complex queries where there are multiple +sets of distinct criteria. A good use-case is for matching on multiple ranges of time periods +like the above example. + +### The 'not' condition + +The `not` condition can be used when a condition **must not** be true, hence it negates the +condition's result. + +Unlike, the `and` / `or` conditions, the `not` condition operates on a single sub-clause and the +syntax looks like the following: + +```json +{ + "criteria": { + "not": { + "filters": { + "eq": "qtY3J" + }, + "timePeriods": { + "gt": { "period": "2021/2022", "code": "AY" } + } + } + } +} +``` + +The results will be filtered so that they **must not**: + +- match filter option `qtY3J` +- be after the 2021/21 academic year + +It's not recommended to use a 'not' condition for cases where a standard facet object would suffice. +A facet object can contain negated comparators such `notEq`, `notIn`, etc, and can typically express +the same things that the 'not' condition can. + +## Chaining conditions + +In more complex queries, you may need to chain multiple conditions together. With the API, it's +possible to do this as the query syntax allows condition objects to contain either facet or +another condition objects in a nested way. + +For example, a query like the following is possible: + +```json +{ + "criteria": { + "and": [ + { + "filters": { + "eq": "qtY3J" + } + }, + { + "or": [ + { + "timePeriods": { + "gte": { "period": "2016/2017", "code": "AY" }, + "lte": { "period": "2017/2018", "code": "AY" } + } + }, + { + "timePeriods": { + "gt": { "period": "2020/2021", "code": "AY" } + } + } + ] + } + ] + } +} +``` + +The results will be filtered so that they: + +- match filter option `qtY3J` +- are between the 2016/17 and 2017/18 academic years, or after the 2020/21 academic year + +The `not` condition can also contain other condition objects. For example: + +```json +{ + "criteria": { + "not": { + "or": [ + { + "filters": { + "eq": "qtY3J" + }, + "timePeriods": { + "gte": { "period": "2016/2017", "code": "AY" }, + "lte": { "period": "2017/2018", "code": "AY" } + } + }, + { + "filters": { + "eq": "psUkV" + }, + "timePeriods": { + "gt": { "period": "2021/2022", "code": "AY" } + } + } + ] + } + } +} +``` + +The results will be filtered so that they **must not**: + +- match filter option `qtY3J` and be between the 2016/17 and 2017/18 academic years +- match filter option `psUkV` and be after the 2021/22 academic year + +There are no limits to how deeply condition objects can be chained and nested. You can use as many +conditions to express your query's requirements as necessary. + +## Sorting results + +By default, query results are sorted by their time period in descending order. This means that the +most recent results will be listed first. + +If you want to change the ordering of the results, you can use the query's `sorts` property. +This property should be a list of sorts that should be applied to the results, for example: + +```json +{ + "sorts": [ + { "field": "timePeriod", "direction": "Asc" } + ] +} +``` + +This will order results by time period in ascending order, meaning that the oldest results would +appear first. + +The `field` property should be the name of the field to sort, and the `direction` controls the +sort direction (i.e. `Asc` for ascending, `Desc` for descending). + +Possible options for `field` include: + +- `timePeriod` to sort by time period +- `geographicLevel` to sort by the geographic level of the data +- `location|{level}` to sort by location options in a geographic level where `{level}` is the level + code (e.g. `REG`, `LA`) +- `filter|{id}` to sort by the options in a filter where `{id}` is the filter ID (e.g. `3RxWP`) +- `indicator|{id}` to sort by the values in an indicator where `{id}` is the indicator ID + (e.g. `6VfPgZ`) + +Any geographic levels, filter IDs and indicator IDs found in the data set's metadata can be used +for sorting. + +You can also have multiple sorts to determine the order in tie-break situations. These will be +applied in the order that they appear in the `sorts` list. For example, given the following: + +```json +{ + "sorts": [ + { "field": "timePeriod", "direction": "Desc" }, + { "field": "location|REG", "direction": "Desc" }, + { "field": "filter|q1o13J", "direction": "Asc" } + ] +} +``` + +The results will be sorted by: + +1. time period in descending order +2. location options at 'Region' level in descending order (when in the same time period) +3. filter options in filter `q1o13J` in ascending order (when in the same location option) + +## Documenting queries with comments + +To make it easier to work with data set queries manually, the query's JSON body is allowed to +contain comments using forward slashes (`//`). You can add comments to any part of the query as long +as it doesn't result in a malformed JSON structure. + +For example, the following is a **valid** query with comments: + +```jsonc +{ + "criteria": { + "filters": { + "in": [ + "n0WqP", // State-funded secondary + "hUfBQ" // Gender male + ] + } + } +} +``` + +The following is an **invalid** query with comments: + +```jsonc +{ + "criteria": { + "filters": { + "in": [ + "n0WqP" // State-funded secondary, + "hUfBQ" // Gender male + ] + } + } +} +``` + +In the invalid query, the comma needed after `"n0WqP"` has been accidentally placed within the +adjacent comment causing the JSON to be syntactically invalid. diff --git a/src/explore-education-statistics-api-docs/source/getting-started/debugging-data-set-queries/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/debugging-data-set-queries/index.html.md.erb new file mode 100644 index 00000000000..8a7fb40ff29 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/getting-started/debugging-data-set-queries/index.html.md.erb @@ -0,0 +1,413 @@ +--- +title: Debugging data set queries +last_reviewed_on: 2024-09-16 +review_in: 12 months +weight: 5 +--- + +# Debugging data set queries + +In this guide, you'll learn various techniques to help debug potential issues with data set queries +when using the explore education statistics (EES) API. + +## What you'll need + +You should already be familiar with the EES API. If not, you should read the [Quick start](/getting-started/quick-start/index.html) +first as this guide will presume some prior knowledge. + +You should already be familiar with the basic usages of the following endpoints to proceed: + +- [Query a data set](/endpoints/QueryDataSetPost/index.html) +- [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) + +## Diagnosing error responses + +In most cases, if the API is unable to process a request successfully, it will return an error +response. The success or failure of a request is primarily indicated by its HTTP status code, +where: + +- 2xx status codes (e.g. 200, 204) indicate a success +- 4xx status codes (e.g. 400, 404) indicate an error with the request itself, such as a validation issue +- 5xx status codes (e.g. 500, 503) indicate an error that occurred within the API whilst processing + +When there is an error response, the body will typically look like: + +```json +{ + "title": "There was a problem processing the request.", + "type": "Internal Server Error", + "status": 500 +} +``` + +The response body is modelled by the [ProblemDetailsViewModel](/schemas/ProblemDetailsViewModel/index.html) schema, +which attempts to detail the reason(s) why the request failed. The following fields are always +included: + +| Property | Type | Description | +|----------|--------|----------------------------------------------------------------| +| `title` | string | The title of the error. Typically summarises the error. | +| `type` | string | The error type. Usually corresponds with the HTTP status code. | +| `status` | number | The HTTP status code. | + +If there are validation issues with the request, the response body will also contain an `errors` +field which looks like: + +```json +{ + "title": "There are validation errors with the request.", + "type": "Bad Request", + "status": 400, + "errors": [ + { + "message": "Error message", + "code": "error.code", + "path": "theField" + } + ] +} +``` + +The `errors` property contains a list of errors. Each error corresponds to a specific problem and +will at least contain a `message` (describing the issue) and a `code` (for further debugging and +parsing). + +If the error relates to a specific part of the request, the `path` property is used to describe the +path to request property that caused the error. If this is omitted or empty, it means the error is +'global' and relates to the entire request. + +Where possible, errors may also contain a `detail` field that provides more detailed information +about the problem. + +Validation errors and error responses in general are covered in much more detail in the section on +[Error handling](/overview/error-handling/index.html). + +## Validation errors for data set queries + +The [Query a data set](/endpoints/QueryDataSetPost/index.html) endpoint will usually try to process a +query as much as possible before a validation error response is sent (instead of failing early). +Consequently, the response typically aggregates as many validation errors as possible. + +Common validation errors will be discussed in more detail in the following sections. + +### Incompatible comparator values + +Validation errors are commonly caused by some data set query criteria containing comparators that +use the wrong data type for their values. For example, a query with the following criteria: + +```json +{ + "criteria": { + "filters": { + "eq": ["filter-1"] + } + } +} +``` + +Will result in an error response like: + +```json +{ + "errors": [ + { + "message": "Must be a valid value. Check that the type and format are correct.", + "path": "criteria.filters.eq", + "code": "InvalidValue" + } + ] +} +``` + +The above query is using a `eq` comparator with an array. Arrays are typically only used with +comparators that can accept **multiple values** like `in` and `notIn`. + +#### Solution + +To correct the error, you can simply change `eq` to `in`: + +```json +{ + "criteria": { + "filters": { + "in": ["filter-1"] + } + } +} +``` + +Alternatively, the array can be replaced with a single filter item ID string: + +```json +{ + "criteria": { + "filters": { + "eq": "filter-1" + } + } +} +``` + +It is recommended that you read the guide on [Creating advanced data set queries](/getting-started/creating-advanced-data-set-queries/index.html#facet-comparators) +as it contains far more detail about each comparator, and how they are used with different facet types. + +### Incorrect use of condition clauses + +Validation errors can commonly occur when writing more complex queries using condition clauses +such as `and`, `or` and `not`. For example: + +```json +{ + "criteria": { + "not": [ + { + "filters": { + "eq": "..." + } + } + ] + } +} +``` + +In the above example, an array is used as the `not` clause value. Unfortunately, arrays are +incompatible and will result in an error response like: + +```json +{ + "errors": [ + { + "message": "Must be a valid value. Check that the type and format are correct.", + "path": "criteria.not", + "code": "InvalidValue" + } + ] +} +``` + +#### Solution + +Check that your query correctly follows the [DataSetQueryRequest](/schemas/DataSetQueryRequest/index.html) +schema. Pay close attention to any usages of condition clauses. + +The `and` / `or` clauses accept **multiple** criteria or condition clauses in an array: + +```json +{ + "criteria": { + "and": [ + { + "filters": { "eq": "..." } + }, + { + "locations": { "eq": "..." } + } + ] + } +} +``` + +The `not` clause only accepts a **single** condition clause: + +```json +{ + "criteria": { + "not": { + "filters": { "eq": "..." } + } + } +} +``` + +For a better understanding of condition clauses, the guide on [Creating advanced data set queries](/getting-started/creating-advanced-data-set-queries/index.html#queries-with-condition-clauses) +goes into much greater detail on this topic. + +## Warnings in successful data set query responses + +In certain cases, a successful data set query may include warnings that indicate something is +potentially wrong with the request. Whilst these are not critical errors, it is advisable that you +double-check that your query to ensure that is functioning as expected. + +Warnings in the response typically look like the following: + +```json +{ + "paging": { + "page": 1, + "pageSize": 100, + "totalResults": 150, + "totalPages": 2 + }, + "warnings": [ + { + "message": "The query did not match any results. You may need to refine your criteria.", + "code": "QueryNoResults" + } + ], + "results": [...] +} +``` + +The format of a warning is the same as an error (see section on [Error handling](/overview/error-handling/index.html) +for more details) and will include at least a `message` and `code`. + +### No query results + +If a query does not return any results, you'll receive a warning like: + +```json +{ + "warnings": [ + { + "message": "The query did not match any results. You may need to refine your criteria.", + "code": "QueryNoResults" + } + ] +} +``` + +Depending on the use-case, this may be correct behaviour, however, it may also indicate that there +is an issue with the query. + +#### Solution + +In most cases, the query is likely using criteria that is too specific and the matching data does +not exist within the data set. Some tweaking of your query may be required to make the criteria +less specific + +If your query previously worked but begins to return no results, this may be due to the data set +itself changing in a backwards incompatible way e.g. the removal of data, or a major change in the +data set's facets. + +The EES API makes every effort to avoid publishing backwards incompatible data that may disrupt +existing queries, however, these types of changes may still occur from time to time (deliberately +or otherwise). + +The versioning policy is outlined in more detail in the [Versioning](/overview/versioning/index.html) +overview. + +### Missing facets + +Before a data set query is executed by the API, the facets in the query are pre-validated to ensure +that they exist in the data set. If some facets are missing, this may cause queries to have zero +results (as they cannot be matched). + +When there are missing facets, a response will look like: + +```json +{ + "warnings": [ + { + "message": "One or more filters could not be found.", + "path": "criteria.filters.in", + "code": "FiltersNotFound", + "details": { + "items": ["invalid-filter-1", "invalid-filter-2"] + } + } + ] +} +``` + +In these types of responses, the `notFound` warning informs you about the specific items that are +missing (e.g. `invalid-filter-1`) in the `details` property. + +#### Solution + +Ensure that all facets in the data set query exist in the corresponding data set metadata. +You should check this by cross-referencing the missing facets with the [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) +endpoint. + +Facets are not usually removed from existing data sets, so there may be a typo (or similar) in +the missing facets. + +## Documenting queries with comments + +Data set queries can be documented with comments in the JSON body using forward slashes (`//`). You +can add comments to any part of the query as long as it doesn't result in a malformed JSON structure. + +For example, the following is a **valid** query with comments: + +```jsonc +{ + "criteria": { + "filters": { + "in": [ + "n0WqP", // State-funded secondary + "hUfBQ" // Gender male + ] + } + } +} +``` + +The following is an **invalid** query with comments: + +```jsonc +{ + "criteria": { + "filters": { + "in": [ + "n0WqP" // State-funded secondary, + "hUfBQ" // Gender male + ] + } + } +} +``` + +In the invalid query, the comma needed after `"n0WqP"` has been accidentally placed within the +adjacent comment causing the JSON to be syntactically invalid. + +## Debug mode + +To assist in debugging unexpected results for a data set query, the [Query a data set](/endpoints/QueryDataSetPost/index.html) +endpoint also accepts a `debug` query parameter that enables debug mode. This can be set in the +request's query string like so: + +``` +<%= api_url %>/api/v1/data-sets/{dataSetId}/query?debug=true +``` + +The response will then be modified to return results that look like the following: + +```json +{ + "filters": { + "Z3PMP :: ethnicity": "bqJZ4 :: Asian - Chinese", + "eW168 :: language": "LVRpO :: Total", + "b0yZ4 :: phase_type_grouping": "RXIeh :: State-funded secondary" + }, + "timePeriod": { + "code": "AY", + "period": "2021/2022" + }, + "geographicLevel": "LA", + "locations": { + "NAT": "dv84z :: England :: E92000001", + "REG": "T4Y1o :: East Midlands :: E12000004", + "LA": "vNVmD :: Derby :: E06000015" + }, + "values": { + "dv84z :: headcount": "79", + "vNVmD :: percent_of_pupils": "0.429161234" + } +} +``` + +The keys and values of `filters`, `locations` and `values` are changed to display human-readable +labels and facet IDs in the format `{facet ID} :: {label}`. + +Enabling debug mode is useful to avoid having to cross-reference the facets of each result with the +data set's metadata (using the [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) endpoint). + +However, it is important to note that debug mode **should not** be used outside of development / debugging +purposes. When your query's issues have been resolved, you should disable debug mode before pushing +your query to production. + +Using debug mode in production comes with significant issues such as: + +- much larger (~2-3x) responses that consume more bandwidth +- slower responses due to extra server-side processing needed +- being subject to lower rate limits +- needing additional client-side parsing of the human-readable labels and facet IDs diff --git a/src/explore-education-statistics-api-docs/source/getting-started/how-to-get-csv-data/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/how-to-get-csv-data/index.html.md.erb new file mode 100644 index 00000000000..026836a6c7a --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/getting-started/how-to-get-csv-data/index.html.md.erb @@ -0,0 +1,46 @@ +--- +title: How to get CSV data +last_reviewed_on: 2024-09-16 +review_in: 6 months +weight: 2 +--- + +# How to get CSV data + +In this guide, you'll learn how to get CSV data from the explore education statistics (EES) API. +This may be particularly useful if you wish to access data sets in their entirety, or find it more +comfortable to work with CSVs. + +## What you'll need + +You should already be familiar with the EES API. If not, you should read the [Quick start](/getting-started/quick-start/index.html) +first as this guide will presume some prior knowledge. + +To run request examples in this guide, it is a good idea to come prepared with a HTTP or API client +tool. Good recommendations for beginners are [Postman](https://www.postman.com/) or [Insomnia](https://insomnia.rest/). + +Some prior knowledge of working with your chosen HTTP client will be necessary to work with the +examples. + +## Get an entire data set as a CSV file + +Every data set is created from an underlying CSV file that contains **all** the data. You may find +it useful (or necessary) to work with the underlying CSV instead of interacting with the data set +via the API. + +The underlying CSV file can be downloaded via the [Download data set CSV](/endpoints/DownloadDataSetCsv/index.html) +endpoint. To use this endpoint, you need to make a `GET` request: + +``` +GET <%= api_url "/api/v1/data-sets/{dataSetId}/csv" %> +``` + +In the request URL, you'd substitute the `{dataSetId}` parameter with your chosen data set's ID. + +Upon making this request, a download containing the CSV data should start. The file will +be gzip compressed (like most of the API responses), meaning that your HTTP client will need to +support gzip decompression to extract the contents. + +It should be noted that the CSV data will be use human-readable labels for its contents instead of +machine-readable identifiers. This means that unlike the rest of the API, there are no backward +compatibility guarantees between different versions of the data set. diff --git a/src/explore-education-statistics-api-docs/source/getting-started/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/index.html.md.erb new file mode 100644 index 00000000000..7b936ecd2c8 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/getting-started/index.html.md.erb @@ -0,0 +1,44 @@ +--- +title: Getting started +last_reviewed_on: 2024-09-16 +review_in: 12 months +weight: 2 +--- + +# Getting started + +The explore education statistics (EES) API provides a way to directly consume published data from +the EES service using HTTP. + +To get started with the EES API, you'll need either: + +- an API client such as [Postman](https://www.postman.com/) or [Insomnia](https://insomnia.rest/) + if you are exploring the API +- an HTTP client in the programming language of your choice if you are developing on top of the API + +The documentation on this website will assume a level of familiarity with how HTTP, web APIs and +REST work. You should research these topics prior to using the EES API if you are not familiar with +them. + +It is recommended that you start with the [Quick start guide](/getting-started/quick-start/index.html) +if you are new to the EES API. This will guide you through essential parts of the API and the workflow to +perform a basic data set query. + +The [Overview](/overview/index.html) section is a good next step for a more in-depth understanding +of the high level parts of the EES API. + +To assist in specific tasks once you've started using the EES API, there are additional guides under +**Getting started** in the navigation menu that you may find useful. + +## Documentation structure + +The [Overview section](/overview/index.html) provides high level documentation about the EES API. +This details things such as message formats, error handling, versioning and the OpenAPI specification. + +The [Endpoints section](/endpoints/index.html) provides reference documentation about the endpoints +available in the API. This details the requests that can be made and their responses. Code samples are +also provided to illustrate how requests could be made. + +The [Schemas section](/schemas/index.html) provides reference documentation about the structure of +all the requests and responses (i.e. their schemas) across the API. Each schema provides in-depth +detail about their properties, including their type and validation rules. diff --git a/src/explore-education-statistics-api-docs/source/getting-started/quick-start/index.html.md.erb b/src/explore-education-statistics-api-docs/source/getting-started/quick-start/index.html.md.erb new file mode 100644 index 00000000000..8bebd9eb796 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/getting-started/quick-start/index.html.md.erb @@ -0,0 +1,592 @@ +--- +title: Quick start +last_reviewed_on: 2024-09-16 +review_in: 6 months +weight: 1 +--- + +# Quick start + +In this guide, you'll learn about how the explore education statistics (EES) API is structured and +how to use it to perform a basic data query. + +## What you'll need + +To run request examples in this guide, it is a good idea to come prepared with a HTTP or API client +tool. Good recommendations for beginners are [Postman](https://www.postman.com/) +or [Insomnia](https://insomnia.rest/). + +Some prior knowledge of working with your chosen HTTP client will be necessary to work with the +examples. + +## How the API is organised + +The API endpoints are organised in a way that reflects how data is organised in EES. + +Data is published in **publications**. Each publication covers a specific topic such as +schools or higher education. + +A publication will contain **data sets** that are relevant to the particular topic. Data sets are +composed of data that has been collected over a period of time at varying geographic levels e.g. for +local authorities. Updates to data sets are typically published at regular intervals e.g. yearly, +monthly, etc. + +Given the above, the API exposes endpoints that mirror this: + +- publication endpoints: `/publications` +- data set endpoints: `/data-sets` + +## Workflow for querying data + +To query the data available on the API, this will require the following steps: + +1. Find the publication you are interested in +2. Find the data set you interested in (from the publication) +3. Get the data set's metadata +4. Create and run a query against the data set + +In the following sections, this guide will walk you through how to perform the above steps. + +### Step 1: Find a publication + +To find a publication that you may be interested in, you'll need to make a `GET` request to the +[List publications](/endpoints/ListPublications/index.html) endpoint: + +``` +GET <%= api_url "/api/v1/publications" %> +``` + +This endpoint will respond with something like the following (parts have been omitted for brevity): + +```json +{ + "paging" : { + "page" : 1, + "pageSize" : 20, + "totalPages" : 3, + "totalResults" : 50 + }, + "results" : [ + { + "id" : "cbbd299f-8297-44bc-92ac-558bcf51f8ad", + "slug" : "Pupil-absence-in-schools-in-England", + "title" : "Pupil absence in schools in England", + "summary": "Pupil absence, including overall, authorised and unauthorised absence...", + "lastPublished": "2023-11-10T09:15:00+00:00" + } + ] +} +``` + +This endpoint does not return all publications in a single request. Instead, it is **paginated** +and returns the publications in pages (or batches), with each page containing a maximum number of +publications. + +You can request additional pages of publications by appending a `page` query parameter to the +endpoint URL. For example: + +``` +# Fetch page 2 +GET <%= api_url "/api/v1/publications?page=2" %> + +# Fetch page 3 +GET <%= api_url "/api/v1/publications?page=3" %> +``` + +The possible values of `page` will be dictated by the total number of results (across all pages) +and the `pageSize` query parameter. For example, the following request would show 30 results per +page instead of the default: + +``` +GET <%= api_url "/api/v1/publications?page=1&pageSize=30" %> +``` + +Each page of results contains a `paging` property which describes the current page and the total +numbers of pages and results. This information can be used to set the query parameters for the next +page of results. + +To make it easier to find a specific publication, you can append a `search` query parameter to the +URL as well. The following example would search for publications matching the term 'pupil absence': + +``` +GET <%= api_url "/api/v1/publications?search=pupil+absence" %> +``` + +Like a typical URL, you can combine query parameters together with `&`. For example, you'd use +the following URL to get page 2 of publications matching the term 'pupil absence': + +``` +GET <%= api_url "/api/v1/publications?search=pupil+absence&page=2" %> +``` + +Once you find a publication you are interested in, proceed to the next step. + +### Step 2: Find a data set + +Now that you have a publication that you are interested, you can use this to find data sets related +to it. This can be done using +the [List a publication's data sets](/endpoints/ListPublicationDataSets/index.html) +endpoint: + +``` +GET <%= api_url "/api/v1/publications/{publicationId}/data-sets" %> +``` + +For this endpoint URL, you'd substitute the `{publicationId}` parameter with the `id` of the +publication you are interested in. + +For example, given the following publication (parts omitted for brevity): + +```json +{ + "id" : "cbbd299f-8297-44bc-92ac-558bcf51f8ad", + "slug" : "Pupil-absence-in-schools-in-England", + "title" : "Pupil absence in schools in England", + "summary": "Pupil absence, including overall, authorised and unauthorised absence...", + "lastPublished": "2023-11-10T09:15:00+00:00" +} +``` + +You'd make the following `GET` request: + +``` +GET <%= api_url "/api/v1/publications/cbbd299f-8297-44bc-92ac-558bcf51f8ad/data-sets" %> +``` + +The endpoint responds with a paginated list of the publication's data sets which will look like the +following: + +```json +{ + "paging": { + "page": 1, + "pageSize": 10, + "totalResults": 1, + "totalPages": 1 + }, + "results": [ + { + "id": "63cfc86e-c334-4e58-2912-08da0807d53c", + "title": "Absence rates", + "summary": "Absence information for full academic year 2020/21 for pupils aged 5-15.", + "status": "Published", + "latestVersion": { + "version": "1.0", + "published": "2022-12-01T12:00:00Z", + "totalResults": 201625, + "file": { + "id": "84ee3cc9-21bf-44d8-89fd-98c6d7fc74f3" + }, + "timePeriods": { + "start": "2020/21", + "end": "2020/21" + }, + "geographicLevels": [ + "National", + "Local authority" + ], + "filters": [ + "Phase type", + "Characteristic" + ], + "indicators": [ + "Number of authorised absence sessions", + "Number of unauthorised absence sessions" + ] + } + } + ] +} +``` + +Each data set result provides high-level information about its contents and metadata. You can use +this information to help identify a data set that you'd be interested in looking at further. + +Once you have chosen a data set, proceed to the next step. + +### Step 3: Get the data set's metadata + +Now that you have a chosen a data set, you'll want to query it for some data. To create a query, +you'll need to use the [Get a data set's metadata](/endpoints/GetDataSetMeta/index.html) endpoint. +This provides information about all the filterable facets and indicators available to a data set. + +**Facets** are specific features / characteristics of the data. These are used in a data set query +to filter down the data that is returned. + +Some examples of facets include: + +- time periods e.g. 2022/23 (academic year), 2023 (calendar year), January (month), Week 1 (week) +- locations e.g. England (country), Yorkshire (region), Sheffield (local authority) +- school type e.g. state-funded primary, state-funded secondary +- pupil characteristics like ethnicity and gender + +**Indicators** are types of data points that were collected, for example: + +- numbers of pupils, sessions, etc +- rates of change +- proportions / percentages + +Both facets and indicators need to be part of a query for data to be returned. + +Facets and indicators are collectively referenced as a data set's **metadata**. To fetch this for +your chosen data set, make the following `GET` request: + +``` +GET <%= api_url "/api/v1/data-sets/{dataSetId}/meta" %> +``` + +To use this URL, substitute in the `{dataSetId}` parameter with the `id` of your chosen data set. + +The endpoint will return something like the following: + +```json +{ + "filters": [ + { + "id": "gIyO9", + "column": "school_type", + "label": "School type", + "hint": "Filter by school type", + "options": [ + { + "id": "1oX7c", + "label": "Total", + "isAggregate": true + }, + { + "id": "uKR2K", + "label": "State-funded primary" + } + ] + } + ], + "indicators": [ + { + "id": "04nTr", + "column": "sess_authorised", + "label": "Number of authorised absence sessions", + "unit": "", + "decimalPlaces": 0 + } + ], + "geographicLevels": [ + { + "code": "NAT", + "label": "National" + }, + { + "code": "REG", + "label": "Regional" + } + ], + "locations": [ + { + "level": { + "code": "NAT", + "label": "National" + }, + "options": [ + { + "id": "2FmYX", + "code": "E92000001", + "name": "England" + } + ] + }, + { + "level": { + "code": "REG", + "label": "Regional" + }, + "options": [ + { + "id": "e0768", + "code": "E12000001", + "name": "North East" + }, + { + "id": "GQbUn", + "code": "E12000002", + "name": "North West" + } + ] + } + ], + "timePeriods": [ + { + "code": "AY", + "label": "2021/22", + "period": "2021/2022" + }, + { + "code": "AY", + "label": "2022/23", + "period": "2022/2023" + } + ] +} +``` + +The **core facets** are found under the `timePeriods`, `geographicLevels` and `locations` properties. + +The `locations` property contains the data set's location options grouped by the geographic level +they reside in. Each location option has an `id` and may contain additional code fields (e.g. ONS +codes) to identify them. + +In the above example, there are location options for: + +- 'England' (`2FmYX`) at 'National' geographic level +- 'North East' (`e0768`) and 'North West' (`GQbUn`) at 'Regional' geographic level + +The `geographicLevels` property contains the different geographic levels that the data was collected +at. Each geographic level is identified by its `code` and a full list of these can be found in the +[GeographicLevelCode schema](/schemas/GeographicLevelCode/index.html). + +In the above example, there are geographic level options for 'National' (`NAT`) and 'Regional' (`REG`) +geographic levels. + +The `timePeriods` property contains the time periods the data was collected at. The time period options +are represented by a `code` that describes the time period's type and a `period` that describes the +date range. A full list of time period codes can be found in the [TimePeriodCode schema](/schemas/TimePeriodCode/index.html). + +In the above example there is a single time period option for 'academic year 2022/23'. + +Any **additional facets** are found under the `filters` property, which groups them by their filter. +Each filter is identified by an `id`. The example above has a 'School type' filter with an ID of `gIyO9`. + +Each filter will have a set of filter options. The example above has 'State-funded primary' and +'Total' options for 'School type'. Filter options also have their own `id` properties that can be used +to identify them. + +Some filter options are aggregates of the entire filter and are marked by an `isAggregate` property +set to true. In the above example, the 'Total' school type is an aggregate of all school types +(e.g. primary, secondary, special schools, etc). + +Finally, the `indicators` property contains the data set's indicators. Each indicator contains an +`id` to identify it and may also contain: + +- a `unit` property specifying the mathematical unit that was used in the measurement +- a `decimalPlaces` property specifying the recommended number of decimal places to use when + displaying the indicator's value + +Spend some time getting familiarised with the metadata response and proceed to the next step when +ready. + +### Step 4: Create and run your data set query + +In this final step, you'll need to use the metadata from the previous step to create and run your +query against the [Query a data set](/endpoints/QueryDataSetPost/index.html) endpoint. + +To use this endpoint, a `POST` request needs to be sent to the endpoint URL with an appropriate +request body. The most basic request would look like the following: + +``` +POST <%= api_url "/api/v1/data-sets/{dataSetId}/query" %> +{ + "indicators": [] +} +``` + +As seen previously, you need to substitute the `{dataSetId}` parameter with the `id` of your chosen +data set. + +The request body must contain an `indicators` property with a list of indicator IDs, which can can +found in the data set's metadata (under each indicator option's `id` property). The data values in +the response will correspond to the indicators that you specify. + +To refine your query to a subset of the data, you will also need to provide some filtering criteria +by adding a `criteria` property to your query request: + +``` +POST <%= api_url "/api/v1/data-sets/{dataSetId}/query" %> +{ + "indicators": [], + "criteria": {} +} +``` + +The `criteria` property at its simplest must be an object that describes which facets the query +should filter on. The facet object has properties that align with the different facet types seen +in the metadata step previously. + +The table below describes the facet properties you can use and how each facet option should be +represented in the query: + +| Property | Description | Facet option examples | +|--------------------|--------------------------------------|----------------------------------------------------------------------------------| +| `filters` | Filter by filter option ID | `1oX7c`, `uKR2K` | +| `geographicLevels` | Filter by geographic level code | `LA`, `REG`, `NAT` | +| `locations` | Filter by location option ID or code | `{ "level: "REG", "code": "E12000001" }`,
    `{ "level: "NAT", "id": "2FmYX" }` | +| `timePeriods` | Filter by time period | `{ "period": "2022/2023", "code": "AY" }` | + +Note that all the facet properties are **optional** so you only need to use the ones relevant to +your query. + +Each facet property must contain an object that describe how the facet options should be compared to +the results when filtering. Some examples of comparators that can be used: + +| Comparator | Description | Multiple values? | Example | +|------------|--------------------------|------------------|--------------------------------------------------| +| `eq` | Equal to | No | `"eq": "1oX7c"` | +| `notEq` | Not equal to | No | `"notEq": "1oX7c"` | +| `in` | In a set | Yes | `"in": ["1oX7c", "uKR2K"]` | +| `notIn` | Not in a set | Yes | `"notIn": ["1oX7c", "uKR2K"]` | +| `lte` | Less than or equal to | No | `"lte": { "period": "2022/2023", "code": "AY" }` | +| `lt` | Less than | No | `"lt": { "period": "2022/2023", "code": "AY" }` | +| `gte` | Greater than or equal to | No | `"gte": { "period": "2022/2023", "code": "AY" }` | +| `gt` | Greater than | No | `"gt": { "period": "2022/2023", "code": "AY" }` | + +Using the above information and the metadata example from the previous step, the following query +could be constructed: + +```json +{ + "criteria": { + "filters": { + "eq": ["uKR2K", "1oX7c"] + } + }, + "indicators": ["04nTr"] +} +``` + +This query would filter so that only results in the 'State-funded primary' (filter option `uKR2K`) +school type would be returned. Each result would then contain the 'Number of authorised absence +sessions' (indicator `04nTr`) in its data values. + +You can add multiple clauses to the facet criteria object to refine your query further. For a fuller +example, you could construct a query like the following: + +```json +{ + "criteria": { + "filters": { + "in": ["uKR2K", "1oX7c"] + }, + "timePeriods": { + "lte": { "period": "2021/2022", "code": "AY" }, + "gte": { "period": "2022/2023", "code": "AY" } + }, + "locations": { + "notIn": [ + { "level": "REG", "id": "e0768" } + ] + }, + "geographicLevels": { + "eq": "LA" + } + }, + "indicators": ["04nTr"] +} +``` + +The above example would query for the 'Number of authorised absence sessions' (indicator `04nTr`) +matching the following criteria: + +- is for 'State-funded primary' (filter option `uKR2K`) or 'Total' (filter option `1oX7c`) school types +- is during or after the 2021/22 academic year (time period `2021/2022` and code `AY`) +- is during or before the 2022/23 academic year (time period `2022/2023` and code `AY`) +- is not in the 'North East' (location `e0768`) +- is collected at 'Local authority' level (geographic level `LA`) + +Different facet values and comparators can be provided to modify the query in different ways. +It's advisable to spend a little time getting more familiar with the query API. + +The [Creating advanced data set queries](/getting-started/creating-advanced-data-set-queries/index.html) +guide explores this topic in greater depth and is recommended for further reading. + +### The data set query response + +Once you have created your query, make the `POST` request to the endpoint. You should receive a +paginated response that looks like: + +```json +{ + "paging": { + "page": 1, + "pageSize": 100, + "totalResults": 150, + "totalPages": 2 + }, + "results": [ + { + "timePeriod": { + "code": "AY", + "period": "2022/2023" + }, + "geographicLevel": "REG", + "locations": { + "NAT": "2FmYX", + "REG": "e0768" + }, + "filters": { + "gIyO9": "uKR2K" + }, + "values": { + "04nTr": "1708016" + } + } + ] +} +``` + +The `results` part of this response contains a list of results containing a combination of facets +and the data matching it. + +The `timePeriod` property describes the time period that the result was collected in. In the above +example, this is academic year 2022/2023. + +The `geographicLevel` property describes the geographic level that the data was collected in. In +the above example, the result's data was collected at 'Regional' (`REG`) level. + +The `locations` property describe the set of locations that correspond to the result. This is a +dictionary where the keys correspond to geographic level codes and the values are an option ID within +the corresponding geographic level. + +In the above example, the result's locations were 'England' (`2FmYX` at 'National' level) and +'North East' (`e0768` at 'Regional' level). + +The `filters` property describes the additional facets corresponding to the result. This is a +dictionary where the keys are the filter ID and the values are an option ID within the corresponding +filter. + +In the above example, the result's 'School type' (`gIyO9`) filter was 'State-funded primary' (`uKR2K`). + +The `values` property of each result is a dictionary where the keys are the indicator IDs and +the values are the respective indicator data values. + +In the above example, the 'Number of authorised absence sessions' (`04nTr`) indicator has a value +of `1708016`. + +Note that reported values may not be numeric. In some instances, it may not be possible to report +the data (e.g. due to suppression for anonymity) and a placeholder value may be used instead. + +Spend some time getting familiar with the structure of the results and try to find some results you +are interested in. + +#### Note on paginated data + +Like some endpoints seen previously, the data set query's response is **paginated** meaning that the +data is returned in multiple pages / batches. The `paging` property is returned as part of each +response and describes the current page of data matching the query. + +You can set `page` and `pageSize` parameters in the query string to request different pages of +results. For example, the following request would fetch page 5, with each page containing +a maximum of 200 results: + +``` +POST <%= api_url "/api/v1/data-sets/{dataSetId}/query?page=5&pageSize=200" %> +``` + +## Conclusions + +This quick start guide has now run you through a basic workflow for retrieving some data from the +EES API. The core workflow is the same for all data sets. The majority of use-cases will simply +require you to adjust the parameters used. + +You should now have the basic tools to get started with the API, but you are encouraged to explore the +documentation further. It is recommended that you read the [Overview section](/overview/index.html) to +get a better understanding of the core API features. + +To learn more about data set queries and how to create more complex ones, it is recommended that you +read the guide to [Creating advanced data set queries](/getting-started/creating-advanced-data-set-queries/index.html). diff --git a/src/explore-education-statistics-api-docs/source/index.html.md.erb b/src/explore-education-statistics-api-docs/source/index.html.md.erb new file mode 100644 index 00000000000..a87b6388b70 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/index.html.md.erb @@ -0,0 +1,21 @@ +--- +title: API documentation +weight: 1 +last_reviewed_on: 2024-09-18 +review_in: 12 months +--- + +# Explore education statistics API documentation + +This is the technical documentation for the [explore education statistics](https://explore-education-statistics.service.gov.uk/) +API. + +Use this documentation if you wish to: + +- learn about and use the API +- integrate your service to consume data from the API + +The [Getting started](/getting-started/index.html) section is a good place to start and will provide essential +information on how to use the API. + +[Contact us](/support/index.html) if you have any questions or feedback. diff --git a/src/explore-education-statistics-api-docs/source/javascripts/application.js b/src/explore-education-statistics-api-docs/source/javascripts/application.js new file mode 100644 index 00000000000..3c7f6ab8761 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/javascripts/application.js @@ -0,0 +1,2 @@ +//= require govuk_tech_docs +//= require ./copy.js diff --git a/src/explore-education-statistics-api-docs/source/javascripts/copy.js b/src/explore-education-statistics-api-docs/source/javascripts/copy.js new file mode 100644 index 00000000000..b64ecd6e957 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/javascripts/copy.js @@ -0,0 +1,46 @@ +(() => { + class Copy { + constructor($module) { + this.$module = $module; + } + + init() { + // Bail if no clipboard support (e.g. IE11) + if (!navigator.clipboard) { + return; + } + + const $button = document.createElement('button'); + $button.className = 'app-copy-button js-copy-button'; + $button.setAttribute('aria-live', 'assertive'); + $button.textContent = 'Copy code'; + + this.$module.insertAdjacentElement('beforebegin', $button); + + $button.addEventListener('click', this.handleCopy); + } + + handleCopy(event) { + const target = event.target.nextElementSibling; + + navigator.clipboard + .writeText(target.textContent) + .then(() => { + // eslint-disable-next-line no-param-reassign + event.target.textContent = 'Code copied'; + + setTimeout(() => { + // eslint-disable-next-line no-param-reassign + event.target.textContent = 'Copy code'; + }, 5000); + }) + .catch(err => { + console.error(err); + }); + } + } + + document.querySelectorAll('pre.highlight').forEach($el => { + new Copy($el).init(); + }); +})(); diff --git a/src/explore-education-statistics-api-docs/source/javascripts/govuk_frontend.js b/src/explore-education-statistics-api-docs/source/javascripts/govuk_frontend.js new file mode 100644 index 00000000000..c3588dfe5d7 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/javascripts/govuk_frontend.js @@ -0,0 +1 @@ +//= require govuk_frontend_all diff --git a/src/explore-education-statistics-api-docs/source/overview/error-handling/index.html.md.erb b/src/explore-education-statistics-api-docs/source/overview/error-handling/index.html.md.erb new file mode 100644 index 00000000000..316324ac50b --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/overview/error-handling/index.html.md.erb @@ -0,0 +1,173 @@ +--- +title: Error handling +last_reviewed_on: 2024-09-18 +review_in: 12 months +weight: 1 +--- + +# Error handling + +The explore education statistics (EES) API uses conventional HTTP status codes and response bodies to +present errors from the API. + +Typically, 2xx status codes indicate a success, 4xx status codes indicate an error with the request +itself (e.g. there were validation errors), and 5xx status codes indicate an error that occurred +within the API itself. + +The HTTP status codes that you can typically expect from the API are summarised by the following table: + +| Status code | Name | Description | +|-------------|-------------------|--------------------------------------------------------------------------------------------------------------------------------| +| 200 | OK | Success. The API should provide a successful response. | +| 204 | No Content | Success. The request succeeded, but there was no content in the response. | +| 400 | Bad Request | There were issues (e.g. validation errors) with the request meaning it could not be processed. | +| 401 | Unauthorized | The request lacks valid authentication credentials and was denied access. | +| 403 | Forbidden | The request has authentication credentials, but was denied access to the requested resource. | +| 404 | Not Found | The requested resource could not be found. | +| 429 | Too Many Requests | There were too many requests in a short amount of time. Avoid making further requests, or use an appropriate backoff strategy. | +| 504 | Gateway Timeout | The request took too long and exceeded the maximum allowed time. | + +## The error response body + +If an error occurs, the EES API will respond with a body that looks like the following: + +```json +{ + "title": "There was a problem processing the request.", + "type": "Internal Server Error", + "status": 500 +} +``` + +The response body is modelled by the [ProblemDetailsViewModel](/schemas/ProblemDetailsViewModel/index.html) +schema, which attempts to detail the reason(s) why the request failed. The following fields are always +included: + +| Property | Type | Description | +|----------|--------|----------------------------------------------------------------| +| `title` | string | The title of the error. Typically summarises the error. | +| `type` | string | The error type. Usually corresponds with the HTTP status code. | +| `status` | number | The HTTP status code. | + +## Handling validation errors + +The API validates requests to ensure that the inputs make sense before they are processed. Issues +can be things like: + +- missing parameters +- values that are not allowed +- values that are malformed or not formatted correctly + +If there are validation errors with the request, the API will respond with a body that looks like +the following: + +```json +{ + "title": "There are validation errors with the request.", + "type": "Bad Request", + "status": 400, + "errors": [ + { + "message": "Error message", + "code": "ErrorCode", + "path": "someField" + } + ] +} +``` + +This response contains the validation errors in the `errors` property. Every validation error will +at least contain a `message` property describing the specific problem to address. + +Errors will also typically contain a `code` property. These can be useful for diagnosing issues +further, or for simply parsing errors with consuming code. + +If the error relates to a specific part of the request, the `path` property is used to describe the +path to request property that caused the error. If this is omitted or empty, it means the error is +'global' and relates to the entire request. + +Where possible, errors may also contain a `detail` property to provide further exact +details about the problem. These do not have a specific structure, but may look something like: + +```json +{ + "message": "Must be one of the allowed values.", + "code": "AllowedValue", + "path": "someField", + "detail": { + "items": [25, 30] + } +} +``` + +In the above validation error, the `detail` indicates that only the numbers 25 and 30 are values +that can be used for the `someField` request property. + +### Global errors + +If a validation error relates to the request as a whole, there will no specific request parameter +that it relates to. + +These types of validation errors are considered 'global' and the `path` will be unset or empty. This +looks like the following: + +```json +{ + "title": "There are validation errors with the request.", + "type": "Bad Request", + "status": 400, + "errors": [ + { + "message": "A global error message", + "code": "GlobalErrorCode" + } + ] +} +``` + +### Deeply nested parameters + +A validation response may also report errors relating to deeply nested parameters in the request. +It does this by describing a path to the specific parameter using JSONPath notation, for example: + + +```json +{ + "title": "There are validation errors with the request.", + "type": "Bad Request", + "status": 400, + "errors": [ + { + "message": "Error message", + "code": "ErrorCode", + "path": "some.nested[1].thing" + } + ] +} +``` + +In the above example, the error would relate to a deeply nested property of a request body that +looks like the following: + +```json +{ + "some": { + "nested": [ + { "thing": "a" }, + { "thing": "b" } + ] + } +} +``` + +The error would relate to the second item in the `nested` array. For that item it would specifically +relate to its `thing` property i.e. the value `"b"`. + +Using the above request structure, validation errors are possible for all the following paths: + +- `some` +- `some.nested` +- `some.nested[0]` +- `some.nested[0].thing` +- `some.nested[1]` +- `some.nested[1].thing` diff --git a/src/explore-education-statistics-api-docs/source/overview/index.html.md.erb b/src/explore-education-statistics-api-docs/source/overview/index.html.md.erb new file mode 100644 index 00000000000..4bc676e96ec --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/overview/index.html.md.erb @@ -0,0 +1,21 @@ +--- +title: Overview +last_reviewed_on: 2024-09-18 +review_in: 6 months +weight: 3 +--- + +# Overview + +The explore education statistics (EES) API adheres to REST principles. It uses standard HTTP verbs, +JSON encoding in message bodies and provides HTTP status codes for different response types. + +The API primarily provides access to data sets published by EES, allowing you to: + +- get summary information about data sets and their related resources +- query data sets based on specific criteria +- download the underlying CSV files of data sets + +Note that not all data sets available in EES are accessible via the API. For a full list of data sets +published by EES, visit the [Data catalogue](https://explore-education-statistics.service.gov.uk/data-catalogue) +on the main website. diff --git a/src/explore-education-statistics-api-docs/source/overview/openapi-specification/index.html.md.erb b/src/explore-education-statistics-api-docs/source/overview/openapi-specification/index.html.md.erb new file mode 100644 index 00000000000..5529019b3e7 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/overview/openapi-specification/index.html.md.erb @@ -0,0 +1,28 @@ +--- +title: OpenAPI specification +last_reviewed_on: 2024-09-18 +review_in: 12 months +weight: 3 +--- + +# OpenAPI specification + +The explore education statistics API is fully compliant with the OpenAPI 3.0 specification. + +## About OpenAPI + +The [OpenAPI specification](https://swagger.io/specification/) provides a standard language for +documenting REST APIs and their capabilities. Using a document that conforms to this specification, +users should be able to understand how to use an API and navigate through it with minimal effort. + +This service's API reference documentation is generated using the OpenAPI specification. + +## Getting our OpenAPI specification + +You can download the service's OpenAPI specification in <%= link_to "JSON format", config[:tech_docs][:api_path] %>. + +Once you have our specification, you can: + +- explore it with a third-party tool such as Postman or Insomnia +- generate an API client for writing integrations +- run automated tests against it to validate your integrations diff --git a/src/explore-education-statistics-api-docs/source/overview/pagination/index.html.md.erb b/src/explore-education-statistics-api-docs/source/overview/pagination/index.html.md.erb new file mode 100644 index 00000000000..eeb85e00953 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/overview/pagination/index.html.md.erb @@ -0,0 +1,66 @@ +--- +title: Pagination +last_reviewed_on: 2024-09-18 +review_in: 12 months +weight: 2 +--- + +# Pagination + +Endpoints that return a list of results may return a paginated response. This is done for +performance reasons to avoid excessively large responses that would result from returning all results +in a single request. + +When responses are paginated, the list of results is split into multiple pages (or 'batches') that +must be requested separately. + +A typical paginated response looks like the following: + +```json +{ + "paging" : { + "page" : 1, + "pageSize" : 3, + "totalPages" : 2, + "totalResults" : 5 + }, + "results" : [ + { "id": 1 }, + { "id": 2 }, + { "id": 3 } + ] +} +``` + +In the above example, the `paging` property contains paging metadata. This metadata tells you that +there are 5 results (denoted by `totalResults`) split over 2 pages (denoted by `totalPages`). +The `pageSize` property indicates the maximum number of results that can be shown in a single page. + +Typically, you need to provide a `page` query parameter in the URL to specify the page of results +you want. In the above example, you'd use a URL like `/the-endpoint?page=2` to get the second +page of results. + +Some endpoints may allow you to control the page size, meaning that you can adjust the number of +results each page contains. In these cases, a `pageSize` query parameter can be used e.g. +`/the-endpoint?pageSize=5` requests that each page contains 5 results. + +The `pageSize` parameter is not always available and may be validated to prevent excessively large +pages being used. + +## Paging metadata in response headers + +Sometimes the paging metadata cannot be embedded in response body itself, for example, if the +response format is specified to be CSV. + +In these cases, the paging metadata will be contained in the response headers instead. This will +look like the following: + +``` +Page: 2 +Page-Size: 3 +Total-Results: 5 +Total-Pages: 5 +``` + +These headers map directly to the metadata found under the typical `paging` property seen in +paginated JSON responses. diff --git a/src/explore-education-statistics-api-docs/source/overview/versioning/index.html.md.erb b/src/explore-education-statistics-api-docs/source/overview/versioning/index.html.md.erb new file mode 100644 index 00000000000..7afdc27c1e9 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/overview/versioning/index.html.md.erb @@ -0,0 +1,58 @@ +--- +title: Versioning +last_reviewed_on: 2024-09-18 +review_in: 6 months +weight: 4 +--- + +# Versioning + +The Explore Education Statistics (EES) API uses URL-based versioning. Each endpoint encodes its current +version within its URL e.g. `/api/v1/the-endpoint`, `/api/v2/the-endpoint`, etc. + +When backwards-incompatible changes to an endpoint are unavoidable, a new version of the endpoint +will be published. Backwards-incompatible changes may include things like: + +- removing properties in the request or response schema +- changing the structure of the request or response schema +- changing validation rules for the request + +In the event that a new endpoint version is necessary, the previous version(s) of the endpoint will +be maintained for as long as possible. This should provide sufficient opportunity to migrate to the +new endpoint version. + +For new major API versions, guidance on migrating your existing code will be published in the +changelog documentation for the API. + +## Data set versioning + +Data sets available through this API may introduce changes over time. Similar to other types of +versioning, major or minor versions will be published as required. + +The following are considered **major** (breaking) changes: + +- facets (filters, geographic levels, locations, time periods) were deleted +- indicators were deleted + +The following are considered **minor** changes: + +- facets were added +- facet options were added +- facet details were updated (e.g. label, hint) +- facet options were updated (e.g. label) +- indicators were added +- indicator details were updated (e.g. label, unit, decimal places) +- data values were updated + +As long as you have specified a data set version to use, any query that you have previously created +should continue to work as normal regardless of new versions being published. + +### Upgrading to newer versions + +When you wish to upgrade a query to use a newer data set version, it's important to check the +changes introduced by the version and consider if they affect your consuming code. + +You can find lists of changes via: + +- the data set's changelog on its details page on the EES website (see [Data catalogue](https://explore-education-statistics.service.gov.uk/data-catalogue)) +- the [Get a data set version's changes](/endpoints/GetDataSetVersionChanges/index.html) endpoint diff --git a/src/explore-education-statistics-api-docs/source/partials/_parameter_table.html.erb b/src/explore-education-statistics-api-docs/source/partials/_parameter_table.html.erb new file mode 100644 index 00000000000..f567347d1cf --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/partials/_parameter_table.html.erb @@ -0,0 +1,27 @@ +
    + + + + + + + + + + + <% parameters.each do |parameter| %> + + + + + + + <% end %> + +
    ParameterTypeRequiredDescription
    <%= parameter.name %><%= render_schema_type(parameter.schema) %><%= parameter.required? %> + <%= partial("partials/schema_description", :locals => { + description: parameter.description, + schema: parameter.schema + }) %> +
    +
    diff --git a/src/explore-education-statistics-api-docs/source/partials/_request_example_curl.md.erb b/src/explore-education-statistics-api-docs/source/partials/_request_example_curl.md.erb new file mode 100644 index 00000000000..d79ae984f3c --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/partials/_request_example_curl.md.erb @@ -0,0 +1,13 @@ +<% command = "curl -X #{http_method} #{url}" + + if body_json + command += " \\" + end +%> +```bash +<%= command %> +<% if body_json %> + -H 'Content-Type: application/json' \ + -d '<%= body_json %>' +<% end %> +``` diff --git a/src/explore-education-statistics-api-docs/source/partials/_request_example_js.md.erb b/src/explore-education-statistics-api-docs/source/partials/_request_example_js.md.erb new file mode 100644 index 00000000000..80793b378d1 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/partials/_request_example_js.md.erb @@ -0,0 +1,19 @@ +```js +const url = '<%= url %>'; +<% if body_json %> +const headers = { + 'Content-Type': 'application/json' +}; +const body = <%= body_json %>; +<% end %> + +const response = await fetch(url, { + method: '<%= http_method %>', +<% if body_json %> + headers, + body: JSON.stringify(body) + <% end %> +}); + +console.log(response.json()); +``` diff --git a/src/explore-education-statistics-api-docs/source/partials/_request_example_py.md.erb b/src/explore-education-statistics-api-docs/source/partials/_request_example_py.md.erb new file mode 100644 index 00000000000..ef230c06abd --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/partials/_request_example_py.md.erb @@ -0,0 +1,15 @@ +```py +import requests + +url = '<%= url %>' +<% if body_json %> +headers = { + 'Content-Type': 'application/json' +} +data = '''<%= body_json %>''' +<% end %> + +response = requests.<%= http_method.downcase %>(url<% if body_json %>, headers=headers, data=data<% end %>) + +print(response.text) +``` diff --git a/src/explore-education-statistics-api-docs/source/partials/_request_example_r.md.erb b/src/explore-education-statistics-api-docs/source/partials/_request_example_r.md.erb new file mode 100644 index 00000000000..453be44f61f --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/partials/_request_example_r.md.erb @@ -0,0 +1,21 @@ +```r +library(httr) + +url <- "<%= url %>" +<% if body_json %> +body <- "<%= body_json %>" +<% end %> + +<% if body_json %> +response <- <%= http_method.upcase %>( + url, + body = body, + encode = "json", + content_type("application/json") +) +<% else %> +response <- <%= http_method.upcase %>(url) +<% end %> + +output <- content(response) +``` diff --git a/src/explore-education-statistics-api-docs/source/partials/_schema_description.html.md.erb b/src/explore-education-statistics-api-docs/source/partials/_schema_description.html.md.erb new file mode 100644 index 00000000000..82243cfae0a --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/partials/_schema_description.html.md.erb @@ -0,0 +1,35 @@ +<%= description %> + +<% if schema.default %> +Defaults to: `<%= schema.default %>` +<% end %> + +<% if has_schema_validations(schema) %> +Validation constraints: + +<%= partial("partials/schema_validations", :locals => { schema: schema }) %> +<% end %> + +<% if !is_referenced_schema(schema) && schema.enum %> +Allowed options: + +<% schema.enum.each do |item| %> +- `<%= item %>` +<% end %> +<% end %> + +<% if schema.type == "array" && schema.items != nil %> +<% if has_schema_validations(schema.items) %> +Validation constraints for child items: + +<%= partial("partials/schema_validations", :locals => { schema: schema.items }) %> +<% end %> + +<% if !is_referenced_schema(schema.items) && schema.items.enum %> +Allowed options for child items: + +<% schema.items.enum.each do |item| %> +- `<%= item %>` +<% end %> +<% end %> +<% end %> diff --git a/src/explore-education-statistics-api-docs/source/partials/_schema_validations.html.md.erb b/src/explore-education-statistics-api-docs/source/partials/_schema_validations.html.md.erb new file mode 100644 index 00000000000..93386488720 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/partials/_schema_validations.html.md.erb @@ -0,0 +1,37 @@ +<% if schema.format %> +- Format: `<%= schema.format %>` +<% end %> + +<% unless schema.max_length.nil? %> +- Maximum length: `<%= schema.max_length %>` +<% end %> +<% if schema.min_length != 0 %> +- Minimum length: `<%= schema.min_length %>` +<% end %> + +<% unless schema.max_items.nil? %> +- Maximum items: `<%= schema.max_items %>` +<% end %> +<% if schema.min_items != 0 %> +- Minimum items: `<%= schema.min_items %>` +<% end %> + +<% if schema.unique_items? %> +- Items must be unique +<% end %> + +<% unless schema.minimum.nil? && schema.maximum.nil? && schema.format.nil? %> +<% if schema.minimum %> +- <%= schema.exclusive_minimum? ? 'Minimum (exclusive)' : 'Minimum' %>: `<%= schema.minimum %>` +<% end %> +<% if schema.maximum %> +- <%= schema.exclusive_maximum? ? 'Maximum (exclusive)' : 'Maximum' %>: `<%= schema.maximum %>` +<% end %> +<% end %> + +<% unless schema.max_properties.nil? %> +- Maximum properties: `<%= schema.max_properties %>` +<% end %> +<% if schema.min_properties != 0 %> +- Minimum properties: `<%= schema.min_properties %>` +<% end %> diff --git a/src/explore-education-statistics-api-docs/source/partials/_tabs.html.erb b/src/explore-education-statistics-api-docs/source/partials/_tabs.html.erb new file mode 100644 index 00000000000..7abd485ec97 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/partials/_tabs.html.erb @@ -0,0 +1,19 @@ +
    + + + <% tabs.each_with_index do |tab, index| %> +
    +

    <%= tab[:panel_title] || tab[:title] %>

    + + <%= tab[:html] %> +
    + <% end %> +
    diff --git a/src/explore-education-statistics-api-docs/source/robots.txt.erb b/src/explore-education-statistics-api-docs/source/robots.txt.erb new file mode 100644 index 00000000000..1f53798bb4f --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/robots.txt.erb @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/src/explore-education-statistics-api-docs/source/schemas/index.html.md.erb b/src/explore-education-statistics-api-docs/source/schemas/index.html.md.erb new file mode 100644 index 00000000000..1c7690e7544 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/schemas/index.html.md.erb @@ -0,0 +1,21 @@ +--- +title: Schemas +last_reviewed_on: 2024-09-18 +review_in: 12 months +--- + +# API Schemas + +This section lists all the schemas available on the explore education statistics API. + +A schema specifies the data structure of a request or response used in the API. Schemas are +typically composed of a set of properties and some information about each of these, including: + +- the property name +- the property type e.g. string, number, boolean, array, object, etc +- if it is required or optional +- a description of the property +- any validation rules that apply + +Examples of each schema are provided to give a flavour of how the schema will look in practice. +Note that examples **may not** accurately reflect a true request / response from the API. diff --git a/src/explore-education-statistics-api-docs/source/schemas/template.html.md.erb b/src/explore-education-statistics-api-docs/source/schemas/template.html.md.erb new file mode 100644 index 00000000000..3deb4f5b190 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/schemas/template.html.md.erb @@ -0,0 +1,59 @@ +# <%= title %> + +<% if schema.description %> +<%= schema.description %> +<% end %> + +<% if schema.enum %> +Allowed options: + +<% schema.enum.each do |item| %> +- `<%= item %>` +<% end %> +<% end %> + +<% properties = get_schema_properties(schema) %> + +<% if properties.any? %> +## Properties + +
    + + + + + + + + + + + <% properties.each do |name, property| %> + + + + + + + <% end %> + +
    PropertyTypeRequiredDescription
    <%= name %><%= render_schema_type(property) %><%= is_required_schema_property?(schema, property) %> + <%= partial("partials/schema_description", :locals => { + description: property.description, + schema: property + }) %> +
    +
    +<% end %> + +<% unless is_primitive_schema(schema) %> +## Example schema + +<% schema_json = json_pretty(schema_example(schema)) %> + +<% unless schema_json.blank? %> +```json +<%= schema_json %> +``` +<% end %> +<% end %> diff --git a/src/explore-education-statistics-api-docs/source/stylesheets/app.scss b/src/explore-education-statistics-api-docs/source/stylesheets/app.scss new file mode 100644 index 00000000000..2e390db1f24 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/stylesheets/app.scss @@ -0,0 +1,33 @@ +.technical-documentation { + max-width: 50em; +} + +.toc__list li { + a:link, + a:visited { + padding: 8px 10px; + } +} + +.collapsible { + word-break: break-word; +} + +.highlight { + position: relative; +} + +.highlight code, +table code { + font-size: 0.9rem !important; + line-height: 1 !important; +} + +.js-enabled .technical-documentation pre { + padding: govuk-spacing(7) govuk-spacing(3) govuk-spacing(3); + + &:focus { + border: 1px solid govuk-colour('black'); + box-shadow: inset 0 0 0 2px govuk-colour('black'); + } +} diff --git a/src/explore-education-statistics-api-docs/source/stylesheets/components/_copy.scss b/src/explore-education-statistics-api-docs/source/stylesheets/components/_copy.scss new file mode 100644 index 00000000000..58aa50a68c5 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/stylesheets/components/_copy.scss @@ -0,0 +1,44 @@ +.app-copy-button { + $copy-button-colour: #00823b; + + @include govuk-font(16); + background-color: govuk-colour('white'); + box-shadow: 0 2px 0 0 govuk-colour('green'); + border: 1px solid $copy-button-colour; + color: $copy-button-colour; + cursor: pointer; + min-width: 110px; + opacity: 0; + padding: 3px 10px; + position: absolute; + right: govuk-spacing(2); + text-align: center; + text-decoration: none; + transition: opacity 0.2s; + top: govuk-spacing(2); + z-index: 1; + + &:focus:not(:hover) { + background-color: $govuk-focus-colour; + box-shadow: 0 2px 0 0 $govuk-focus-text-colour; + color: $govuk-focus-text-colour; + } + + &:active, + &:focus { + border: 2px solid $govuk-focus-colour; + box-shadow: none; + opacity: 1; + outline: 2px solid transparent; + padding: 2px 10px; // Counter increased border size + } + + &:active { + box-shadow: none; + margin-top: 2px; + } +} + +.highlight:hover > .app-copy-button { + opacity: 1; +} diff --git a/src/explore-education-statistics-api-docs/source/stylesheets/components/_table.scss b/src/explore-education-statistics-api-docs/source/stylesheets/components/_table.scss new file mode 100644 index 00000000000..d16334a404a --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/stylesheets/components/_table.scss @@ -0,0 +1,12 @@ +.app-table-container { + &:focus { + box-shadow: 0 0 0 2px; + outline: $govuk-focus-width solid $govuk-focus-colour; + outline-offset: 4px; + } + + @include govuk-media-query($until: desktop) { + display: block; + overflow-x: auto; + } +} diff --git a/src/explore-education-statistics-api-docs/source/stylesheets/print.css.scss b/src/explore-education-statistics-api-docs/source/stylesheets/print.css.scss new file mode 100644 index 00000000000..8d75fe40e91 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/stylesheets/print.css.scss @@ -0,0 +1,3 @@ +$is-print: true; + +@import 'govuk_tech_docs'; diff --git a/src/explore-education-statistics-api-docs/source/stylesheets/screen.css.scss b/src/explore-education-statistics-api-docs/source/stylesheets/screen.css.scss new file mode 100644 index 00000000000..c8edb234d10 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/stylesheets/screen.css.scss @@ -0,0 +1,9 @@ +@import 'govuk_tech_docs'; +@import 'govuk/components/notification-banner/index'; +@import 'govuk/components/tabs/index'; + +@import './app'; +@import './utils'; + +@import './components/copy'; +@import './components/table'; diff --git a/src/explore-education-statistics-api-docs/source/stylesheets/utils.scss b/src/explore-education-statistics-api-docs/source/stylesheets/utils.scss new file mode 100644 index 00000000000..ea97030f987 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/stylesheets/utils.scss @@ -0,0 +1,3 @@ +.app-word-break--normal { + word-break: normal !important; +} diff --git a/src/explore-education-statistics-api-docs/source/support/index.html.md.erb b/src/explore-education-statistics-api-docs/source/support/index.html.md.erb new file mode 100644 index 00000000000..03e0bad3e96 --- /dev/null +++ b/src/explore-education-statistics-api-docs/source/support/index.html.md.erb @@ -0,0 +1,16 @@ +--- +title: Support +last_reviewed_on: 2024-09-19 +review_in: 24 months +hide_in_navigation: true +--- + +# Support + +The explore education statistics API is operated by the Department for Education (DfE). + +If you need help and support or have a question about the API, contact: + +**Explore education statistics team** + +Email: [explore.statistics@education.gov.uk](mailto:explore.statistics@education.gov.uk).