Improve error handling, tests

Try to ensure the proxy always returns a valid JSON response. Add specific handling of upstream error responses to proxy.py, and add tests and fixtures for that. Replace the Proxy._on_done static method with an instance method. In main.py, bail out upon receiving invalid input JSON, instead of printing an error response and then continuing to try to use it, resulting in printing another error response. Have main.py use the callbacks on the proxy as given, instead of always overwriting them with the defaults from callbacks.py. Rework entrypoint.py so that it should always print a JSON response, and add tests for it.
freedomofpress · Jan 3, 2020 · 20064fc · 20064fc
1 parent ee43bbd
commit 20064fc
Show file tree

Hide file tree

Showing 19 changed files with 1,044 additions and 76 deletions.
diff --git a/fixtures/main_error_response.yaml b/fixtures/main_error_response.yaml
@@ -0,0 +1,30 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Content-Length:
+      - '0'
+    method: ''
+    uri: https://jsonplaceholder.typicode.com/
+  response:
+    body:
+      string: "<html>\r\n<head><title>400 Bad Request</title></head>\r\n<body>\r\n\
+        <center><h1>400 Bad Request</h1></center>\r\n<hr><center>cloudflare</center>\r\
+        \n</body>\r\n</html>\r\n"
+    headers:
+      CF-RAY:
+      - '-'
+      Connection:
+      - close
+      Content-Length:
+      - '155'
+      Content-Type:
+      - text/html
+      Date:
+      - Mon, 16 Dec 2019 22:11:39 GMT
+      Server:
+      - cloudflare
+    status:
+      code: 400
+      message: Bad Request
+version: 1
diff --git a/fixtures/main_input_body.yaml b/fixtures/main_input_body.yaml
@@ -0,0 +1,59 @@
+interactions:
+- request:
+    body: id=42&title=test
+    headers:
+      Content-Length:
+      - '16'
+      Content-Type:
+      - application/x-www-form-urlencoded
+    method: POST
+    uri: https://jsonplaceholder.typicode.com/posts
+  response:
+    body:
+      string: "{\n  \"id\": 101,\n  \"title\": \"test\"\n}"
+    headers:
+      Access-Control-Allow-Credentials:
+      - 'true'
+      Access-Control-Expose-Headers:
+      - Location
+      CF-Cache-Status:
+      - DYNAMIC
+      CF-RAY:
+      - 546ab6213f81f11a-IAD
+      Cache-Control:
+      - no-cache
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '34'
+      Content-Type:
+      - application/json; charset=utf-8
+      Date:
+      - Tue, 17 Dec 2019 17:45:33 GMT
+      Etag:
+      - W/"22-i04alCk7PdGrJ2UKCUwBOO0LB3w"
+      Expect-CT:
+      - max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"
+      Expires:
+      - '-1'
+      Location:
+      - http://jsonplaceholder.typicode.com/posts/101
+      Pragma:
+      - no-cache
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cfduid=d1fa880178dc2db917f26b8ce4e0d56e41576604733; expires=Thu, 16-Jan-20
+        17:45:33 GMT; path=/; domain=.typicode.com; HttpOnly; SameSite=Lax
+      Vary:
+      - Origin, X-HTTP-Method-Override, Accept-Encoding
+      Via:
+      - 1.1 vegur
+      X-Content-Type-Options:
+      - nosniff
+      X-Powered-By:
+      - Express
+    status:
+      code: 201
+      message: Created
+version: 1
diff --git a/fixtures/main_json_response.yaml b/fixtures/main_json_response.yaml
@@ -0,0 +1,92 @@
+interactions:
+- request:
+    body: null
+    headers: {}
+    method: GET
+    uri: https://jsonplaceholder.typicode.com/posts?userId=1
+  response:
+    body:
+      string: "[\n  {\n    \"userId\": 1,\n    \"id\": 1,\n    \"title\": \"sunt aut\
+        \ facere repellat provident occaecati excepturi optio reprehenderit\",\n \
+        \   \"body\": \"quia et suscipit\\nsuscipit recusandae consequuntur expedita\
+        \ et cum\\nreprehenderit molestiae ut ut quas totam\\nnostrum rerum est autem\
+        \ sunt rem eveniet architecto\"\n  },\n  {\n    \"userId\": 1,\n    \"id\"\
+        : 2,\n    \"title\": \"qui est esse\",\n    \"body\": \"est rerum tempore\
+        \ vitae\\nsequi sint nihil reprehenderit dolor beatae ea dolores neque\\nfugiat\
+        \ blanditiis voluptate porro vel nihil molestiae ut reiciendis\\nqui aperiam\
+        \ non debitis possimus qui neque nisi nulla\"\n  },\n  {\n    \"userId\":\
+        \ 1,\n    \"id\": 3,\n    \"title\": \"ea molestias quasi exercitationem repellat\
+        \ qui ipsa sit aut\",\n    \"body\": \"et iusto sed quo iure\\nvoluptatem\
+        \ occaecati omnis eligendi aut ad\\nvoluptatem doloribus vel accusantium quis\
+        \ pariatur\\nmolestiae porro eius odio et labore et velit aut\"\n  },\n  {\n\
+        \    \"userId\": 1,\n    \"id\": 4,\n    \"title\": \"eum et est occaecati\"\
+        ,\n    \"body\": \"ullam et saepe reiciendis voluptatem adipisci\\nsit amet\
+        \ autem assumenda provident rerum culpa\\nquis hic commodi nesciunt rem tenetur\
+        \ doloremque ipsam iure\\nquis sunt voluptatem rerum illo velit\"\n  },\n\
+        \  {\n    \"userId\": 1,\n    \"id\": 5,\n    \"title\": \"nesciunt quas odio\"\
+        ,\n    \"body\": \"repudiandae veniam quaerat sunt sed\\nalias aut fugiat\
+        \ sit autem sed est\\nvoluptatem omnis possimus esse voluptatibus quis\\nest\
+        \ aut tenetur dolor neque\"\n  },\n  {\n    \"userId\": 1,\n    \"id\": 6,\n\
+        \    \"title\": \"dolorem eum magni eos aperiam quia\",\n    \"body\": \"\
+        ut aspernatur corporis harum nihil quis provident sequi\\nmollitia nobis aliquid\
+        \ molestiae\\nperspiciatis et ea nemo ab reprehenderit accusantium quas\\\
+        nvoluptate dolores velit et doloremque molestiae\"\n  },\n  {\n    \"userId\"\
+        : 1,\n    \"id\": 7,\n    \"title\": \"magnam facilis autem\",\n    \"body\"\
+        : \"dolore placeat quibusdam ea quo vitae\\nmagni quis enim qui quis quo nemo\
+        \ aut saepe\\nquidem repellat excepturi ut quia\\nsunt ut sequi eos ea sed\
+        \ quas\"\n  },\n  {\n    \"userId\": 1,\n    \"id\": 8,\n    \"title\": \"\
+        dolorem dolore est ipsam\",\n    \"body\": \"dignissimos aperiam dolorem qui\
+        \ eum\\nfacilis quibusdam animi sint suscipit qui sint possimus cum\\nquaerat\
+        \ magni maiores excepturi\\nipsam ut commodi dolor voluptatum modi aut vitae\"\
+        \n  },\n  {\n    \"userId\": 1,\n    \"id\": 9,\n    \"title\": \"nesciunt\
+        \ iure omnis dolorem tempora et accusantium\",\n    \"body\": \"consectetur\
+        \ animi nesciunt iure dolore\\nenim quia ad\\nveniam autem ut quam aut nobis\\\
+        net est aut quod aut provident voluptas autem voluptas\"\n  },\n  {\n    \"\
+        userId\": 1,\n    \"id\": 10,\n    \"title\": \"optio molestias id quia eum\"\
+        ,\n    \"body\": \"quo et expedita modi cum officia vel magni\\ndoloribus\
+        \ qui repudiandae\\nvero nisi sit\\nquos veniam quod sed accusamus veritatis\
+        \ error\"\n  }\n]"
+    headers:
+      Access-Control-Allow-Credentials:
+      - 'true'
+      Age:
+      - '4646'
+      CF-Cache-Status:
+      - HIT
+      CF-RAY:
+      - 54640259fa6fe0d2-IAD
+      Cache-Control:
+      - max-age=14400
+      Connection:
+      - keep-alive
+      Content-Type:
+      - application/json; charset=utf-8
+      Date:
+      - Mon, 16 Dec 2019 22:14:15 GMT
+      Etag:
+      - W/"aa6-j2NSH739l9uq40OywFMn7Y0C/iY"
+      Expect-CT:
+      - max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"
+      Expires:
+      - '-1'
+      Pragma:
+      - no-cache
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cfduid=d8ac1a396d31b9e9c1f816924e5bf186d1576534455; expires=Wed, 15-Jan-20
+        22:14:15 GMT; path=/; domain=.typicode.com; HttpOnly
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Origin, Accept-Encoding
+      Via:
+      - 1.1 vegur
+      X-Content-Type-Options:
+      - nosniff
+      X-Powered-By:
+      - Express
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/fixtures/main_non_json_response.yaml b/fixtures/main_non_json_response.yaml
@@ -0,0 +1,145 @@
+interactions:
+- request:
+    body: null
+    headers: {}
+    method: GET
+    uri: https://jsonplaceholder.typicode.com/
+  response:
+    body:
+      string: "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\"\
+        \ />\n<meta name=\"viewport\" content=\"width=device-width, initial-scale=1,\
+        \ shrink-to-fit=no\" />\n<link rel=\"stylesheet\" href=\"https://cdnjs.cloudflare.com/ajax/libs/modern-normalize/0.5.0/modern-normalize.min.css\"\
+        \ />\n<link rel=\"stylesheet\" href=\"./style.css\" />\n<link rel=\"stylesheet\"\
+        \ href=\"//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css\"\
+        \ />\n<script src=\"//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js\"\
+        ></script>\n<link rel=\"stylesheet\" href=\"https://use.fontawesome.com/releases/v5.5.0/css/all.css\"\
+        \ integrity=\"sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU\"\
+        \ crossorigin=\"anonymous\" />\n<script>\n      hljs.initHighlightingOnLoad();\n\
+        \    </script>\n<title>JSONPlaceholder - Fake online REST API for developers</title>\n\
+        </head>\n<body>\n<header>\n<nav class=\"container\">\n<ul>\n<li><a href=\"\
+        /\">JSONPlaceholder</a></li>\n<li><a href=\"/guide.html\">Guide</a></li>\n\
+        <li><a href=\"https://github.com/typicode\">GitHub</a></li>\n<li>\n<a href=\"\
+        https://my-json-server.typicode.com\">My JSON Server</a>\n</li>\n</ul>\n</nav>\n\
+        <a href=\"https://github.com/users/typicode/sponsorship\" class=\"announcement\"\
+        >\n<i class=\"fab fa-github-alt\"></i> Announcement: You can now support\n\
+        JSONPlaceholder on GitHub Sponsors!\n</a>\n</header>\n<div class=\"container\"\
+        >\n\n<div class=\"hero\">\n<div class=\"container\">\n<h1>\nJSONPlaceholder\n\
+        </h1>\n<p class=\"subtitle\">\nFake Online REST API for Testing and Prototyping\n\
+        <br> <b>Serving ~350M requests per month</b>\n<br> Powered by\n<a href=\"\
+        https://github.com/typicode/json-server\">JSON Server</a>\n+\n<a href=\"https://github.com/typicode/lowdb\"\
+        >LowDB</a>\n</p>\n\n\n\n\n\n</div>\n</div>\n\n<div class=\"sponsors\">\n<h3>Gold\
+        \ Sponsors</h3>\n<p>\n<a href=\"https://tryretool.com/?utm_source=sponsor&utm_campaign=typicode\"\
+        \ target=\"_blank\">\n<img src=\"https://i.imgur.com/IBItATn.png\" height=\"\
+        70px\">\n</a>\n</p>\n<p>\n<a href=\"https://www.patreon.com/bePatron?c=784328\"\
+        >Your Company Logo Here</a>\n</p>\n</div>\n\n<div class=\"container\">\n<main>\n\
+        \n<h2>Intro</h2>\n<p>\nJSONPlaceholder is a free online REST API that you\
+        \ can use whenever you need some fake data.\n<br>It's great for tutorials,\
+        \ testing new libraries, sharing code examples, ...\n</p>\n\n<h2>Example</h2>\n\
+        <p>\nRun this code in a console or from any site:\n</p>\n<pre><code id=\"\
+        example\" class=\"javascript\">fetch('https://jsonplaceholder.typicode.com/todos/1')\n\
+        \  .then(response => response.json())\n  .then(json => console.log(json))\n\
+        </code></pre>\n<p>\n<button id=\"run-button\">Try it</button>\n</p>\n<pre><code\
+        \ id=\"result\" class=\"json\"></code></pre>\n<span id=\"run-message\">Congrats\
+        \ you've made your first call to JSONPlaceholder! \U0001F603 \U0001F389</span>\n\
+        <p>\n<strong>Tip</strong>: you can use\n<b>\nhttp://\n</b> or\n<b>\nhttps://\n\
+        </b> when making requests to JSONPlaceholder.\n</p>\n<div id=\"codefund\"\
+        ></div>\n<script src=\"https://codefund.io/properties/338/funder.js\" async=\"\
+        async\"></script>\n\n<h2>Resources</h2>\n<p>\nJSONPlaceholder comes with a\
+        \ set of 6 common resources:\n</p>\n<table class=\"resources\">\n<tr>\n<td>\n\
+        \ <a href=\"/posts\">/posts</a>\n</td>\n<td>100 posts</td>\n</tr>\n<tr>\n\
+        <td>\n<a href=\"/comments\">/comments</a>\n</td>\n<td>500 comments</td>\n\
+        </tr>\n<tr>\n<td>\n<a href=\"/albums\">/albums</a>\n</td>\n<td>100 albums</td>\n\
+        </tr>\n<tr>\n<td>\n<a href=\"/photos\">/photos</a>\n</td>\n<td>5000 photos</td>\n\
+        </tr>\n<tr>\n<td>\n<a href=\"/todos\">/todos</a>\n</td>\n<td>200 todos</td>\n\
+        </tr>\n<tr>\n<td>\n<a href=\"/users\">/users</a>\n</td>\n<td>10 users</td>\n\
+        </tr>\n</table>\n<p>\n<strong>Note</strong>: resources have relations. For\
+        \ example:\n<b>posts</b> have many\n<b>comments</b>,\n<b>albums</b> have many\n\
+        <b>photos</b>, ... see below for routes examples.\n</p>\n\n<h2>Routes</h2>\n\
+        <p>\nAll HTTP methods are supported.\n</p>\n<table>\n<tr>\n<td>GET</td>\n\
+        <td>\n<a href=\"/posts\">/posts</a>\n</td>\n</tr>\n<tr>\n<td>GET</td>\n<td>\n\
+        <a href=\"/posts/1\">/posts/1</a>\n</td>\n</tr>\n<tr>\n<td>GET</td>\n<td>\n\
+        <a href=\"/posts/1/comments\">/posts/1/comments</a>\n</td>\n</tr>\n<tr>\n\
+        <td>GET</td>\n<td>\n<a href=\"/comments?postId=1\">/comments?postId=1</a>\n\
+        </td>\n</tr>\n<tr>\n<td>GET</td>\n<td>\n<a href=\"/posts?userId=1\">/posts?userId=1</a>\n\
+        </td>\n</tr>\n<tr>\n<td>POST</td>\n<td>/posts</td>\n</tr>\n<tr>\n<td>PUT</td>\n\
+        <td>/posts/1</td>\n</tr>\n<tr>\n<td>PATCH</td>\n<td>/posts/1</td>\n</tr>\n\
+        <tr>\n<td>DELETE</td>\n<td>/posts/1</td>\n</tr>\n</tr>\n</table>\n<p>\n<strong>Note</strong>:\
+        \ you can view detailed examples\n<a href=\"guide.html\">here</a>.\n</p>\n\
+        \n<h2>Use your own data</h2>\n\n<p>\nWith <a href=\"https://my-json-server.typicode.com\"\
+        >My JSON Server</a> online service and a simple GitHub repo, you can have\
+        \ your own online fake REST server in seconds.\n</p>\n</main>\n</div>\n</div>\n\
+        \n<footer>\n<div class=\"container\">\n<div style=\"margin-bottom: 2rem\"\
+        >\n\n\n\n\n\n\n<a href=\"https://github.com/users/typicode/sponsorship\">\n\
+        <i class=\"fab fa-github-alt\"></i> Sponsor this project on GitHub\n</a>\n\
+        </div>\n<div>\nCoded and built with \u2764\uFE0F by\n<a href=\"https://github.com/typicode\"\
+        >typicode</a>\n<br />Source code and CHANGELOG available on\n<a href=\"https://github.com/typicode/jsonplaceholder\"\
+        >GitHub</a>\n</div>\n</div>\n</footer>\n\n<script>\n      (function(i, s,\
+        \ o, g, r, a, m) {\n        i[\"GoogleAnalyticsObject\"] = r;\n        (i[r]\
+        \ =\n          i[r] ||\n          function() {\n            (i[r].q = i[r].q\
+        \ || []).push(arguments);\n          }),\n          (i[r].l = 1 * new Date());\n\
+        \        (a = s.createElement(o)), (m = s.getElementsByTagName(o)[0]);\n \
+        \       a.async = 1;\n        a.src = g;\n        m.parentNode.insertBefore(a,\
+        \ m);\n      })(\n        window,\n        document,\n        \"script\",\n\
+        \        \"//www.google-analytics.com/analytics.js\",\n        \"ga\"\n  \
+        \    );\n      ga(\"create\", \"UA-44497010-1\", \"typicode.com\");\n    \
+        \  ga(\"send\", \"pageview\");\n      var trackOutboundLink = function(url)\
+        \ {\n        ga(\"send\", \"event\", \"outbound\", \"click\", url, {\n   \
+        \       transport: \"beacon\"\n        });\n      };\n    </script>\n<script\
+        \ src=\"//cdnjs.cloudflare.com/ajax/libs/fetch/2.0.3/fetch.min.js\"></script>\n\
+        <script>\n      // Use http or https based on location.protocol\n      var\
+        \ example = document.getElementById(\"example\");\n      example.textContent\
+        \ = example.textContent.replace(\n        \"http:\",\n        location.protocol\n\
+        \      );\n\n      // Highlight result element\n      var result = document.getElementById(\"\
+        result\");\n      hljs.highlightBlock(result);\n\n      // Run example\n \
+        \     var runButton = document.getElementById(\"run-button\");\n      runButton.onclick\
+        \ = function() {\n        var root = location.protocol + \"//jsonplaceholder.typicode.com\"\
+        ;\n        var runMessage = document.getElementById(\"run-message\");\n\n\
+        \        // Hide or disable things during API call\n        runButton.disabled\
+        \ = true;\n        runMessage.style.opacity = 0;\n        result.style.opacity\
+        \ = 0;\n\n        fetch(\"https://jsonplaceholder.typicode.com/todos/1\")\n\
+        \          .then(response => response.json())\n          .then(json => {\n\
+        \            var str = JSON.stringify(json, null, \"\\t\");\n\n          \
+        \  // Format result\n            result.innerHTML = str\n              .replace(/\\\
+        n/g, \"<br/>\")\n              .replace(/\\\\n/g, \" \")\n              .replace(/\\\
+        t/g, \"&nbsp;&nbsp;\");\n\n            hljs.highlightBlock(result);\n\n  \
+        \          // Show and enable things after API call\n            runButton.disabled\
+        \ = false;\n            runMessage.style.opacity = 1;\n            result.style.opacity\
+        \ = 1;\n          });\n      };\n    </script>\n</body>\n</html>\n"
+    headers:
+      Access-Control-Allow-Credentials:
+      - 'true'
+      Age:
+      - '5668'
+      CF-Cache-Status:
+      - HIT
+      CF-RAY:
+      - 5463fe6ccb0ecf04-IAD
+      Cache-Control:
+      - public, max-age=14400
+      Connection:
+      - keep-alive
+      Content-Type:
+      - text/html; charset=UTF-8
+      Date:
+      - Mon, 16 Dec 2019 22:11:34 GMT
+      Expect-CT:
+      - max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"
+      Last-Modified:
+      - Mon, 05 Aug 2019 03:07:14 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cfduid=d13b55a7f6e786f74d5f9f1f084a183a31576534294; expires=Wed, 15-Jan-20
+        22:11:34 GMT; path=/; domain=.typicode.com; HttpOnly
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Origin, Accept-Encoding
+      Via:
+      - 1.1 vegur
+      X-Powered-By:
+      - Express
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/fixtures/proxy_bad_request.yaml b/fixtures/proxy_bad_request.yaml
@@ -0,0 +1,20 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - application/json
+    method: GET
+    uri: http://localhost:8000/bad
+  response:
+    body:
+      string: ''
+    headers:
+      Date:
+      - Mon, 16 Dec 2019 23:44:53 GMT
+      Server:
+      - BaseHTTP/0.6 Python/3.7.3
+    status:
+      code: 400
+      message: Bad Request
+version: 1