diff --git a/data-services/html-source/README.md b/data-services/html-source/README.md index fc57704..b2f1f4a 100644 --- a/data-services/html-source/README.md +++ b/data-services/html-source/README.md @@ -2,9 +2,9 @@ Uses simple web scrapping technique with Cherrio using the mimi.pnca.edu web site (for now) -## JavaScript and a TypeScript examples +## JavaScript examples -The two examples are found in mimi-server and mimi-server__typescript. See the READMEs in those directories for instructions on installing and running, and example data. +The two examples are found in mimi-server and mimi-server__structured. See the READMEs in those directories for instructions on installing and running, and example data. ## Example HTML diff --git a/data-services/html-source/mimi-server--structured/.babelrc b/data-services/html-source/mimi-server--structured/.babelrc new file mode 100644 index 0000000..2f01e1d --- /dev/null +++ b/data-services/html-source/mimi-server--structured/.babelrc @@ -0,0 +1,3 @@ +{ + "presets": ["env"] +} \ No newline at end of file diff --git a/data-services/html-source/mimi-server--structured/README.md b/data-services/html-source/mimi-server--structured/README.md index fe5a2c6..6f8b2b2 100644 --- a/data-services/html-source/mimi-server--structured/README.md +++ b/data-services/html-source/mimi-server--structured/README.md @@ -1,4 +1,12 @@ -# This simple example performs the same functionality as the example found in ../mimi-server, but uses graphql-tools from Apollo +# This simple example is the same as ../mimi-server but structured + +This is a fairly standard structure, with the schema and resolvers in to their own files. + +## Other changes from ../mimi-server + +### import and export instead of require + +[esm](https://www.npmjs.com/package/esm) is used to provide support for standard JavaScript import and export, which are not supported natively in node yet (as of 13.9). The `--experimental-support` flag could also be used if one does not want to use esm. + -See https://www.apollographql.com/docs/graphql-tools/ for documentation diff --git a/data-services/html-source/mimi-server--structured/collections.js b/data-services/html-source/mimi-server--structured/collections.js new file mode 100644 index 0000000..4f7b04f --- /dev/null +++ b/data-services/html-source/mimi-server--structured/collections.js @@ -0,0 +1,30 @@ +const fetch = require('node-fetch'); +import cheerio from 'cheerio'; + + +const getCollections = async(kind) => { + return fetch(`https://mimi.pnca.edu/f/${kind}`, {"credentials":"exclude","headers":{"accept":"text/html,application/xhtml+xml","accept-language":"en-US","cache-control":"no-cache","pragma":"no-cache"},"method":"GET"}) + .then((response) => { + return response.text() + }).then((html) => { + const collections = [] + const $ = cheerio.load(html) + $('a[data-no-turbolink="false"] h3').each(function (i, e) { + const collectionsTitle = $(this).text(); + if (collectionsTitle !== 'Collections') { + const item = {} + + //TODO a bit of a hack parsing the html, which is prone to failure, add some checks on elements + const style = $(this).parent().attr('style') + const backgroundImage = style.indexOf("''") === -1 ? + (style.split('background-image:url(\'')[1]).split('?')[0]: '' + item['href'] = $(this).parent().parent().attr('href') + item['backgroundImage'] = backgroundImage + item['title'] = collectionsTitle.trim() + collections.push(item); + } + }); + return collections + }) + } +export { getCollections } \ No newline at end of file diff --git a/data-services/html-source/mimi-server--structured/index.js b/data-services/html-source/mimi-server--structured/index.js index 95b5a7e..6862048 100644 --- a/data-services/html-source/mimi-server--structured/index.js +++ b/data-services/html-source/mimi-server--structured/index.js @@ -1,5 +1,9 @@ const { ApolloServer, gql } = require('apollo-server'); -const fetch = require('node-fetch'); -const cheerio = require('cheerio'); -const { resolverMap } = require('./resolvers'); -// const schema = require('schema'); +import resolverMap from './resolvers'; +import typeDefs from './schema'; + +const server = new ApolloServer({ typeDefs, resolvers: resolverMap }); + +server.listen().then(({ url }) => { + console.log(`🚀 Server ready at ${url}`); +}); \ No newline at end of file diff --git a/data-services/html-source/mimi-server--structured/package-lock.json b/data-services/html-source/mimi-server--structured/package-lock.json index 47104ea..2e28291 100644 --- a/data-services/html-source/mimi-server--structured/package-lock.json +++ b/data-services/html-source/mimi-server--structured/package-lock.json @@ -1016,6 +1016,11 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, + "esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" + }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", diff --git a/data-services/html-source/mimi-server--structured/package.json b/data-services/html-source/mimi-server--structured/package.json index 3e8d17a..0623e31 100644 --- a/data-services/html-source/mimi-server--structured/package.json +++ b/data-services/html-source/mimi-server--structured/package.json @@ -4,6 +4,7 @@ "description": "", "main": "index.js", "scripts": { + "start": "nodemon -r esm index.js", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], @@ -12,6 +13,7 @@ "dependencies": { "apollo-server": "^2.11.0", "cheerio": "^1.0.0-rc.3", + "esm": "^3.2.25", "graphql": "^14.6.0", "node-fetch": "^2.6.0" }, diff --git a/data-services/html-source/mimi-server--structured/resolvers.js b/data-services/html-source/mimi-server--structured/resolvers.js index 6120853..34cc191 100644 --- a/data-services/html-source/mimi-server--structured/resolvers.js +++ b/data-services/html-source/mimi-server--structured/resolvers.js @@ -1,3 +1,4 @@ +import { getCollections } from './collections' const resolverMap = { Query: { collections: async(_, args) => {