diff --git a/app/actions/sessions.js b/app/actions/sessions.js index a4211b0a6..7a7eeaf3f 100644 --- a/app/actions/sessions.js +++ b/app/actions/sessions.js @@ -361,6 +361,7 @@ function PREVIEW_QUERY (dialect, table, database = '') { switch (dialect) { case DIALECTS.IBM_DB2: return `SELECT * FROM ${table} FETCH FIRST 1000 ROWS ONLY`; + case DIALECTS.APACHE_IMPALA: case DIALECTS.APACHE_SPARK: case DIALECTS.MYSQL: case DIALECTS.SQLITE: diff --git a/app/components/Settings/DialectSelector/DialectSelector.react.js b/app/components/Settings/DialectSelector/DialectSelector.react.js index 4fff79eb5..5f7eb9eea 100644 --- a/app/components/Settings/DialectSelector/DialectSelector.react.js +++ b/app/components/Settings/DialectSelector/DialectSelector.react.js @@ -11,6 +11,7 @@ export default function DialectSelector(props) { const logos = values(DIALECTS).map(DIALECT => (
+ { + Logger.log(err); + throw new Error(err); + }); +} + +export function tables(connection) { + const code = (connection.database) ? + `show tables in ${connection.database}` : + 'show tables'; + return createClient(connection).query(code) + .then(json => { + let tableNames = json.map(t => t.name); + if (connection.database) tableNames = tableNames.map(tn => `${connection.database}.${tn}`); + tableNames = tableNames.map(tn => tn.toUpperCase()); + + return tableNames; + }).catch(err => { + Logger.log(err); + throw new Error(err); + }); +} + +export function schemas(connection) { + let columnnames = ['tablename', 'column_name', 'data_type']; + const showTables = (connection.database) ? + `show tables in ${connection.database}` : + 'show tables'; + + return createClient(connection).query(showTables) + .then(json => { + let tableNames = json.map(t => t.name); + if (connection.database) tableNames = tableNames.map(tn => `${connection.database}.${tn}`); + + /* + * The last column in the output of describe statement is 'comment', + * so we remove it(using Ramda.init) before sending out the result. + */ + const promises = map(tableName => { + return query(`describe ${tableName}`, connection) + .then(json => map(row => prepend(tableName, init(row)), json.rows)); + }, tableNames); + + // Wait for all the describe-table promises to resolve before resolving: + return Promise.all(promises); + }).then(res => { + + // The results are nested inside a list, so we need to un-nest first: + const rows = unnest(res); + return {columnnames, rows}; + }).catch(err => { + Logger.log(err); + throw new Error(err); + }); +} + +export function query(query, connection) { + + return createClient(connection).query(query) + .then(json => { + let columnnames = []; + let rows = [[]]; + if (json.length !== 0) { + columnnames = keys(json[0]); + rows = json.map(obj => values(obj)); + } + return {columnnames, rows}; + }).catch(err => { + Logger.log(err); + throw new Error(err) + }); +} diff --git a/package.json b/package.json index 907d366f8..807e173f8 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test-unit-certificates": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --timeout 20000 --compilers js:babel-register test/backend/certificates.spec.js", "test-unit-ibmdb": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --timeout 20000 --compilers js:babel-register test/backend/datastores.ibmdb.spec.js", "test-unit-livy": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --timeout 20000 --compilers js:babel-register test/backend/datastores.livy.spec.js", + "test-unit-impala": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --timeout 20000 --compilers js:babel-register test/backend/datastores.impala.spec.js", "test-unit-routes": "cross-env NODE_ENV=test BABEL_DISABLE_CACHE=1 electron-mocha --timeout 20000 --compilers js:babel-register test/backend/routes.spec.js", "package": "cross-env NODE_ENV=production node -r babel-register package.js", "package-all": "yarn run package -- --all", @@ -194,6 +195,7 @@ "mysql": "^2.10.2", "node-fetch": "^1.7.2", "node-gyp": "^3.3.1", + "node-impala": "^2.0.4", "node-libs-browser": "^1.0.0", "node-restify": "^0.2.1", "pg": "^4.5.5", @@ -217,7 +219,7 @@ "react-select": "^1.0.0-beta13", "react-split-pane": "^0.1.66", "react-tabs": "^1.1.0", - "react-tooltip": "^3.1.7", + "react-tooltip": "^3.4.0", "react-treeview": "^0.4.7", "redux": "^3.4.0", "redux-actions": "^0.9.1", diff --git a/sample-storage/connections.yaml b/sample-storage/connections.yaml index 9e834a471..5d95aa8bb 100644 --- a/sample-storage/connections.yaml +++ b/sample-storage/connections.yaml @@ -86,3 +86,11 @@ database: plotly port: 8998 host: 104.154.141.189 + +- + dialect: apache impala + id: apache impala-159e0b47-0428-4c9e-b4b9-8201b86f8ca2 + timeout: 180 + database: plotly + port: 21000 + host: 35.184.155.127 diff --git a/test/backend/datastores.impala.spec.js b/test/backend/datastores.impala.spec.js new file mode 100644 index 000000000..0d81ce3b4 --- /dev/null +++ b/test/backend/datastores.impala.spec.js @@ -0,0 +1,55 @@ +import {assert} from 'chai'; + +import {DIALECTS} from '../../app/constants/constants.js'; +import {apacheImpalaConnection as connection} from './utils.js'; +import { + query, connect, tables +} from '../../backend/persistent/datastores/Datastores.js'; + +describe('Apache Impala:', function () { + + it('connect succeeds', function() { + this.timeout(180 * 1000); + return connect(connection); + }); + + it('tables returns list of tables', function() { + return tables(connection).then(result => { + const tableName = (connection.database) ? + `${connection.database}.ALCOHOL_CONSUMPTION_BY_COUNTRY_2010`.toUpperCase() : + 'ALCOHOL_CONSUMPTION_BY_COUNTRY_2010'; + + assert.deepEqual(result, [tableName]); + }); + }); + + it('query returns rows and column names', function() { + const tableName = (connection.database) ? + `${connection.database}.ALCOHOL_CONSUMPTION_BY_COUNTRY_2010`.toUpperCase() : + 'ALCOHOL_CONSUMPTION_BY_COUNTRY_2010'; + + return query(`SELECT * FROM ${tableName}\nLIMIT 5`, connection).then(results => { + assert.deepEqual(results.rows, [ + ['Belarus', "17.5"], + ['Moldova', "16.8"], + ['Lithuania', "15.4"], + ['Russia', "15.1"], + ['Romania', "14.4"] + ]); + assert.deepEqual(results.columnnames, ['loc', 'alcohol']); + }); + }); + + it('connect for invalid credentials fails', function() { + connection.host = 'http://lah-lah.lemons.com'; + + return connect(connection).catch(err => { + // reset hostname + connection.host = '35.184.155.127'; + + assert.equal(err, ('Error: Error: getaddrinfo ENOTFOUND ' + + 'http://lah-lah.lemons.com ' + + 'http://lah-lah.lemons.com:21000')); + }); + }); +}); diff --git a/test/backend/routes.spec.js b/test/backend/routes.spec.js index e6f9a5770..6fa5bb0c7 100644 --- a/test/backend/routes.spec.js +++ b/test/backend/routes.spec.js @@ -559,6 +559,9 @@ describe('Routes - ', () => { `${connection.database}.dbo.ebola_2014` ); } + if (connection.dialect === 'apache impala') { + sampleQuery = 'SELECT * FROM PLOTLY.ALCOHOL_CONSUMPTION_BY_COUNTRY_2010 LIMIT 1'; + } POST(`connections/${connectionId}/query`, { query: sampleQuery }) @@ -569,6 +572,8 @@ describe('Routes - ', () => { expectedColumnNames = ['Country', 'Month', 'Year', 'Lat', 'Lon', 'Value']; } else if (connection.dialect === 'sqlite') { expectedColumnNames = ['index', 'Country', 'Month', 'Year', 'Lat', 'Lon', 'Value']; + } else if (connection.dialect === 'apache impala') { + expectedColumnNames = ['loc', 'alcohol']; } else { expectedColumnNames = ['country', 'month', 'year', 'lat', 'lon', 'value']; } @@ -583,7 +588,8 @@ describe('Routes - ', () => { 'mysql': ['Guinea', 3, 14, 10, -10, '122'], 'mariadb': ['Guinea', 3, 14, 10, -10, '122'], 'mssql': ['Guinea', 3, 14, 10, -10, '122'], - 'sqlite': [0, 'Guinea', 3, 14, 9.95, -9.7, 122] + 'sqlite': [0, 'Guinea', 3, 14, 9.95, -9.7, 122], + 'apache impala': ['Belarus', '17.5'] })[connection.dialect] ], columnnames: expectedColumnNames @@ -615,7 +621,8 @@ describe('Routes - ', () => { 'corresponds to your MariaDB server version for the right syntax to use near ' + '\'SELECZ\' at line 1', mssql: "Could not find stored procedure 'SELECZ'.", - sqlite: 'SQLITE_ERROR: near "SELECZ": syntax error' + sqlite: 'SQLITE_ERROR: near "SELECZ": syntax error', + 'apache impala': 'BeeswaxException: AnalysisException: Syntax error in line 1:\nSELECZ\n^\nEncountered: IDENTIFIER\nExpected: ALTER, COMPUTE, CREATE, DESCRIBE, DROP, EXPLAIN, INSERT, INVALIDATE, LOAD, REFRESH, SELECT, SHOW, USE, VALUES, WITH\n\nCAUSED BY: Exception: Syntax error' })[connection.dialect] }} ); @@ -634,6 +641,10 @@ describe('Routes - ', () => { `${connection.database}.dbo.ebola_2014` ); } + if (connection.dialect === 'apache impala') { + query = 'SELECT * FROM PLOTLY.ALCOHOL_CONSUMPTION_BY_COUNTRY_2010 LIMIT 0'; + } + POST(`connections/${connectionId}/query`, {query}) .then(res => res.json().then(json => { assert.equal(res.status, 200); @@ -679,6 +690,9 @@ describe('Routes - ', () => { 'spatial_ref_sys' ]).sort(); } + if (connection.dialect === 'apache impala') { + tables = ['PLOTLY.ALCOHOL_CONSUMPTION_BY_COUNTRY_2010']; + } assert.deepEqual( json, tables ); @@ -906,6 +920,11 @@ describe('Routes - ', () => { [ 'apple_stock_2014', 'AAPL_x', 'datetime', 8, '23/3' ], [ 'apple_stock_2014', 'AAPL_y', 'decimal', 17, '38/38' ] ]; + } else if (connection.dialect === 'apache impala') { + rows = [ + [ 'plotly.alcohol_consumption_by_country_2010', 'loc', 'string' ], + [ 'plotly.alcohol_consumption_by_country_2010', 'alcohol', 'double' ], + ]; } else { rows = [ [ 'alcohol_consumption_by_country_2010', 'location', 'varchar' ], @@ -1200,9 +1219,7 @@ describe('Routes - ', () => { connectionTypo = merge(connection, {username: 'typo'}); } else if (connection.dialect === 's3') { connectionTypo = merge(connection, {secretAccessKey: 'typo'}); - } else if (connection.dialect === 'elasticsearch') { - connectionTypo = merge(connection, {host: 'https://lahlahlemons.com'}); - } else if (connection.dialect === 'apache drill') { + } else if (contains(connection.dialect, ['elasticsearch', 'apache drill', 'apache impala'])) { connectionTypo = merge(connection, {host: 'https://lahlahlemons.com'}); } else if (connection.dialect === 'sqlite') { connectionTypo = merge(connection, {storage: 'typo'}); @@ -1230,7 +1247,8 @@ describe('Routes - ', () => { 'failed, reason: getaddrinfo ENOTFOUND lahlahlemons.com lahlahlemons.com:9243', ['apache drill']: 'request to https://lahlahlemons.com:8047/query.json failed, ' + 'reason: getaddrinfo ENOTFOUND lahlahlemons.com lahlahlemons.com:8047', - sqlite: 'SQLite file at path "typo" does not exist.' + sqlite: 'SQLite file at path "typo" does not exist.', + ['apache impala']: 'Error: getaddrinfo ENOTFOUND https://lahlahlemons.com https://lahlahlemons.com:21000' })[connection.dialect] } }); diff --git a/test/backend/utils.js b/test/backend/utils.js index 21b501953..e01c3e149 100644 --- a/test/backend/utils.js +++ b/test/backend/utils.js @@ -131,6 +131,12 @@ export const sqliteConnection = { dialect: 'sqlite', storage: `${__dirname}/plotly_datasets.db` }; +export const apacheImpalaConnection = { + dialect: 'apache impala', + host: '35.184.155.127', + port: 21000, + database: 'plotly' +}; // TODO - Add sqlite here // TODO - Add postgis in here @@ -144,7 +150,8 @@ export const testConnections = [ sqliteConnection, elasticsearchConnections, publicReadableS3Connections, - apacheDrillConnections + apacheDrillConnections, + apacheImpalaConnection ]; export const testSqlConnections = [ @@ -152,7 +159,8 @@ export const testSqlConnections = [ mysqlConnection, mariadbConnection, redshiftConnection, - mssqlConnection + mssqlConnection, + apacheImpalaConnection ]; export const configuration = dissoc('password', sqlConnections); diff --git a/yarn.lock b/yarn.lock index 657508288..17fd0353f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6331,6 +6331,10 @@ nan@^2.3.0, nan@^2.3.3, nan@~2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46" +nan@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-1.0.0.tgz#ae24f8850818d662fcab5acf7f3b95bfaa2ccf38" + nan@~2.3.5: version "2.3.5" resolved "https://registry.yarnpkg.com/nan/-/nan-2.3.5.tgz#822a0dc266290ce4cd3a12282ca3e7e364668a08" @@ -6459,6 +6463,16 @@ node-gyp@^3.3.1, node-gyp@^3.6.0: tar "^2.0.0" which "1" +node-impala@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/node-impala/-/node-impala-2.0.4.tgz#57e0eb89c3a3925f8c37d8dfd7135d5ad90bf9af" + dependencies: + thrift "^0.10.0" + +node-int64@~0.3.0: + version "0.3.3" + resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.3.3.tgz#2d6e6b2ece5de8588b43d88d1bc41b26cd1fa84d" + node-libs-browser@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/node-libs-browser/-/node-libs-browser-1.1.1.tgz#2a38243abedd7dffcd07a97c9aca5668975a6fea" @@ -7459,7 +7473,7 @@ promise@^7.1.1: dependencies: asap "~2.0.3" -prop-types@^15.0.0, prop-types@^15.5.0, prop-types@^15.5.10, prop-types@^15.5.4, prop-types@^15.5.7, prop-types@^15.5.8: +prop-types@^15.0.0, prop-types@^15.5.0, prop-types@^15.5.10, prop-types@^15.5.4, prop-types@^15.5.7, prop-types@^15.5.8, prop-types@^15.6.0: version "15.6.0" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.6.0.tgz#ceaf083022fc46b4a35f69e13ef75aed0d639856" dependencies: @@ -7515,6 +7529,10 @@ pure-color@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/pure-color/-/pure-color-1.3.0.tgz#1fe064fb0ac851f0de61320a8bf796836422f33e" +q@1.0.x: + version "1.0.1" + resolved "https://registry.yarnpkg.com/q/-/q-1.0.1.tgz#11872aeedee89268110b10a718448ffb10112a14" + q@^1.1.2: version "1.5.0" resolved "https://registry.yarnpkg.com/q/-/q-1.5.0.tgz#dd01bac9d06d30e6f219aecb8253ee9ebdc308f1" @@ -7800,12 +7818,12 @@ react-tabs@^1.1.0: classnames "^2.2.0" prop-types "^15.5.0" -react-tooltip@^3.1.7: - version "3.3.0" - resolved "https://registry.yarnpkg.com/react-tooltip/-/react-tooltip-3.3.0.tgz#51c08ae0221075e2c43d83cd47fc78466612df7d" +react-tooltip@^3.4.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/react-tooltip/-/react-tooltip-3.4.0.tgz#037f38f797c3e6b1b58d2534ccc8c2c76af4f52d" dependencies: - classnames "^2.2.0" - prop-types "^15.5.8" + classnames "^2.2.5" + prop-types "^15.6.0" react-transform-catch-errors@^1.0.2: version "1.0.2" @@ -9322,6 +9340,14 @@ text-table@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" +thrift@^0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/thrift/-/thrift-0.10.0.tgz#339af65921677b30560aa51d6f7ab1b8091c9376" + dependencies: + node-int64 "~0.3.0" + q "1.0.x" + ws "~0.4.32" + throttleit@0.0.2: version "0.0.2" resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-0.0.2.tgz#cfedf88e60c00dd9697b61fdd2a8343a9b680eaf" @@ -9390,6 +9416,10 @@ tinycolor2@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/tinycolor2/-/tinycolor2-1.4.1.tgz#f4fad333447bc0b07d4dc8e9209d8f39a8ac77e8" +tinycolor@0.x: + version "0.0.1" + resolved "https://registry.yarnpkg.com/tinycolor/-/tinycolor-0.0.1.tgz#320b5a52d83abb5978d81a3e887d4aefb15a6164" + tmp@0.0.24: version "0.0.24" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.24.tgz#d6a5e198d14a9835cc6f2d7c3d9e302428c8cf12" @@ -10034,6 +10064,15 @@ ws@^1.0.1: options ">=0.0.5" ultron "1.0.x" +ws@~0.4.32: + version "0.4.32" + resolved "https://registry.yarnpkg.com/ws/-/ws-0.4.32.tgz#787a6154414f3c99ed83c5772153b20feb0cec32" + dependencies: + commander "~2.1.0" + nan "~1.0.0" + options ">=0.0.5" + tinycolor "0.x" + xdg-basedir@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4"