Skip to content

Commit

Permalink
fix(PostgreSQL): wrong values exporting table content
Browse files Browse the repository at this point in the history
  • Loading branch information
@Fabio286
Fabio286 committed Apr 2, 2022
1 parent 638a88a commit 0f9c991
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 102 deletions.
1 change: 1 addition & 0 deletions src/common/customizations/defaults.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ module.exports = {
schemaEdit: false,
schemaDrop: false,
schemaExport: false,
exportByChunks: false,
schemaImport: false,
tableSettings: false,
tableOptions: false,
Expand Down
1 change: 1 addition & 0 deletions src/common/customizations/mysql.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ module.exports = {
schemaEdit: true,
schemaDrop: true,
schemaExport: true,
exportByChunks: true,
schemaImport: true,
tableSettings: true,
viewSettings: true,
Expand Down
144 changes: 63 additions & 81 deletions src/main/libs/exporters/sql/PostgreSQLExporter.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { SqlExporter } from './SqlExporter';
import { BLOB, BIT, DATE, DATETIME, FLOAT, SPATIAL, IS_MULTI_SPATIAL, NUMBER } from 'common/fieldTypes';
import { BLOB, BIT, DATE, DATETIME, FLOAT, NUMBER, TEXT_SEARCH } from 'common/fieldTypes';
import hexToBinary from 'common/libs/hexToBinary';
import { getArrayDepth } from 'common/libs/getArrayDepth';
import moment from 'moment';
Expand All @@ -17,12 +17,14 @@ SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
-- SELECT pg_catalog.set_config('search_path', '', false);
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;\n\n\n`;

if (this.schemaName !== 'public') dump += `CREATE SCHEMA "${this.schemaName}";\n\n`;

dump += await this.getCreateTypes();

return dump;
Expand All @@ -43,19 +45,19 @@ SET row_security = off;\n\n\n`;
};

// Table columns
const { rows } = await this._client
.select('*')
.schema('information_schema')
.from('columns')
.where({ table_schema: `= '${this.schemaName}'`, table_name: `= '${tableName}'` })
.orderBy({ ordinal_position: 'ASC' })
.run();
const { rows } = await this._client.raw(`
SELECT *
FROM "information_schema"."columns"
WHERE "table_schema" = '${this.schemaName}'
AND "table_name" = '${tableName}'
ORDER BY "ordinal_position" ASC
`, { schema: 'information_schema' });

if (!rows.length) return '';

for (const column of rows) {
let fieldType = column.data_type;
if (fieldType === 'USER-DEFINED') fieldType = column.udt_name;
if (fieldType === 'USER-DEFINED') fieldType = `"${this.schemaName}".${column.udt_name}`;
else if (fieldType === 'ARRAY') {
if (Object.keys(arrayTypes).includes(fieldType))
fieldType = arrayTypes[type] + '[]';
Expand All @@ -71,8 +73,7 @@ SET row_security = off;\n\n\n`;
if (column.column_default) {
columnArr.push(`DEFAULT ${column.column_default}`);
if (column.column_default.includes('nextval')) {
let sequenceName = column.column_default.split('\'')[1];
if (sequenceName.includes('.')) sequenceName = sequenceName.split('.')[1];
const sequenceName = column.column_default.split('\'')[1];
sequences.push(sequenceName);
}
}
Expand All @@ -82,7 +83,9 @@ SET row_security = off;\n\n\n`;
}

// Table sequences
for (const sequence of sequences) {
for (let sequence of sequences) {
if (sequence.includes('.')) sequence = sequence.split('.')[1];

const { rows } = await this._client
.select('*')
.schema('information_schema')
Expand All @@ -91,7 +94,7 @@ SET row_security = off;\n\n\n`;
.run();

if (rows.length) {
createSql += `CREATE SEQUENCE "${sequence}"
createSql += `CREATE SEQUENCE "${this.schemaName}"."${sequence}"
START WITH ${rows[0].start_value}
INCREMENT BY ${rows[0].increment}
MINVALUE ${rows[0].minimum_value}
Expand All @@ -103,7 +106,7 @@ SET row_security = off;\n\n\n`;
}

// Table create
createSql += `\nCREATE TABLE "${tableName}"(
createSql += `\nCREATE TABLE "${this.schemaName}"."${tableName}"(
${columnsSql.join(',\n ')}
);\n`;

Expand All @@ -119,7 +122,7 @@ SET row_security = off;\n\n\n`;
.run();

for (const index of indexes)
createSql += `${index.indexdef.replaceAll(`${this.schemaName}.`, '')};\n`;
createSql += `${index.indexdef};\n`;

// Table foreigns
const { rows: foreigns } = await this._client.raw(`
Expand Down Expand Up @@ -147,15 +150,15 @@ SET row_security = off;\n\n\n`;
`);

for (const foreign of foreigns) {
this._postTablesSql += `\nALTER TABLE ONLY "${tableName}"
ADD CONSTRAINT "${foreign.constraint_name}" FOREIGN KEY ("${foreign.column_name}") REFERENCES "${foreign.foreign_table_name}" ("${foreign.foreign_column_name}") ON UPDATE ${foreign.update_rule} ON DELETE ${foreign.delete_rule};\n`;
this._postTablesSql += `\nALTER TABLE ONLY "${this.schemaName}"."${tableName}"
ADD CONSTRAINT "${foreign.constraint_name}" FOREIGN KEY ("${foreign.column_name}") REFERENCES "${this.schemaName}"."${foreign.foreign_table_name}" ("${foreign.foreign_column_name}") ON UPDATE ${foreign.update_rule} ON DELETE ${foreign.delete_rule};\n`;
}

return createSql;
}

getDropTable (tableName) {
return `DROP TABLE IF EXISTS "${tableName}";`;
return `DROP TABLE IF EXISTS "${this.schemaName}"."${tableName}";`;
}

async * getTableInsert (tableName) {
Expand All @@ -166,16 +169,12 @@ SET row_security = off;\n\n\n`;
if (countResults.rows.length === 1) rowCount = countResults.rows[0].count;

if (rowCount > 0) {
let queryLength = 0;
let rowsWritten = 0;
const { sqlInsertDivider, sqlInsertAfter } = this._options;
const columns = await this._client.getTableColumns({
table: tableName,
schema: this.schemaName
});

const notGeneratedColumns = columns.filter(col => !col.generated);
const columnNames = notGeneratedColumns.map(col => '"' + col.name + '"').join(', ');
const columnNames = columns.map(col => '"' + col.name + '"').join(', ');

yield sqlStr;

Expand All @@ -190,20 +189,12 @@ SET row_security = off;\n\n\n`;
return;
}

let sqlInsertString = `INSERT INTO "${tableName}" (${columnNames}) VALUES`;

if (
(sqlInsertDivider === 'bytes' && queryLength >= sqlInsertAfter * 1024) ||
(sqlInsertDivider === 'rows' && rowsWritten === sqlInsertAfter)
) {
queryLength = 0;
rowsWritten = 0;
}
let sqlInsertString = `INSERT INTO "${this.schemaName}"."${tableName}" (${columnNames}) VALUES`;

sqlInsertString += ' (';

for (const i in notGeneratedColumns) {
const column = notGeneratedColumns[i];
for (const i in columns) {
const column = columns[i];
const val = row[column.name];

if (val === null) sqlInsertString += 'NULL';
Expand All @@ -221,31 +212,24 @@ SET row_security = off;\n\n\n`;
? this.escapeAndQuote(moment(val).format(`YYYY-MM-DD HH:mm:ss${datePrecision}`))
: this.escapeAndQuote(val);
}
else if (column.isArray) {
let parsedVal;
if (Array.isArray(val))
parsedVal = JSON.stringify(val).replaceAll('[', '{').replaceAll(']', '}');
else
parsedVal = typeof val === 'string' ? val.replaceAll('[', '{').replaceAll(']', '}') : '';
sqlInsertString += `'${parsedVal}'`;
}
else if (TEXT_SEARCH.includes(column.type))
sqlInsertString += `'${val.replaceAll('\'', '\'\'')}'`;
else if (BIT.includes(column.type))
sqlInsertString += `b'${hexToBinary(Buffer.from(val).toString('hex'))}'`;
else if (BLOB.includes(column.type))
sqlInsertString += `X'${val.toString('hex').toUpperCase()}'`;
sqlInsertString += `decode('${val.toString('hex').toUpperCase()}', 'hex')`;
else if (NUMBER.includes(column.type))
sqlInsertString += val;
else if (FLOAT.includes(column.type))
sqlInsertString += parseFloat(val);
else if (SPATIAL.includes(column.type)) {
let geoJson;
if (IS_MULTI_SPATIAL.includes(column.type)) {
const features = [];
for (const element of val)
features.push(this.getMarkers(element));

geoJson = {
type: 'FeatureCollection',
features
};
}
else
geoJson = this._getGeoJSON(val);

sqlInsertString += `ST_GeomFromGeoJSON('${JSON.stringify(geoJson)}')`;
}
else if (val === '') sqlInsertString += '\'\'';
else {
sqlInsertString += typeof val === 'string'
Expand All @@ -255,14 +239,12 @@ SET row_security = off;\n\n\n`;
: val;
}

if (parseInt(i) !== notGeneratedColumns.length - 1)
if (parseInt(i) !== columns.length - 1)
sqlInsertString += ', ';
}

sqlInsertString += ');\n';

queryLength += sqlInsertString.length;
rowsWritten++;
yield sqlInsertString;
}

Expand Down Expand Up @@ -293,7 +275,7 @@ SET row_security = off;\n\n\n`;
}, []);

for (const type of typesArr) {
sqlString += `CREATE TYPE "${type.name}" AS ENUM (
sqlString += `CREATE TYPE "${this.schemaName}"."${type.name}" AS ENUM (
${type.enums.join(',\n\t')}
);`;
}
Expand Down Expand Up @@ -339,7 +321,7 @@ SET row_security = off;\n\n\n`;
);

if (aggregateDef.length)
sqlString += '\n\n' + aggregateDef[0].format.replaceAll(`${this.schemaName}.`, '');
sqlString += '\n\n' + aggregateDef[0].format;
}
}

Expand All @@ -353,25 +335,25 @@ SET row_security = off;\n\n\n`;
for (const view of views) {
sqlString += `\nDROP VIEW IF EXISTS "${view.viewname}";\n`;

const { rows: columns } = await this._client
.select('*')
.schema('information_schema')
.from('columns')
.where({ table_schema: `= '${this.schemaName}'`, table_name: `= '${view.viewname}'` })
.orderBy({ ordinal_position: 'ASC' })
.run();

sqlString += `
CREATE VIEW "${view.viewname}" AS
SELECT
${columns.reduce((acc, curr) => {
const fieldType = curr.data_type === 'USER-DEFINED' ? curr.udt_name : curr.data_type;
acc.push(`NULL::${fieldType}${curr.character_maximum_length ? `(${curr.character_maximum_length})` : ''} AS "${curr.column_name}"`);
return acc;
}, []).join(',\n ')};
`;

sqlString += `\nCREATE OR REPLACE VIEW "${view.viewname}" AS \n${view.definition}\n`;
// const { rows: columns } = await this._client
// .select('*')
// .schema('information_schema')
// .from('columns')
// .where({ table_schema: `= '${this.schemaName}'`, table_name: `= '${view.viewname}'` })
// .orderBy({ ordinal_position: 'ASC' })
// .run();

// sqlString += `
// CREATE VIEW "${this.schemaName}"."${view.viewname}" AS
// SELECT
// ${columns.reduce((acc, curr) => {
// const fieldType = curr.data_type === 'USER-DEFINED' ? curr.udt_name : curr.data_type;
// acc.push(`NULL::${fieldType}${curr.character_maximum_length ? `(${curr.character_maximum_length})` : ''} AS "${curr.column_name}"`);
// return acc;
// }, []).join(',\n ')};
// `;

sqlString += `\nCREATE OR REPLACE VIEW "${this.schemaName}"."${view.viewname}" AS \n${view.definition}\n`;
}

return sqlString;
Expand All @@ -389,7 +371,7 @@ SELECT
const { rows: functionDef } = await this._client.raw(
`SELECT pg_get_functiondef((SELECT oid FROM pg_proc WHERE proname = '${func.name}')) AS definition`
);
sqlString += `\n${functionDef[0].definition.replaceAll(`${this.schemaName}.`, '')};\n`;
sqlString += `\n${functionDef[0].definition};\n`;
}

const { rows: triggers } = await this._client.raw(
Expand All @@ -409,7 +391,7 @@ SELECT
}, []);

for (const trigger of remappedTriggers)
sqlString += `\nCREATE TRIGGER "${trigger.trigger_name}" ${trigger.action_timing} ${trigger.events.join(' OR ')} ON "${trigger.event_object_table}" FOR EACH ${trigger.action_orientation} ${trigger.action_statement};\n`;
sqlString += `\nCREATE TRIGGER "${trigger.trigger_name}" ${trigger.action_timing} ${trigger.events.join(' OR ')} ON "${this.schemaName}"."${trigger.event_object_table}" FOR EACH ${trigger.action_orientation} ${trigger.action_statement};\n`;

return sqlString;
}
Expand All @@ -424,7 +406,7 @@ SELECT
const { rows: functionDef } = await this._client.raw(
`SELECT pg_get_functiondef((SELECT oid FROM pg_proc WHERE proname = '${func.name}')) AS definition`
);
sqlString += `\n${functionDef[0].definition.replaceAll(`${this.schemaName}.`, '')};\n`;
sqlString += `\n${functionDef[0].definition};\n`;
}

sqlString += await this.getCreateAggregates();
Expand All @@ -442,7 +424,7 @@ SELECT
const { rows: functionDef } = await this._client.raw(
`SELECT pg_get_functiondef((SELECT oid FROM pg_proc WHERE proname = '${func.name}')) AS definition`
);
sqlString += `\n${functionDef[0].definition.replaceAll(`${this.schemaName}.`, '')};\n`;
sqlString += `\n${functionDef[0].definition};\n`;
}

return sqlString;
Expand Down
46 changes: 25 additions & 21 deletions src/renderer/components/ModalExportSchema.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,28 +192,29 @@
>
<input v-model="options.includes[key]" type="checkbox"><i class="form-icon" /> {{ $tc(`word.${key}`, 2) }}
</label>

<div class="h6 mt-4 mb-2">
{{ $t('message.newInserStmtEvery') }}:
</div>
<div class="columns">
<div class="column col-6">
<input
v-model.number="options.sqlInsertAfter"
type="number"
class="form-input"
value="250"
>
<div v-if="customizations.exportByChunks">
<div class="h6 mt-4 mb-2">
{{ $t('message.newInserStmtEvery') }}:
</div>
<div class="column col-6">
<select v-model="options.sqlInsertDivider" class="form-select">
<option value="bytes">
KiB
</option>
<option value="rows">
{{ $tc('word.row', 2) }}
</option>
</select>
<div class="columns">
<div class="column col-6">
<input
v-model.number="options.sqlInsertAfter"
type="number"
class="form-input"
value="250"
>
</div>
<div class="column col-6">
<select v-model="options.sqlInsertDivider" class="form-select">
<option value="bytes">
KiB
</option>
<option value="rows">
{{ $tc('word.row', 2) }}
</option>
</select>
</div>
</div>
</div>

Expand Down Expand Up @@ -306,6 +307,9 @@ export default {
currentWorkspace () {
return this.getWorkspace(this.selectedWorkspace);
},
customizations () {
return this.currentWorkspace.customizations;
},
schemaItems () {
const db = this.currentWorkspace.structure.find(db => db.name === this.selectedSchema);
if (db)
Expand Down

0 comments on commit 0f9c991

Please sign in to comment.