From 433e734be4a9afcd9dc2f24834c0903622ee4748 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Mon, 20 Mar 2023 16:44:45 +0100 Subject: [PATCH 01/12] fix #2399 Initial conversion of JDBCMetaData plugin from PDI to Hop --- .../plugins/transforms/jdbc-metadata/pom.xml | 44 + .../jdbc-metadata/src/assembly/assembly.xml | 50 + .../src/main/resources/version.xml | 20 + assemblies/plugins/transforms/pom.xml | 1 + plugins/transforms/jdbc-metadata/pom.xml | 34 + .../transform/jdbcmetadata/JdbcMetadata.java | 705 +++++++++++++ .../jdbcmetadata/JdbcMetadataData.java | 68 ++ .../jdbcmetadata/JdbcMetadataDialog.java | 955 ++++++++++++++++++ .../jdbcmetadata/JdbcMetadataMeta.java | 829 +++++++++++++++ .../transform/jdbcmetadata/OutputField.java | 54 + .../src/main/resources/jdbcmetadata.svg | 71 ++ .../messages/messages_en_US.properties | 108 ++ .../messages/messages_it_IT.properties | 108 ++ plugins/transforms/pom.xml | 1 + .../apache/hop/ui/core/widget/ComboVar.java | 2 +- 15 files changed, 3049 insertions(+), 1 deletion(-) create mode 100644 assemblies/plugins/transforms/jdbc-metadata/pom.xml create mode 100644 assemblies/plugins/transforms/jdbc-metadata/src/assembly/assembly.xml create mode 100644 assemblies/plugins/transforms/jdbc-metadata/src/main/resources/version.xml create mode 100644 plugins/transforms/jdbc-metadata/pom.xml create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/OutputField.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/resources/jdbcmetadata.svg create mode 100644 plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties create mode 100644 plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties diff --git a/assemblies/plugins/transforms/jdbc-metadata/pom.xml b/assemblies/plugins/transforms/jdbc-metadata/pom.xml new file mode 100644 index 00000000000..564eac05d76 --- /dev/null +++ b/assemblies/plugins/transforms/jdbc-metadata/pom.xml @@ -0,0 +1,44 @@ + + + + + 4.0.0 + + + org.apache.hop + hop-assemblies-plugins-transforms + 2.4.0-SNAPSHOT + + + + hop-assemblies-plugins-transforms-jdbc-metadata + 2.4.0-SNAPSHOT + pom + + Hop Assemblies Plugins Transforms JDBC Metadata + + + + + org.apache.hop + hop-transform-jdbc-metadata + ${project.version} + + + \ No newline at end of file diff --git a/assemblies/plugins/transforms/jdbc-metadata/src/assembly/assembly.xml b/assemblies/plugins/transforms/jdbc-metadata/src/assembly/assembly.xml new file mode 100644 index 00000000000..5d090b1cb2d --- /dev/null +++ b/assemblies/plugins/transforms/jdbc-metadata/src/assembly/assembly.xml @@ -0,0 +1,50 @@ + + + + hop-assemblies-plugins-transforms-jdbc-metadata + + zip + + transforms/jdbc-metadata + + + ${project.basedir}/src/main/resources/version.xml + . + true + + + + + lib + + **/* + + + + + + false + + org.apache.hop:hop-transform-jdbc-metadata:jar + + + + \ No newline at end of file diff --git a/assemblies/plugins/transforms/jdbc-metadata/src/main/resources/version.xml b/assemblies/plugins/transforms/jdbc-metadata/src/main/resources/version.xml new file mode 100644 index 00000000000..6be576acae9 --- /dev/null +++ b/assemblies/plugins/transforms/jdbc-metadata/src/main/resources/version.xml @@ -0,0 +1,20 @@ + + + +${project.version} \ No newline at end of file diff --git a/assemblies/plugins/transforms/pom.xml b/assemblies/plugins/transforms/pom.xml index d668bcfc32f..8f83c29a673 100644 --- a/assemblies/plugins/transforms/pom.xml +++ b/assemblies/plugins/transforms/pom.xml @@ -92,6 +92,7 @@ ifnull janino javascript + jdbc-metadata joinrows json kafka diff --git a/plugins/transforms/jdbc-metadata/pom.xml b/plugins/transforms/jdbc-metadata/pom.xml new file mode 100644 index 00000000000..e68fd34c6ae --- /dev/null +++ b/plugins/transforms/jdbc-metadata/pom.xml @@ -0,0 +1,34 @@ + + + + + 4.0.0 + + + org.apache.hop + hop-plugins-transforms + 2.4.0-SNAPSHOT + + + hop-transform-jdbc-metadata + jar + + Hop Plugins JDBC Metadata + + \ No newline at end of file diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java new file mode 100644 index 00000000000..36c0e45d202 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java @@ -0,0 +1,705 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.LogLevel; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +import java.lang.reflect.Array; +import java.sql.*; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class JdbcMetadata extends BaseTransform { + public JdbcMetadata( + TransformMeta transformMeta, + JdbcMetadataMeta meta, + JdbcMetadataData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + private Object stringToArgumentValue(String stringValue, Class type) + throws IllegalArgumentException { + Object argument; + if (type == String.class) { + argument = (String) stringValue; + } else if (type == Boolean.class) { + argument = "Y".equals(stringValue) ? Boolean.TRUE : Boolean.FALSE; + } else if (type == Integer.class) { + argument = new Integer(stringValue); + } else { + throw new IllegalArgumentException("Can't handle valueType " + type.getName()); + } + return argument; + } + + private Object stringListToObjectArray(String stringValue, Class type) { + if (stringValue == null) return null; + String[] stringValues = stringValue.split(","); + int n = stringValues.length; + Object result = Array.newInstance(type, n); + for (int i = 0; i < n; i++) { + Array.set(result, i, stringToArgumentValue(stringValues[i], type)); + } + return result; + } + + /** + * Utility to set up the method and its arguments + * + * @param meta + * @param data + * @throws Exception + */ + private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exception { + // set up the method to call + /* logDebug("Setting up method to call."); + Method method = meta.getMethod(); + data.method = method; + + //Try to set up the arguments for the method + logDebug("Setting up method arguments."); + Class[] argumentTypes = method.getParameterTypes(); + int argc = argumentTypes.length; + logDebug("Method has " + argc + " arguments."); + List arguments = meta.getArguments(); + int argSize = arguments.size(); + logDebug("We expected " + argSize + " arguments."); + if (argc != argSize) { + throw new Exception("Method has a " + argc + " arguments, we expected " + argSize); + } + logDebug("Allocating arguments array."); + data.arguments = new ArrayList(); + String stringArgument; + if (meta.getArgumentSourceFields()) { + //arguments are specified by values coming from fields. + //we don't know about the fields yet, so we + //initialize with bogus value so we can check if all fields were found + logDebug("Allocating fieldindices array for arguments."); + data.argumentFieldIndices = new int[argc]; + for (int i = 0; i < argc; i++) { + data.argumentFieldIndices[i] = -1; + } + } + else { + //arguments are specified directly by the user in the step. + //here we convert the string values into proper argument values + Class argumentType; + String argument; + for (int i = 0; i < argc; i++) { + argumentType = argumentTypes[i]; + stringArgument = arguments.get(i); + if (stringArgument == null) { + argument = null; + } + else { + stringArgument = variables.resolve(stringArgument); + if (argumentType.isArray()) { + if (stringArgument.length() == 0) { + argument = null; + } + else { + argument = stringListToObjectArray(stringArgument, argumentType.getComponentType()); + } + } + else { + argument = stringToArgumentValue(stringArgument, argumentType); + } + } + data.arguments.add(i, argument); + } + }*/ + } + + /** + * Utility to create a jdbc connection. (Used when the connection source is JDBC or JDBCFields) + * + * @param driver The fully qualified classname identifying the jdbc driver + * @param url The driver-specific url to create the connection + * @param user The database user that wants to establish a connection + * @param password The password required to establish the connection + * @return A jdbc connection object. + * @throws Exception + */ + private Connection createJdbcConnection(String driver, String url, String user, String password) + throws Exception { + Class.forName(driver); + return DriverManager.getConnection(url, user, password); + } + + /** + * Initialize the connection + * + * @param meta + * @param data + * @throws Exception + */ + private void initConnection(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exception { + // Try to establish a connection in advance + logDebug("Try to establish a connection in advance."); + String connectionSource = meta.getConnectionSource(); + logDebug("Connection source: " + connectionSource); + Connection connection; + + if (JdbcMetadataMeta.connectionSourceOptionConnection.equals(connectionSource)) { + // connection is a named kettle connection + DatabaseMeta dbMeta = getPipelineMeta().findDatabase(meta.getConnectionName(), variables); + Database database = new Database(this, this, dbMeta); + database.connect(); + connection = database.getConnection(); + if (connection == null) { + throw new Exception("Connection returned by database object is null!"); + } + data.database = database; + } else if (JdbcMetadataMeta.connectionSourceOptionJDBC.equals(connectionSource)) { + // connection is a user-entered jdbc connection + String jdbcDriver = variables.resolve(meta.getJdbcDriverField()); + String jdbcUrl = variables.resolve(meta.getJdbcUrlField()); + String jdbcUser = variables.resolve(meta.getJdbcUserField()); + String jdbcPassword = variables.resolve(meta.getJdbcPasswordField()); + logDebug("Attempt to create JDBC connection."); + logDebug("Driver: " + jdbcDriver); + logDebug("Url: " + jdbcUrl); + logDebug("User: " + jdbcUser); + connection = createJdbcConnection(jdbcDriver, jdbcUrl, jdbcUser, jdbcPassword); + } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { + // Connection is a jdbc connection specified by field values + // we don't know the field values yet, but we can initialize a few vars to access their value + // later on + connection = null; + data.jdbcDriverField = -1; + data.jdbcUrlField = -1; + data.jdbcUserField = -1; + data.jdbcPasswordField = -1; + } else if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { + // Connection is a named kettle connection specified by a field value + // we don't know about the field value yet, but we can initialize a var to access its value + // later on + connection = null; + data.connectionField = -1; + } else { + // should not arrive here, just initialize the connection to make the compiler shut up. + connection = null; + } + // if the connection is null at this point, then we need to establish it on a row by row basis + data.connection = connection; + } + + private void initOutputFields(JdbcMetadataMeta meta, JdbcMetadataData data) { + List outputFields = meta.getOutputFields(); + int n = outputFields.size(); + data.resultSetIndices = new int[n]; + + IValueMeta[] fields = meta.getMethodResultSetDescriptor(); + int m = fields.length; + + for (int i = 0; i < n; i++) { + String fieldName = outputFields.get(i).getName(); + if (fieldName == null) { + continue; + } + for (int j = 0; j < m; j++) { + IValueMeta field = fields[j]; + if (!fieldName.equals(field.getName())) continue; + data.resultSetIndices[i] = j + 1; + break; + } + } + } + + @Override + public boolean init() { + boolean result = true; + if (!super.init()) return false; + + try { + data.databases = new HashMap(); + data.connections = new HashMap(); + data.connectionKey = new String[4]; + initMethod(meta, data); + initConnection(meta, data); + initOutputFields(meta, data); + } catch (Exception exception) { + logError( + "Unexpected " + + exception.getClass().getName() + + " initializing step: " + + exception.getMessage()); + exception.printStackTrace(); + result = false; + } + return result; + } + + private String getConnectionNameFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.connectionField]); + } + + private String getJdbcDriverFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.jdbcDriverField]); + } + + private String getJdbcUrlFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.jdbcUrlField]); + } + + private String getJdbcUserFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.jdbcUserField]); + } + + private String getJdbcPasswordFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.jdbcPasswordField]); + } + + /** + * This is called in the processRow function to obtain the contain to apply the metadata method + * to. + * + * @param meta + * @param data + * @param row + * @return + * @throws Exception + */ + private Connection getConnection(JdbcMetadataMeta meta, JdbcMetadataData data, Object[] row) + throws Exception { + Connection connection; + if (data.connection == null) { + // connection could not be initialized at init, so we have to obtain it now on a row by row + // basis. + String connectionSource = meta.getConnectionSource(); + if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { + // connection is a named kettle connection specified by a field value + String connectionName = getConnectionNameFromRow(data, row); + // try to get this named connection from the cache + Database database = data.databases.get(connectionName); + if (database == null) { + // we haven't seen this named connection before, try to find it. + DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(connectionName, variables); + database = new Database(this, this, databaseMeta); + connection = database.getConnection(); + if (connection == null) { + throw new IllegalArgumentException("Connection returned by database is null!"); + } + // cache the database for later use + data.databases.put(connectionName, database); + } else { + // database found in cache, get its connection + connection = database.getConnection(); + } + } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { + // database connectin is a jdbc connection defined by field values. + // try to find this connection in the cache + String[] key = data.connectionKey; + key[0] = getJdbcDriverFromRow(data, row); + key[1] = getJdbcUrlFromRow(data, row); + key[2] = getJdbcUserFromRow(data, row); + key[3] = getJdbcPasswordFromRow(data, row); + connection = data.connections.get(key); + if (connection == null) { + // connection not yet in the cache. Let's create it and cahce it. + connection = createJdbcConnection(key[0], key[1], key[2], key[3]); + data.connections.put(key, connection); + } + } else { + throw new Exception("Unexpected error acquiring connection"); + } + } else { + connection = data.connection; + } + return connection; + } + + /** + * This is called in the processRow function to get the actual arguments for the jdbc metadata + * method + * + * @param meta + * @param data + * @param row + * @throws Exception + */ + private void prepareMethodArguments(JdbcMetadataMeta meta, JdbcMetadataData data, Object[] row) + throws Exception { + // if the arguments are not from fields, then we already took care of it in the init phase, so + // leave + /* if (!meta.getArgumentSourceFields()) return; + //if the arguments are from fields, then we already stored the right indices to take the values from + Object[] args = data.arguments; + int[] indices = data.argumentFieldIndices; + int index; + Class[] argumentTypes = data.method.getParameterTypes(); + Class argumentType; + Object argument; + for (int i = 0; i < args.length; i++){ + index = indices[i]; + if (index == -2) { + argument = null; + } + else { + argument = row[index]; + } + argumentType = argumentTypes[i]; + if (argumentType.isArray() && argument != null) { + if ("".equals(argument)) { + logDebug("Converted empty string to null for argument array"); + argument = null; + } + else { + argument = stringListToObjectArray((String)argument, argumentType.getComponentType()); + } + } + args[i] = argument; + }*/ + } + + private Object[] createOutputRow( + JdbcMetadataMeta meta, JdbcMetadataData data, Object[] inputRow) { + Object[] outputRow = new Object[data.outputRowMeta.size()]; + if (data.inputFieldsToCopy == null) { + System.arraycopy(inputRow, 0, outputRow, 0, getInputRowMeta().size()); + } else { + for (int i = 0; i < data.inputFieldsToCopy.length; i++) { + outputRow[i] = inputRow[data.inputFieldsToCopy[i]]; + } + } + return outputRow; + } + + public boolean processRow() throws HopException { + + // get incoming row, getRow() potentially blocks waiting for more rows, returns null if no more + // rows expected + Object[] r = getRow(); + + // if no more rows are expected, indicate step is finished and processRow() should not be called + // again + if (r == null) { + setOutputDone(); + return false; + } + + // the "first" flag is inherited from the base step implementation + // it is used to guard some processing tasks, like figuring out field indexes + // in the row structure that only need to be done once + if (first) { + first = false; + IRowMeta inputRowMeta = getInputRowMeta(); + data.outputRowOffset = inputRowMeta.size(); + String connectionSource = meta.getConnectionSource(); + boolean argumentSourceFields = meta.isArgumentSourceFields(); + // check if we need to use fields. + // If so, store the indices so we can easily extract their values during transformation + if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource) + || JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource) + || argumentSourceFields) { + logDebug("Looking up indices of input fields."); + String fieldName; + String[] fieldNames = inputRowMeta.getFieldNames(); + logDebug("We have " + fieldNames.length + " input fields."); + List arguments = meta.getArguments(); + int argc = arguments.size(); + String stringArgument; + for (int i = 0; i < fieldNames.length; i++) { + fieldName = fieldNames[i]; + logDebug("Looking at field: " + fieldName); + // store indices for connection source fields + if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { + if (fieldName.equals(meta.getConnectionField())) { + logDebug("Found the connection field at index: " + i); + data.connectionField = i; + } + } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { + if (fieldName.equals(meta.getJdbcDriverField())) { + logDebug("Found the jdbcDriverField field at index: " + i); + data.jdbcDriverField = i; + } + if (fieldName.equals(meta.getJdbcUrlField())) { + logDebug("Found the jdbcUrlField field at index: " + i); + data.jdbcUrlField = i; + } + if (fieldName.equals(meta.getJdbcUserField())) { + logDebug("Found the jdbcUserField field at index: " + i); + data.jdbcUserField = i; + } + if (fieldName.equals(meta.getJdbcPasswordField())) { + logDebug("Found the jdbcPasswordField field at index: " + i); + data.jdbcPasswordField = i; + } + } + // store indices for argument fields + if (argumentSourceFields) { + logDebug("Trying to match argument fields against: " + fieldName); + for (int j = 0; j < argc; j++) { + stringArgument = arguments.get(j); + logDebug("Found argument " + j + ": " + stringArgument); + if (fieldName.equals(stringArgument)) { + logDebug("Match, storing index " + i); + data.argumentFieldIndices[j] = i; + } + } + } + } // end fields loop + + // ensure that we have all required fields. + if ((JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource) + && (data.jdbcDriverField == -1 + || data.jdbcUrlField == -1 + || data.jdbcUserField == -1 + || data.jdbcPasswordField == -1)) + || (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource) + && data.connectionField == -1)) { + throw new HopException("Not all fields for the connection source were found."); + } + + if (argumentSourceFields) { + // keep track of how many fields we used as args. + // We need this in case remove argument fields is enabled + // as this is the number of fields we need to discard from the input row. + int fieldsUsedAsArgs = 0; + // ensure all argument fields are bound to a valid field + argumentFields: + for (int j = 0; j < argc; j++) { + logDebug("Argument indices at " + j + ": " + data.argumentFieldIndices[j]); + if (data.argumentFieldIndices[j] == -1) { + // this argument does not point to any existing field. + if (arguments.get(j) == null || arguments.get(j).length() == 0) { + // the argument is blank, this is ok: we will pass null instead + data.argumentFieldIndices[j] = -2; + } else { + // this argument is not blank - this means it points to a non-existing field. + // this is an error. + Object[] descriptor = meta.getMethodDescriptor(); + Object[] args = (Object[]) descriptor[1]; + Object[] arg = (Object[]) args[j]; + throw new HopException( + "No field \"" + + arguments.get(j) + + "\" found for argument " + + j + + ": " + + ((String) arg[0])); + } + } else { + // this argument points to a valid field. + // let's check if this same field was already used as arg: + for (int i = 0; i < j; i++) { + if (data.argumentFieldIndices[i] == data.argumentFieldIndices[j]) { + // yes, it was used already. Let's check the next argument. + continue argumentFields; + } + } + // this field was not used already, so mark it as used. + fieldsUsedAsArgs++; + } + } + + if (meta.isRemoveArgumentFields()) { + int n = data.outputRowOffset; + data.outputRowOffset -= fieldsUsedAsArgs; + data.inputFieldsToCopy = new int[data.outputRowOffset]; + + inputFieldsToCopy: + for (int i = 0, j = 0; i < n; i++) { // for each field in the input row + for (int k = 0; k < argc; k++) { // for each method argument + if (data.argumentFieldIndices[k] == i) { + // this input field is used as argument. Continue to the next field. + continue inputFieldsToCopy; + } + } + // this field was not used as argument. make sure we copy it to the output. + data.inputFieldsToCopy[j++] = i; + } + } + } + logDebug("Done looking up indices of input fields."); + } + + // clone the input row structure and place it in our data object + data.outputRowMeta = (IRowMeta) inputRowMeta.clone(); + // use meta.getFields() to change it, so it reflects the output row structure + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); + } // end of first + + try { + logRowlevel("Processing 1 input row"); + Connection connection = getConnection(meta, data, r); + prepareMethodArguments(meta, data, r); + if (getLogLevel() == LogLevel.ROWLEVEL) { + logRowlevel("About to invoke method"); + /* for (int i = 0; i < data.arguments.size(); i++) { + logRowlevel( + "Argument " + + i + + "; " + + (data.arguments[i] == null + ? "null" + : data.arguments[i].toString() + + "; " + + data.arguments[i].getClass().getName())); + }*/ + } + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet resultSet = (ResultSet) data.method.invoke(databaseMetaData, data.arguments); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + int columnCount = resultSetMetaData.getColumnCount(); + IValueMeta IValueMeta; + Object value; + boolean outputRows = false; + int k; + /* while (resultSet.next()) { + logRowlevel("Processing 1 output row."); + Object[] outputRow = createOutputRow(meta, data, r); + for (int i = data.outputRowOffset, j = 0; i < data.outputRowMeta.size(); i++, j++) { + k = data.resultSetIndices[j]; + if (k > columnCount) continue; + IValueMeta valueMetaInterface = data.outputRowMeta.getValueMeta(i); + switch (valueMetaInterface.getType()) { + case valueMetaInterface + .TYPE_BOOLEAN: // while the JDBC spec prescribes boolean, not all drivers actually + // can deliver. + boolean v; + switch (resultSetMetaData.getColumnType(k)) { + case Types.INTEGER: + case Types.SMALLINT: + case Types.TINYINT: + v = resultSet.getInt(k) == 1 ? true : false; + break; + default: + v = resultSet.getBoolean(k); + } + value = new Boolean(v); + break; + case valueMetaInterface.TYPE_INTEGER: + value = new Long(resultSet.getInt(k)); + break; + default: + value = resultSet.getObject(k); + } + outputRow[i] = value; + outputRows = true; + } + // put the row to the output row stream + putRow(data.outputRowMeta, outputRow); + logRowlevel("Done processing 1 output row."); + }*/ + resultSet.close(); + if (!outputRows && meta.isAlwaysPassInputRow()) { + Object[] outputRow = createOutputRow(meta, data, r); + putRow(data.outputRowMeta, outputRow); + } + logRowlevel("Done processing 1 input row."); + } catch (Exception exception) { + exception.printStackTrace(); + if (exception instanceof HopException) { + throw (HopException) exception; + } else { + throw new HopException(exception); + } + } + + // log progress if it is time to to so + if (checkFeedback(getLinesRead())) { + logBasic("Linenr " + getLinesRead()); // Some basic logging + } + + // indicate that processRow() should be called again + return true; + } + + @Override + public void dispose() { + // clean up the database + try { + if (data.database != null) { + data.database.disconnect(); + data.connection = null; + data.database = null; + } + } catch (Exception ex) { + logError("Error cleaning up database: " + ex.getMessage()); + } + + // clean up the connection + try { + if (data.connection != null && !data.connection.isClosed()) { + data.connection.close(); + } + } catch (Exception ex) { + logError("Error cleaning up connection: " + ex.getMessage()); + } + data.connection = null; + + // clean up the database map + Iterator> dbIterator = data.databases.entrySet().iterator(); + while (dbIterator.hasNext()) { + Map.Entry current = dbIterator.next(); + Database database = current.getValue(); + try { + database.disconnect(); + } catch (Exception ex) { + logError("Error cleaning up database " + current.getKey() + ": " + ex.getMessage()); + } + } + data.databases.clear(); + data.databases = null; + + // clean up the connection map + Iterator> connectionIterator = + data.connections.entrySet().iterator(); + while (connectionIterator.hasNext()) { + Connection connection = connectionIterator.next().getValue(); + try { + if (!connection.isClosed()) { + connection.close(); + } + } catch (Exception ex) { + logError("Error cleaning up connection: " + ex.getMessage()); + } + } + data.connections.clear(); + data.connections = null; + + data.arguments = null; + data.method = null; + data.argumentFieldIndices = null; + data.inputFieldsToCopy = null; + data.resultSetIndices = null; + data.connectionKey = null; + + data.jdbcDriverField = -1; + data.jdbcUrlField = -1; + data.jdbcUserField = -1; + data.jdbcPasswordField = -1; + + super.dispose(); + } +} diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java new file mode 100644 index 00000000000..d9c3ecd0678 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.core.database.Database; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.ITransformData; + +import java.lang.reflect.Method; +import java.sql.Connection; +import java.util.List; +import java.util.Map; + +public class JdbcMetadataData extends BaseTransformData implements ITransformData { + + public IRowMeta outputRowMeta; + // used to store the named connection + public Database database; + // used to store the actual jdbc connection + public Connection connection; + // used to store the DatabaseMetaData method that generates the data + public Method method; + // use to store the arguments to the method + public List arguments; + // named kettle connection cache. Used when connection is a named connection specified by input + // fields + public Map databases = null; + // connection cache. Used when connection is a jdbc connection specified by input fields + public Map connections = null; + // key to the connection cache. + public String[] connectionKey = null; + // field index for named kettle connection + public int connectionField = -1; + // field indices for jdbc connections + public int jdbcDriverField = -1; + public int jdbcUrlField = -1; + public int jdbcUserField = -1; + public int jdbcPasswordField = -1; + // indices for fields used to specify method arguments + public int[] argumentFieldIndices; + // the offset in the output row from where we can add our metadata fields. + // (we need this in case we're required to remove arguments fields from the input) + public int outputRowOffset = -1; + // + public int[] inputFieldsToCopy; + // the indices of the columns in the resultset + public int[] resultSetIndices; + + public JdbcMetadataData() { + super(); + } +} diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java new file mode 100644 index 00000000000..d61fbcc8b53 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java @@ -0,0 +1,955 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.core.Const; +import org.apache.hop.core.Props; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.ITransformDialog; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.widget.ColumnInfo; +import org.apache.hop.ui.core.widget.ComboVar; +import org.apache.hop.ui.core.widget.MetaSelectionLine; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; +import org.eclipse.swt.events.*; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.*; + +import java.util.ArrayList; +import java.util.List; + +public class JdbcMetadataDialog extends BaseTransformDialog implements ITransformDialog { + private static final Class PKG = JdbcMetadataMeta.class; // for i18n purposes + private JdbcMetadataMeta meta; + + private int middle = props.getMiddlePct(); + private int margin = PropsUi.getMargin(); + + private boolean dialogChanged; + private ModifyListener lsMod; + + private Composite metadataComposite; + // + private MetaSelectionLine + wConnectionSource; // text field holding the name of the field containing the connection name + private CCombo connectionField; + // text field holding the name of the field containing the driver name + private ComboVar jdbcDriverField; + // text field holding the name of the field containing the url + private ComboVar jdbcUrlField; + // text field holding the name of the field containing the user + private ComboVar jdbcUserField; + // text field holding the name of the field containing the password + private ComboVar jdbcPasswordField; + // + + private Label alwaysPassInputRowLabel; + // + private Button alwaysPassInputRowButton; + // + private Label methodLabel; + // + // private CCombo methodCombo; + private Combo methodCombo; + + private Label argumentSourceLabel; + // + private Button argumentSourceFields; + // + private Label removeArgumentFieldsLabel; + // + private Button removeArgumentFieldsButton; + // + private TableView outputFieldsTableView; + + public JdbcMetadataDialog( + Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { + super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); + meta = (JdbcMetadataMeta) in; + } + + private final String[] emptyFieldList = new String[0]; + + private String[] getFieldListForCombo() { + String[] items; + try { + IRowMeta r = pipelineMeta.getPrevTransformFields(variables, transformName); + items = r.getFieldNames(); + } catch (HopException exception) { + items = emptyFieldList; + } + return items; + } + + private void connectionSourceUpdated() { + int selectedIndex = wConnectionSource.getSelectionIndex(); + String option = JdbcMetadataMeta.connectionSourceOptions[selectedIndex]; + boolean connectionComboEnabled, connectionFieldEnabled, otherFieldsEnabled; + otherFieldsEnabled = connectionComboEnabled = connectionFieldEnabled = false; + String[] fields = emptyFieldList; + if (JdbcMetadataMeta.connectionSourceOptionConnection.equals(option)) { + connectionComboEnabled = true; + } else { + if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(option)) { + connectionFieldEnabled = true; + connectionField.setItems(getFieldListForCombo()); + } else { + otherFieldsEnabled = true; + if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(option)) { + fields = getFieldListForCombo(); + } + } + } + wConnectionSource.setEnabled(connectionComboEnabled); + connectionField.setEnabled(connectionFieldEnabled); + jdbcDriverField.setEnabled(otherFieldsEnabled); + jdbcDriverField.setItems(fields); + jdbcUrlField.setEnabled(otherFieldsEnabled); + jdbcUrlField.setItems(fields); + jdbcUserField.setEnabled(otherFieldsEnabled); + jdbcUserField.setItems(fields); + jdbcPasswordField.setEnabled(otherFieldsEnabled); + jdbcPasswordField.setItems(fields); + } + + /** Remove the UI to enter method arguments The current values are stored and returned. */ + private List removeArgumentsUI() { + Control[] controls = metadataComposite.getChildren(); + logBasic( + "[removeArgumentsUI] metadataComposite start - Children # -> " + + controls.length); + List currentValues = new ArrayList(); + for (Control control : controls) { + if (control == alwaysPassInputRowLabel + || control == alwaysPassInputRowButton + || control == methodLabel + || control == methodCombo + || control == argumentSourceLabel + || control == argumentSourceFields + || control == removeArgumentFieldsLabel + || control == removeArgumentFieldsButton) continue; + if (control instanceof Combo ) { + if (control.isDisposed()) { + logBasic("removeArgumentsUI - trying to access a disposed control!"); + } + if (!control.isDisposed() && (((Combo) control).getText() != null || ((Combo) control).getText().length()>0)) { + currentValues.add(((Combo) control).getText()); + logBasic("removeArgumentsUI - control.getText(): " + ((Combo) control).getText()); + } + } + if (!control.isDisposed()) { + logBasic("removeArgumentsUI - Disposing control!"); + control.dispose(); + logBasic("removeArgumentsUI - number of children in parent composite: " + controls.length); + } + } + + logBasic( + "[removeArgumentsUI] metadataComposite end - Children # -> " + + metadataComposite.getChildren().length); + return currentValues; + } + + /** + * Create the UI to enter one argument. + * + * @param argumentDescriptor + * @param lastControl + * @param items + * @return The combobox where the user enters the argument. + */ + // private ComboVar createArgumentUI( + private Combo createArgumentUI( + Object[] argumentDescriptor, Control lastControl, String[] items) { + String argumentName = (String) argumentDescriptor[0]; + Label label = new Label(metadataComposite, SWT.RIGHT); + label.setText(BaseMessages.getString(PKG, "JdbcMetadata.arguments." + argumentName + ".Label")); + label.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.arguments." + argumentName + ".Tooltip")); + props.setLook(label); + FormData labelFormData = new FormData(); + labelFormData.left = new FormAttachment(0, 0); + labelFormData.right = new FormAttachment(middle, -margin); + labelFormData.top = new FormAttachment(lastControl, margin); + label.setLayoutData(labelFormData); + +// ComboVar comboVar = +// new ComboVar(variables, metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + Combo comboVar = + new Combo (metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(comboVar); + FormData comboVarFormData = new FormData(); + comboVarFormData.left = new FormAttachment(middle, 0); + comboVarFormData.right = new FormAttachment(100, 0); + comboVarFormData.top = new FormAttachment(lastControl, margin); + comboVar.setLayoutData(comboVarFormData); + comboVar.setItems(items); + + // comboVar.addModifyListener(lsMod); + + return comboVar; + } + /** Create UI to enter arguments. Return a new set of arguments to store in the meta object */ + private List createArgumentsUI(Object[] argumentDescriptors, List currentValues) { + logBasic( + "createArgumentsUI, currentValues = " + + (currentValues == null ? "null" : currentValues.size())); + Object[] argumentDescriptor; + int argc = argumentDescriptors.length; + List newArguments = new ArrayList(argc); + Control lastControl = removeArgumentFieldsButton; + String[] items = argumentSourceFields.getSelection() ? getFieldListForCombo() : emptyFieldList; + for (int i = 0; i < argc; i++) { + argumentDescriptor = (Object[]) argumentDescriptors[i]; + // ComboVar comboVar = createArgumentUI(argumentDescriptor, lastControl, items); + Combo comboVar = createArgumentUI(argumentDescriptor, lastControl, items); + lastControl = comboVar; + + // copy the old argument values to the new arguments array + if (i >= currentValues.size()) continue; + String argumentValue = currentValues.get(i); + newArguments.add(i, argumentValue); + if (argumentValue == null) continue; + comboVar.setText(argumentValue); + } + return newArguments; + } + + /** fill the fields table with output fields. */ + private void populateFieldsTable(Object[] methodDescriptor) { + logBasic("populateFieldsTable 1"); + List outputFields = getOutputFields(); + outputFieldsTableView.clearAll(); + IValueMeta[] fields = (IValueMeta[]) methodDescriptor[2]; + int n = fields.length; + Table table = outputFieldsTableView.table; + table.setItemCount(n); + TableItem tableItem; + String fieldName; + IValueMeta field; + outputFieldsTableView.optWidth(true, n); + int m = (outputFields == null) ? 0 : outputFields.size(); + OutputField outputField; + for (int i = 0; i < n; i++) { + field = fields[i]; + tableItem = table.getItem(i); + fieldName = field.getName(); + tableItem.setText(1, fieldName); + // initially, the field is renamed unto itself. + tableItem.setText(2, fieldName); + // now see if the meta object renamed this field. + for (int j = 0; j < m; j++) { + outputField = outputFields.get(j); + if (!fieldName.equals(outputField.getName())) continue; + tableItem.setText(2, outputField.getRename()); + break; + } + } + } + + private void populateFieldsTable() { + logBasic("populateFieldsTable 2"); + populateFieldsTable(JdbcMetadataMeta.getMethodDescriptor(methodCombo.getSelectionIndex())); + } + + private void updateOutputFields(List outputFields) { + logBasic("updateOutputFields " + outputFields); + if (outputFields == null) return; + outputFieldsTableView.clearAll(); + OutputField outputField; + int n = outputFields.size(); + Table table = outputFieldsTableView.table; + table.setItemCount(n); + TableItem tableItem; + String text; + for (int i = 0; i < n; i++) { + outputField = outputFields.get(i); + tableItem = table.getItem(i); + + if (outputField == null) continue; + text = outputField.getName(); + tableItem.setText(1, text == null ? "" : text); + + if (outputField.getRename() == null) continue; + text = outputField.getRename(); + tableItem.setText(2, text == null ? "" : text); + } + } + + /** + * When the method is updated, we need the ui to change to allow the user to enter arguments. This + * takes care of that. + */ + private void methodUpdated(List argumentValues) { + logBasic( + "methodUpdated, argumentValues = " + + (argumentValues == null ? "null" : argumentValues.size())); + // first, remove the controls for the previous set of arguments + List currentValues = removeArgumentsUI(); + logBasic("currentValue = " + currentValues.size()); + if (argumentValues == null) { + argumentValues = new ArrayList(currentValues.size()); + currentValues.addAll(argumentValues); + } + + // setup controls for the current set of arguments + int index = methodCombo.getSelectionIndex(); + Object[] methodDescriptor = (Object[]) JdbcMetadataMeta.methodDescriptors[index]; + Object[] argumentDescriptors = (Object[]) methodDescriptor[1]; + + List newArguments = createArgumentsUI(argumentDescriptors, argumentValues); + // update the arguments in the meta object + meta.setArguments(newArguments); + // show / hide the argument source ui depending on whether we have arguments + boolean visible = newArguments.size() > 0; + /* (SR) Temporarily removed + argumentSourceFields.setVisible(visible); + argumentSourceLabel.setVisible(visible); + removeArgumentFieldsLabel.setVisible(visible); + removeArgumentFieldsButton.setVisible(visible); + */ + metadataComposite.layout(); + } + + private void methodUpdated() { + logBasic("Parameterless methodUpdated called."); + methodUpdated(null); + } + + public String open() { + dialogChanged = false; + // store some convenient SWT variables + Shell parent = getParent(); + + // SWT code for preparing the dialog + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MIN | SWT.MAX); + props.setLook(shell); + setShellImage(shell, meta); + + changed = meta.hasChanged(); + + lsMod = e -> meta.setChanged(); + SelectionListener lsSelection = + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + meta.setChanged(); + } + }; + // ------------------------------------------------------- // + // SWT code for building the actual settings dialog // + // ------------------------------------------------------- // + Control lastControl; + + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = PropsUi.getFormMargin(); + formLayout.marginHeight = PropsUi.getFormMargin(); + + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "JdbcMetadata.Name")); + + wlTransformName = new Label(shell, SWT.RIGHT); + wlTransformName.setText(BaseMessages.getString(PKG, "System.Label.TransformName")); + props.setLook(wlTransformName); + fdlTransformName = new FormData(); + fdlTransformName.left = new FormAttachment(0, 0); + fdlTransformName.right = new FormAttachment(middle, -margin); + fdlTransformName.top = new FormAttachment(0, margin); + wlTransformName.setLayoutData(fdlTransformName); + + wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wTransformName.setText(transformName); + props.setLook(wTransformName); + wTransformName.addModifyListener(lsMod); + fdTransformName = new FormData(); + fdTransformName.left = new FormAttachment(middle, 0); + fdTransformName.top = new FormAttachment(0, margin); + fdTransformName.right = new FormAttachment(100, 0); + wTransformName.setLayoutData(fdTransformName); + + lastControl = wTransformName; + + // pass the row checkbox + alwaysPassInputRowLabel = new Label(shell, SWT.RIGHT); + alwaysPassInputRowLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.passRow.Label")); + alwaysPassInputRowLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.passRow.Tooltip")); + props.setLook(alwaysPassInputRowLabel); + FormData alwaysPassInputRowLabelFormData = new FormData(); + alwaysPassInputRowLabelFormData.left = new FormAttachment(0, 0); + alwaysPassInputRowLabelFormData.right = new FormAttachment(middle, -margin); + alwaysPassInputRowLabelFormData.top = new FormAttachment(lastControl, margin); + alwaysPassInputRowLabel.setLayoutData(alwaysPassInputRowLabelFormData); + + alwaysPassInputRowButton = new Button(shell, SWT.CHECK); + props.setLook(alwaysPassInputRowButton); + FormData alwaysPassInputRowButtonFormData = new FormData(); + alwaysPassInputRowButtonFormData.left = new FormAttachment(middle, 0); + alwaysPassInputRowButtonFormData.right = new FormAttachment(100, 0); + alwaysPassInputRowButtonFormData.top = new FormAttachment(lastControl, margin); + alwaysPassInputRowButton.setLayoutData(alwaysPassInputRowButtonFormData); + + lastControl = alwaysPassInputRowButton; + + // + CTabFolder cTabFolder = new CTabFolder(shell, SWT.BORDER); + props.setLook(cTabFolder, Props.WIDGET_STYLE_TAB); + + // Connection tab + CTabItem connectionTab = new CTabItem(cTabFolder, SWT.NONE); + connectionTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.ConnectionTab.Label")); + connectionTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.ConnectionTab.Tooltip")); + + FormLayout connectionTabLayout = new FormLayout(); + connectionTabLayout.marginWidth = PropsUi.getFormMargin(); + connectionTabLayout.marginHeight = PropsUi.getFormMargin(); + + Composite connectionComposite = new Composite(cTabFolder, SWT.NONE); + props.setLook(connectionComposite); + connectionComposite.setLayout(connectionTabLayout); + + // Connection line + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(meta.getConnectionName(), variables); + wConnectionSource = + addConnectionLine( + connectionComposite, + wTransformName, + databaseMeta, + lsMod, + BaseMessages.getString(PKG, "JdbcMetadata.connectionSource.Label"), + BaseMessages.getString(PKG, "JdbcMetadata.connectionSource.Tooltip")); + wConnectionSource.addSelectionListener(lsSelection); + lastControl = wConnectionSource; + + // connection name field + Label connectionFieldLabel = new Label(connectionComposite, SWT.RIGHT); + connectionFieldLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.connectionField.Label")); + connectionFieldLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.connectionField.Tooltip")); + props.setLook(connectionFieldLabel); + FormData connectionFieldLabelFormData = new FormData(); + connectionFieldLabelFormData.left = new FormAttachment(0, 0); + connectionFieldLabelFormData.right = new FormAttachment(middle, -margin); + connectionFieldLabelFormData.top = new FormAttachment(lastControl, margin); + connectionFieldLabel.setLayoutData(connectionFieldLabelFormData); + + connectionField = new CCombo(connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(connectionField); + connectionField.addModifyListener(lsMod); + FormData connectionFieldFormData = new FormData(); + connectionFieldFormData.left = new FormAttachment(middle, 0); + connectionFieldFormData.right = new FormAttachment(100, 0); + connectionFieldFormData.top = new FormAttachment(lastControl, margin); + connectionField.setLayoutData(connectionFieldFormData); + + lastControl = connectionField; + + // jdbc driver field + Label jdbcDriverLabel = new Label(connectionComposite, SWT.RIGHT); + jdbcDriverLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.driverField.Label")); + jdbcDriverLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.driverField.Tooltip")); + props.setLook(jdbcDriverLabel); + FormData jdbcDriverLabelFormData = new FormData(); + jdbcDriverLabelFormData.left = new FormAttachment(0, 0); + jdbcDriverLabelFormData.right = new FormAttachment(middle, -margin); + jdbcDriverLabelFormData.top = new FormAttachment(lastControl, margin); + jdbcDriverLabel.setLayoutData(jdbcDriverLabelFormData); + + jdbcDriverField = + new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(jdbcDriverField); + jdbcDriverField.addModifyListener(lsMod); + FormData jdbcDriverFieldFormData = new FormData(); + jdbcDriverFieldFormData.left = new FormAttachment(middle, 0); + jdbcDriverFieldFormData.right = new FormAttachment(100, 0); + jdbcDriverFieldFormData.top = new FormAttachment(lastControl, margin); + jdbcDriverField.setLayoutData(jdbcDriverFieldFormData); + + lastControl = jdbcDriverField; + + // jdbc url field + Label jdbcUrlLabel = new Label(connectionComposite, SWT.RIGHT); + jdbcUrlLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.urlField.Label")); + jdbcUrlLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.urlField.Tooltip")); + props.setLook(jdbcUrlLabel); + FormData jdbcUrlLabelFormData = new FormData(); + jdbcUrlLabelFormData.left = new FormAttachment(0, 0); + jdbcUrlLabelFormData.right = new FormAttachment(middle, -margin); + jdbcUrlLabelFormData.top = new FormAttachment(lastControl, margin); + jdbcUrlLabel.setLayoutData(jdbcUrlLabelFormData); + + jdbcUrlField = new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(jdbcUrlField); + jdbcUrlField.addModifyListener(lsMod); + FormData jdbcUrlFieldFormData = new FormData(); + jdbcUrlFieldFormData.left = new FormAttachment(middle, 0); + jdbcUrlFieldFormData.right = new FormAttachment(100, 0); + jdbcUrlFieldFormData.top = new FormAttachment(lastControl, margin); + jdbcUrlField.setLayoutData(jdbcUrlFieldFormData); + + lastControl = jdbcUrlField; + + // jdbc user field + Label jdbcUserLabel = new Label(connectionComposite, SWT.RIGHT); + jdbcUserLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.userField.Label")); + jdbcUserLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.userField.Tooltip")); + props.setLook(jdbcUserLabel); + FormData jdbcUserLabelFormData = new FormData(); + jdbcUserLabelFormData.left = new FormAttachment(0, 0); + jdbcUserLabelFormData.right = new FormAttachment(middle, -margin); + jdbcUserLabelFormData.top = new FormAttachment(lastControl, margin); + jdbcUserLabel.setLayoutData(jdbcUserLabelFormData); + + jdbcUserField = + new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(jdbcUserField); + jdbcUserField.addModifyListener(lsMod); + FormData jdbcUserFieldFormData = new FormData(); + jdbcUserFieldFormData.left = new FormAttachment(middle, 0); + jdbcUserFieldFormData.right = new FormAttachment(100, 0); + jdbcUserFieldFormData.top = new FormAttachment(lastControl, margin); + jdbcUserField.setLayoutData(jdbcUserFieldFormData); + + lastControl = jdbcUserField; + + // jdbc password field + Label jdbcPasswordLabel = new Label(connectionComposite, SWT.RIGHT); + jdbcPasswordLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.passwordField.Label")); + jdbcPasswordLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.passwordField.Tooltip")); + props.setLook(jdbcPasswordLabel); + FormData jdbcPasswordLabelFormData = new FormData(); + jdbcPasswordLabelFormData.left = new FormAttachment(0, 0); + jdbcPasswordLabelFormData.right = new FormAttachment(middle, -margin); + jdbcPasswordLabelFormData.top = new FormAttachment(lastControl, margin); + jdbcPasswordLabel.setLayoutData(jdbcPasswordLabelFormData); + + jdbcPasswordField = + new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(jdbcPasswordField); + jdbcPasswordField.addModifyListener(lsMod); + FormData jdbcPasswordFieldFormData = new FormData(); + jdbcPasswordFieldFormData.left = new FormAttachment(middle, 0); + jdbcPasswordFieldFormData.right = new FormAttachment(100, 0); + jdbcPasswordFieldFormData.top = new FormAttachment(lastControl, margin); + jdbcPasswordField.setLayoutData(jdbcPasswordFieldFormData); + + lastControl = jdbcPasswordField; + + // layout the connection tab + FormData connectionTabFormData = new FormData(); + connectionTabFormData.left = new FormAttachment(0, 0); + connectionTabFormData.top = new FormAttachment(0, 0); + connectionTabFormData.right = new FormAttachment(100, 0); + connectionTabFormData.bottom = new FormAttachment(100, 0); + connectionComposite.setLayoutData(connectionTabFormData); + connectionComposite.layout(); + connectionTab.setControl(connectionComposite); + + // Metadata tab + CTabItem metadataTab = new CTabItem(cTabFolder, SWT.NONE); + metadataTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.MetaDataTab.Label")); + metadataTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.MetaDataTab.Tooltip")); + + FormLayout metadataTabLayout = new FormLayout(); + connectionTabLayout.marginWidth = Const.FORM_MARGIN; + connectionTabLayout.marginHeight = Const.FORM_MARGIN; + + metadataComposite = new Composite(cTabFolder, SWT.NONE); + props.setLook(metadataComposite); + metadataComposite.setLayout(metadataTabLayout); + + // method + methodLabel = new Label(metadataComposite, SWT.RIGHT); + methodLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.metadataMethod.Label")); + methodLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.metadataMethod.Tooltip")); + props.setLook(methodLabel); + FormData methodLabelFormData = new FormData(); + methodLabelFormData.left = new FormAttachment(0, 0); + methodLabelFormData.right = new FormAttachment(middle, -margin); + methodLabelFormData.top = new FormAttachment(0, margin); + methodLabel.setLayoutData(methodLabelFormData); + + methodCombo = new Combo(metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(methodCombo); + // methodCombo.setEditable(false); + methodCombo.addModifyListener(lsMod); + FormData methodComboFormData = new FormData(); + methodComboFormData.left = new FormAttachment(middle, 0); + methodComboFormData.right = new FormAttachment(100, 0); + methodComboFormData.top = new FormAttachment(lastControl, margin); + methodCombo.setLayoutData(methodComboFormData); + + Object[] methodDescriptor; + String methodName; + for (int i = 0; i < JdbcMetadataMeta.methodDescriptors.length; i++) { + methodDescriptor = (Object[]) JdbcMetadataMeta.methodDescriptors[i]; + methodName = (String) methodDescriptor[0]; + methodCombo.add(BaseMessages.getString(PKG, "JdbcMetadata.methods." + methodName)); + } + + SelectionListener methodComboSelectionListener = + new SelectionListener() { + @Override + public void widgetDefaultSelected(SelectionEvent selectionEvent) {} + + @Override + public void widgetSelected(SelectionEvent selectionEvent) { + logBasic("methodCombo changed, calling parameterless methodUpdated"); + methodUpdated(); + populateFieldsTable(); + meta.setChanged(); + } + }; + methodCombo.addSelectionListener(methodComboSelectionListener); + lastControl = methodCombo; + + // argument source + argumentSourceLabel = new Label(metadataComposite, SWT.RIGHT); + argumentSourceLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.argumentSource.Label")); + argumentSourceLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.argumentSource.Tooltip")); + props.setLook(argumentSourceLabel); + FormData argumentSourceLabelFormData = new FormData(); + argumentSourceLabelFormData.left = new FormAttachment(0, 0); + argumentSourceLabelFormData.right = new FormAttachment(middle, -margin); + argumentSourceLabelFormData.top = new FormAttachment(lastControl, margin); + argumentSourceLabel.setLayoutData(argumentSourceLabelFormData); + + argumentSourceFields = new Button(metadataComposite, SWT.CHECK); + props.setLook(argumentSourceFields); + FormData argumentSourceFieldsFormData = new FormData(); + argumentSourceFieldsFormData.left = new FormAttachment(middle, 0); + argumentSourceFieldsFormData.right = new FormAttachment(100, 0); + argumentSourceFieldsFormData.top = new FormAttachment(lastControl, margin); + argumentSourceFields.setLayoutData(argumentSourceFieldsFormData); + SelectionListener argumentSourceFieldsSelectionListener = + new SelectionListener() { + @Override + public void widgetDefaultSelected(SelectionEvent selectionEvent) {} + + @Override + public void widgetSelected(SelectionEvent selectionEvent) { + Control[] controls = metadataComposite.getChildren(); + boolean selection = argumentSourceFields.getSelection(); + removeArgumentFieldsButton.setEnabled(selection); + String[] items = selection ? getFieldListForCombo() : emptyFieldList; + for (Control control : controls) { + if (!(control instanceof ComboVar)) continue; + ComboVar comboVar = (ComboVar) control; + comboVar.setItems(items); + } + meta.setChanged(); + } + }; + argumentSourceFields.addSelectionListener(argumentSourceFieldsSelectionListener); + + lastControl = argumentSourceFields; + + // remove arguments + removeArgumentFieldsLabel = new Label(metadataComposite, SWT.RIGHT); + removeArgumentFieldsLabel.setText( + BaseMessages.getString(PKG, "JdbcMetadata.removeArgumentFields.Label")); + removeArgumentFieldsLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.removeArgumentFields.Tooltip")); + props.setLook(removeArgumentFieldsLabel); + FormData removeArgumentFieldsLabelFormData = new FormData(); + removeArgumentFieldsLabelFormData.left = new FormAttachment(0, 0); + removeArgumentFieldsLabelFormData.right = new FormAttachment(middle, -margin); + removeArgumentFieldsLabelFormData.top = new FormAttachment(lastControl, margin); + removeArgumentFieldsLabel.setLayoutData(removeArgumentFieldsLabelFormData); + + removeArgumentFieldsButton = new Button(metadataComposite, SWT.CHECK); + props.setLook(removeArgumentFieldsButton); + FormData removeArgumentFieldsButtonFormData = new FormData(); + removeArgumentFieldsButtonFormData.left = new FormAttachment(middle, 0); + removeArgumentFieldsButtonFormData.right = new FormAttachment(100, 0); + removeArgumentFieldsButtonFormData.top = new FormAttachment(lastControl, margin); + removeArgumentFieldsButton.setLayoutData(removeArgumentFieldsButtonFormData); + + // layout the metdata tab + FormData metadataTabFormData = new FormData(); + metadataTabFormData.left = new FormAttachment(0, 0); + metadataTabFormData.top = new FormAttachment(0, 0); + metadataTabFormData.right = new FormAttachment(100, 0); + metadataTabFormData.bottom = new FormAttachment(100, 0); + metadataComposite.setLayoutData(metadataTabFormData); + metadataComposite.layout(); + metadataTab.setControl(metadataComposite); + + // Fields tab + CTabItem fieldsTab = new CTabItem(cTabFolder, SWT.NONE); + fieldsTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.FieldsTab.Label")); + fieldsTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.FieldsTab.Tooltip")); + + FormLayout fieldsTabLayout = new FormLayout(); + fieldsTabLayout.marginWidth = Const.FORM_MARGIN; + fieldsTabLayout.marginHeight = Const.FORM_MARGIN; + + Composite fieldsComposite = new Composite(cTabFolder, SWT.NONE); + props.setLook(fieldsComposite); + fieldsComposite.setLayout(fieldsTabLayout); + + // add UI for the fields tab. + Label outputFieldsTableViewLabel = new Label(fieldsComposite, SWT.NONE); + outputFieldsTableViewLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.FieldsTab.Label")); + outputFieldsTableViewLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.FieldsTab.Tooltip")); + props.setLook(outputFieldsTableViewLabel); + FormData outputFieldsTableViewLabelFormData = new FormData(); + outputFieldsTableViewLabelFormData.left = new FormAttachment(0, 0); + outputFieldsTableViewLabelFormData.top = new FormAttachment(0, margin); + outputFieldsTableViewLabel.setLayoutData(outputFieldsTableViewLabelFormData); + + ColumnInfo[] columnInfo = + new ColumnInfo[] { + new ColumnInfo( + BaseMessages.getString(PKG, "JdbcMetadata.FieldName.Label"), + ColumnInfo.COLUMN_TYPE_NONE), + new ColumnInfo( + BaseMessages.getString(PKG, "JdbcMetadata.OutputFieldName.Label"), + ColumnInfo.COLUMN_TYPE_TEXT) + }; + outputFieldsTableView = + new TableView( + variables, + fieldsComposite, + SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL, + columnInfo, + 10, + lsMod, + props); + + Button getFieldsButton = new Button(fieldsComposite, SWT.PUSH); + getFieldsButton.setText(BaseMessages.getString(PKG, "JdbcMetadata.getFieldsButton.Label")); + getFieldsButton.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.getFieldsButton.Tooltip")); + FormData getFieldsButtonFormData = new FormData(); + getFieldsButtonFormData.top = new FormAttachment(outputFieldsTableViewLabel, margin); + getFieldsButtonFormData.right = new FormAttachment(100, 0); + getFieldsButton.setLayoutData(getFieldsButtonFormData); + getFieldsButton.addSelectionListener( + new SelectionListener() { + + @Override + public void widgetDefaultSelected(SelectionEvent arg0) {} + + @Override + public void widgetSelected(SelectionEvent arg0) { + populateFieldsTable(); + meta.setChanged(); + } + }); + + FormData outputFieldsTableViewFormData = new FormData(); + outputFieldsTableViewFormData.left = new FormAttachment(0, 0); + outputFieldsTableViewFormData.top = new FormAttachment(outputFieldsTableViewLabel, margin); + outputFieldsTableViewFormData.right = new FormAttachment(getFieldsButton, -margin); + outputFieldsTableViewFormData.bottom = new FormAttachment(100, -2 * margin); + outputFieldsTableView.setLayoutData(outputFieldsTableViewFormData); + + // layout the fields tab + FormData fieldsTabFormData = new FormData(); + fieldsTabFormData.left = new FormAttachment(0, 0); + fieldsTabFormData.top = new FormAttachment(0, 0); + fieldsTabFormData.right = new FormAttachment(100, 0); + fieldsTabFormData.bottom = new FormAttachment(100, 0); + fieldsComposite.setLayoutData(metadataTabFormData); + fieldsComposite.layout(); + fieldsTab.setControl(fieldsComposite); + + // OK and cancel buttons + wOk = new Button(shell, SWT.PUSH); + wOk.setText(BaseMessages.getString(PKG, "System.Button.OK")); + wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); + + setButtonPositions(new Button[] {wOk, wCancel}, margin, null); + + // Tabfolder + FormData cTabFolderFormData = new FormData(); + cTabFolderFormData.left = new FormAttachment(0, 0); + cTabFolderFormData.top = new FormAttachment(alwaysPassInputRowButton, margin); + cTabFolderFormData.right = new FormAttachment(100, 0); + cTabFolderFormData.bottom = new FormAttachment(wOk, -margin); + cTabFolder.setLayoutData(cTabFolderFormData); + cTabFolder.setSelection(0); + + wOk.addListener(SWT.Selection, e -> ok()); + wCancel.addListener(SWT.Selection, e -> cancel()); + // default listener (for hitting "enter") + // lsDef = + // new SelectionAdapter() { + // public void widgetDefaultSelected(SelectionEvent e) { + // ok(); + // } + // }; + // wTransformName.addSelectionListener(lsDef); + // jdbcDriverField.addSelectionListener(lsDef); + + // Detect X or ALT-F4 or something that kills this window and cancel the dialog properly + shell.addShellListener( + new ShellAdapter() { + public void shellClosed(ShellEvent e) { + cancel(); + } + }); + + setSize(); + populateDialog(); + meta.setChanged(changed); + + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + + return transformName; + } + + private void selectConnectionSource(String connectionSourceOption) { + int index = JdbcMetadataMeta.getConnectionSourceOptionIndex(connectionSourceOption); + wConnectionSource.select(index); + connectionSourceUpdated(); + } + + private void setMethod(String method) { + int index = JdbcMetadataMeta.getMethodDescriptorIndex(method); + if (index == -1) throw new IllegalArgumentException("Index for method " + method + " is -1."); + methodCombo.select(index); + logBasic("setMethod called, calling parameterless method updated"); + } + + /** + * This helper method puts the step configuration stored in the meta object and puts it into the + * dialog controls. + */ + private void populateDialog() { + wTransformName.selectAll(); + String value; + + value = meta.getConnectionSource(); + selectConnectionSource(value); + + value = meta.getConnectionName(); + if (value != null) wConnectionSource.setText(value); + + value = meta.getConnectionField(); + if (value != null) connectionField.setText(value); + + value = meta.getJdbcDriverField(); + if (value != null) jdbcDriverField.setText(value); + + value = meta.getJdbcUrlField(); + if (value != null) jdbcUrlField.setText(value); + + value = meta.getJdbcUserField(); + if (value != null) jdbcUserField.setText(value); + + value = meta.getJdbcPasswordField(); + if (value != null) jdbcPasswordField.setText(value); + + alwaysPassInputRowButton.setSelection(meta.isAlwaysPassInputRow()); + + value = meta.getMethodName(); + if (value != null) setMethod(value); + + argumentSourceFields.setSelection(meta.isArgumentSourceFields()); + if (meta.getArguments() != null && meta.getArguments().size() > 0) { + methodUpdated(meta.getArguments()); + } + + logBasic("Calling methodUpdated from populate dialog."); + if (meta.getOutputFields() != null && meta.getOutputFields().size() > 0) { + updateOutputFields(meta.getOutputFields()); + } + + removeArgumentFieldsButton.setSelection(meta.isRemoveArgumentFields()); + removeArgumentFieldsButton.setEnabled(meta.isArgumentSourceFields()); + } + + /** Called when the user cancels the dialog. */ + private void cancel() { + transformName = null; + meta.setChanged(changed); + dispose(); + } + + /** Called when the user confirms the dialog */ + private void ok() { + + transformName = wTransformName.getText(); + // Save settings to the meta object + meta.setConnectionSource( + JdbcMetadataMeta.connectionSourceOptions[wConnectionSource.getSelectionIndex()]); + meta.setConnectionName(wConnectionSource.getText()); + meta.setConnectionField(connectionField.getText()); + meta.setJdbcDriverField(jdbcDriverField.getText()); + meta.setJdbcUrlField(jdbcUrlField.getText()); + meta.setJdbcUserField(jdbcUserField.getText()); + meta.setJdbcPasswordField(jdbcPasswordField.getText()); + meta.setAlwaysPassInputRow(alwaysPassInputRowButton.getSelection()); + meta.setMethodName(JdbcMetadataMeta.getMethodName(methodCombo.getSelectionIndex())); + meta.setArgumentSourceFields(argumentSourceFields.getSelection()); + meta.setArguments(getArguments()); + meta.setRemoveArgumentFields(removeArgumentFieldsButton.getSelection()); + meta.setOutputFields(getOutputFields()); + + meta.setChanged(dialogChanged || changed); + // close the SWT dialog window + dispose(); + } + + private List getArguments() { + logBasic("getArguments"); + List arguments = new ArrayList(); + Control[] controls = metadataComposite.getChildren(); + String text; + for (Control control : controls) { + if (!(control instanceof ComboVar)) continue; + ComboVar comboVar = (ComboVar) control; + text = comboVar.getText(); + arguments.add(text); + } + return arguments; + } + + private List getOutputFields() { + Table table = outputFieldsTableView.table; + List outputFields = new ArrayList<>(); + TableItem tableItem; + for (int i = 0; i < table.getItemCount(); i++) { + tableItem = table.getItem(i); + OutputField item = new OutputField(tableItem.getText(1), tableItem.getText(2)); + outputFields.add(i, item); + } + return outputFields; + } +} diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java new file mode 100644 index 00000000000..b9a92ec7cff --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java @@ -0,0 +1,829 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +import java.lang.reflect.Method; +import java.sql.DatabaseMetaData; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Transform( + id = "JdbcMetadata", + image = "jdbcmetadata.svg", + name = "i18n::JdbcMetadata.Name", + description = "i18n::JdbcMetaData.Description", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Utility", + keywords = "i18n::JdbcMetaData.keyword", + documentationUrl = "/pipeline/transforms/jdbcmetadata.html") +public class JdbcMetadataMeta extends BaseTransformMeta { + + public static Class DatabaseMetaDataClass; + + private static final Map OPTIONS_SCOPE = new HashMap(); + + static { + OPTIONS_SCOPE.put(DatabaseMetaData.bestRowSession, "bestRowSession"); + OPTIONS_SCOPE.put(DatabaseMetaData.bestRowTemporary, "bestRowTemporary"); + OPTIONS_SCOPE.put(DatabaseMetaData.bestRowTransaction, "bestRowTransaction"); + } + + // following list of COL_ static members represent columns of metadata result sets + private static final IValueMeta COL_TABLE_CAT = new ValueMetaString("TABLE_CAT"); + private static final IValueMeta COL_PKTABLE_CAT = new ValueMetaString("PKTABLE_CAT"); + private static final IValueMeta COL_FKTABLE_CAT = new ValueMetaString("FKTABLE_CAT"); + private static final IValueMeta COL_TABLE_CATALOG = new ValueMetaString("TABLE_CATALOG"); + private static final IValueMeta COL_TABLE_SCHEM = new ValueMetaString("TABLE_SCHEM"); + private static final IValueMeta COL_PKTABLE_SCHEM = new ValueMetaString("PKTABLE_SCHEM"); + private static final IValueMeta COL_FKTABLE_SCHEM = new ValueMetaString("FKTABLE_SCHEM"); + private static final IValueMeta COL_TABLE_NAME = new ValueMetaString("TABLE_NAME"); + private static final IValueMeta COL_PKTABLE_NAME = new ValueMetaString("PKTABLE_NAME"); + private static final IValueMeta COL_FKTABLE_NAME = new ValueMetaString("FKTABLE_NAME"); + private static final IValueMeta COL_TABLE_TYPE = new ValueMetaString("TABLE_TYPE"); + private static final IValueMeta COL_COLUMN_NAME = new ValueMetaString("COLUMN_NAME"); + private static final IValueMeta COL_PKCOLUMN_NAME = new ValueMetaString("PKCOLUMN_NAME"); + private static final IValueMeta COL_FKCOLUMN_NAME = new ValueMetaString("FKCOLUMN_NAME"); + private static final IValueMeta COL_PK_NAME = new ValueMetaString("PK_NAME"); + private static final IValueMeta COL_FK_NAME = new ValueMetaString("FK_NAME"); + private static final IValueMeta COL_KEY_SEQ = new ValueMetaInteger("KEY_SEQ"); + private static final IValueMeta COL_UPDATE_RULE = new ValueMetaInteger("UPDATE_RULE"); + private static final IValueMeta COL_DELETE_RULE = new ValueMetaInteger("DELETE_RULE"); + private static final IValueMeta COL_DEFERRABILITY = new ValueMetaInteger("DEFERRABILITY"); + private static final IValueMeta COL_TYPE_NAME = new ValueMetaString("TYPE_NAME"); + private static final IValueMeta COL_DATA_TYPE = new ValueMetaInteger("DATA_TYPE"); + private static final IValueMeta COL_PRECISION = new ValueMetaInteger("PRECISION"); + private static final IValueMeta COL_COLUMN_SIZE = new ValueMetaInteger("COLUMN_SIZE"); + private static final IValueMeta COL_BUFFER_LENGTH = new ValueMetaString("BUFFER_LENGTH"); + private static final IValueMeta COL_LITERAL_PREFIX = new ValueMetaString("LITERAL_PREFIX"); + private static final IValueMeta COL_LITERAL_SUFFIX = new ValueMetaString("LITERAL_SUFFIX"); + private static final IValueMeta COL_CREATE_PARAMS = new ValueMetaString("CREATE_PARAMS"); + private static final IValueMeta COL_NULLABLE = new ValueMetaInteger("NULLABLE"); + private static final IValueMeta COL_CASE_SENSITIVE = new ValueMetaBoolean("CASE_SENSITIVE"); + private static final IValueMeta COL_SEARCHABLE = new ValueMetaInteger("SEARCHABLE"); + private static final IValueMeta COL_UNSIGNED_ATTRIBUTE = + new ValueMetaBoolean("UNSIGNED_ATTRIBUTE"); + private static final IValueMeta COL_FIXED_PREC_SCALE = new ValueMetaBoolean("FIXED_PREC_SCALE"); + private static final IValueMeta COL_AUTO_INCREMENT = new ValueMetaBoolean("AUTO_INCREMENT"); + private static final IValueMeta COL_LOCAL_TYPE_NAME = new ValueMetaString("LOCAL_TYPE_NAME"); + private static final IValueMeta COL_MINIMUM_SCALE = new ValueMetaInteger("MINIMUM_SCALE"); + private static final IValueMeta COL_MAXIMUM_SCALE = new ValueMetaInteger("MAXIMUM_SCALE"); + private static final IValueMeta COL_DECIMAL_DIGITS = new ValueMetaInteger("DECIMAL_DIGITS"); + private static final IValueMeta COL_SQL_DATA_TYPE = new ValueMetaInteger("SQL_DATA_TYPE"); + private static final IValueMeta COL_SQL_DATETIME_SUB = new ValueMetaInteger("SQL_DATETIME_SUB"); + private static final IValueMeta COL_SOURCE_DATA_TYPE = new ValueMetaInteger("SOURCE_DATA_TYPE"); + private static final IValueMeta COL_NUM_PREC_RADIX = new ValueMetaInteger("NUM_PREC_RADIX"); + private static final IValueMeta COL_REMARKS = new ValueMetaString("REMARKS"); + private static final IValueMeta COL_TYPE_CAT = new ValueMetaString("TYPE_CAT"); + private static final IValueMeta COL_TYPE_SCHEM = new ValueMetaString("TYPE_SCHEM"); + private static final IValueMeta COL_SELF_REFERENCING_COL_NAME = + new ValueMetaString("SELF_REFERENCING_COL_NAME"); + private static final IValueMeta COL_REF_GENERATION = new ValueMetaString("REF_GENERATION"); + private static final IValueMeta COL_SCOPE = new ValueMetaInteger("SCOPE"); + private static final IValueMeta COL_PSEUDO_COLUMN = new ValueMetaInteger("COL_PSEUDO_COLUMN"); + private static final IValueMeta COL_GRANTOR = new ValueMetaString("GRANTOR"); + private static final IValueMeta COL_GRANTEE = new ValueMetaString("GRANTEE"); + private static final IValueMeta COL_PRIVILEGE = new ValueMetaString("PRIVILEGE"); + private static final IValueMeta COL_IS_GRANTABLE = new ValueMetaString("IS_GRANTABLE"); + private static final IValueMeta COL_COLUMN_DEF = new ValueMetaString("COLUMN_DEF"); + private static final IValueMeta COL_CHAR_OCTET_LENGTH = new ValueMetaInteger("CHAR_OCTET_LENGTH"); + private static final IValueMeta COL_ORDINAL_POSITION = new ValueMetaInteger("ORDINAL_POSITION"); + private static final IValueMeta COL_IS_NULLABLE = new ValueMetaString("IS_NULLABLE"); + private static final IValueMeta COL_SCOPE_CATALOG = new ValueMetaString("SCOPE_CATALOG"); + private static final IValueMeta COL_SCOPE_SCHEMA = new ValueMetaString("SCOPE_SCHEMA"); + private static final IValueMeta COL_SCOPE_TABLE = new ValueMetaString("SCOPE_TABLE"); + private static final IValueMeta COL_IS_AUTOINCREMENT = new ValueMetaString("IS_AUTOINCREMENT"); + private static final IValueMeta COL_IS_GENERATEDCOLUMN = + new ValueMetaString("IS_GENERATEDCOLUMN"); + + // following of argument descriptors describe arguments to metdata methods + // 1) name of the argument + // 2) java type of the argument. + private static final Object[] ARG_CATALOG = new Object[] {"catalog", String.class}; + private static final Object[] ARG_SCHEMA = new Object[] {"schema", String.class}; + private static final Object[] ARG_TABLE = new Object[] {"table", String.class}; + private static final Object[] ARG_COLUMN_NAME_PATTERN = + new Object[] {"columnNamePattern", String.class}; + private static final Object[] ARG_NULLABLE = + new Object[] {"nullable", Boolean.class, new Object[] {}}; + private static final Object[] ARG_SCHEMA_PATTERN = new Object[] {"schemaPattern", String.class}; + private static final Object[] ARG_SCOPE = new Object[] {"scope", Integer.class, OPTIONS_SCOPE}; + private static final Object[] ARG_TABLE_TYPES = new Object[] {"tableTypes", String[].class}; + private static final Object[] ARG_TABLE_NAME_PATTERN = + new Object[] {"tableNamePattern", String.class}; + private static final Object[] ARG_PARENT_CATALOG = new Object[] {"parentCatalog", String.class}; + private static final Object[] ARG_PARENT_SCHEMA = new Object[] {"parentSchema", String.class}; + private static final Object[] ARG_PARENT_TABLE = new Object[] {"parentTable", String.class}; + private static final Object[] ARG_FOREIGN_CATALOG = new Object[] {"foreignCatalog", String.class}; + private static final Object[] ARG_FOREIGN_SCHEMA = new Object[] {"foreignSchema", String.class}; + private static final Object[] ARG_FOREIGN_TABLE = new Object[] {"foreignTable", String.class}; + + // this is a map of the methods we can get metadata from. + // 1) name of the java.sql.DatabaseMetaData method + // 2) array of argument descriptors + // 3) array of return fields + // 4) initially empty slot where the actual Method object is lazily stored. + public static final Object[] methodDescriptors = + new Object[] { + new Object[] {"getCatalogs", new Object[] {}, new IValueMeta[] {COL_TABLE_CAT}, null}, + new Object[] { + "getBestRowIdentifier", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE, ARG_SCOPE, ARG_NULLABLE}, + new IValueMeta[] { + COL_SCOPE, COL_COLUMN_NAME, COL_DATA_TYPE, COL_TYPE_NAME, + COL_COLUMN_SIZE, COL_BUFFER_LENGTH, COL_DECIMAL_DIGITS, COL_PSEUDO_COLUMN + }, + null + }, + new Object[] { + "getColumnPrivileges", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE, ARG_COLUMN_NAME_PATTERN}, + new IValueMeta[] { + COL_TABLE_CAT, COL_TABLE_SCHEM, COL_TABLE_NAME, COL_COLUMN_NAME, + COL_GRANTOR, COL_GRANTEE, COL_PRIVILEGE, COL_IS_GRANTABLE + }, + null + }, + new Object[] { + "getColumns", + new Object[] { + ARG_CATALOG, ARG_SCHEMA_PATTERN, ARG_TABLE_NAME_PATTERN, ARG_COLUMN_NAME_PATTERN + }, + new IValueMeta[] { + COL_TABLE_CAT, + COL_TABLE_SCHEM, + COL_TABLE_NAME, + COL_COLUMN_NAME, + COL_DATA_TYPE, + COL_TYPE_NAME, + COL_COLUMN_SIZE, + COL_BUFFER_LENGTH, + COL_DECIMAL_DIGITS, + COL_NUM_PREC_RADIX, + COL_NULLABLE, + COL_REMARKS, + COL_COLUMN_DEF, + COL_SQL_DATA_TYPE, + COL_SQL_DATETIME_SUB, + COL_CHAR_OCTET_LENGTH, + COL_ORDINAL_POSITION, + COL_IS_NULLABLE, + COL_SCOPE_CATALOG, + COL_SCOPE_SCHEMA, + COL_SCOPE_TABLE, + COL_SOURCE_DATA_TYPE, + COL_IS_AUTOINCREMENT, + COL_IS_GENERATEDCOLUMN + }, + null + }, + new Object[] { + "getCrossReference", + new Object[] { + ARG_PARENT_CATALOG, ARG_PARENT_SCHEMA, ARG_PARENT_TABLE, + ARG_FOREIGN_CATALOG, ARG_FOREIGN_SCHEMA, ARG_FOREIGN_TABLE, + }, + new IValueMeta[] { + COL_PKTABLE_CAT, + COL_PKTABLE_SCHEM, + COL_PKTABLE_NAME, + COL_PKCOLUMN_NAME, + COL_FKTABLE_CAT, + COL_FKTABLE_SCHEM, + COL_FKTABLE_NAME, + COL_FKCOLUMN_NAME, + COL_KEY_SEQ, + COL_UPDATE_RULE, + COL_DELETE_RULE, + COL_FK_NAME, + COL_PK_NAME, + COL_DEFERRABILITY + }, + null + }, + new Object[] { + "getExportedKeys", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE}, + new IValueMeta[] { + COL_PKTABLE_CAT, + COL_PKTABLE_SCHEM, + COL_PKTABLE_NAME, + COL_PKCOLUMN_NAME, + COL_FKTABLE_CAT, + COL_FKTABLE_SCHEM, + COL_FKTABLE_NAME, + COL_FKCOLUMN_NAME, + COL_KEY_SEQ, + COL_UPDATE_RULE, + COL_DELETE_RULE, + COL_FK_NAME, + COL_PK_NAME, + COL_DEFERRABILITY + }, + null + }, + new Object[] { + "getImportedKeys", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE}, + new IValueMeta[] { + COL_PKTABLE_CAT, + COL_PKTABLE_SCHEM, + COL_PKTABLE_NAME, + COL_PKCOLUMN_NAME, + COL_FKTABLE_CAT, + COL_FKTABLE_SCHEM, + COL_FKTABLE_NAME, + COL_FKCOLUMN_NAME, + COL_KEY_SEQ, + COL_UPDATE_RULE, + COL_DELETE_RULE, + COL_FK_NAME, + COL_PK_NAME, + COL_DEFERRABILITY + }, + null + }, + new Object[] { + "getPrimaryKeys", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE}, + new IValueMeta[] { + COL_TABLE_CAT, + COL_TABLE_SCHEM, + COL_TABLE_NAME, + COL_COLUMN_NAME, + COL_KEY_SEQ, + COL_PK_NAME + }, + null + }, + new Object[] { + "getSchemas", new Object[] {}, new IValueMeta[] {COL_TABLE_SCHEM, COL_TABLE_CATALOG}, null + }, + /* We'd love to use this version of getSchemas, but we found that calling it throws AbstractMethodError in h2 and sqlite (possibly others) + new Object[]{ + "getSchemas", + new Object[]{ARG_CATALOG, ARG_SCHEMA_PATTERN}, + new IValueMeta[]{COL_TABLE_SCHEM, COL_TABLE_CATALOG}, + null + }, + */ + new Object[] { + "getTablePrivileges", + new Object[] {ARG_CATALOG, ARG_SCHEMA_PATTERN, ARG_TABLE_NAME_PATTERN}, + new IValueMeta[] { + COL_TABLE_CAT, + COL_TABLE_SCHEM, + COL_TABLE_NAME, + COL_GRANTOR, + COL_GRANTEE, + COL_PRIVILEGE, + COL_IS_GRANTABLE + }, + null + }, + new Object[] {"getTableTypes", new Object[] {}, new IValueMeta[] {COL_TABLE_TYPE}, null}, + new Object[] { + "getTables", + new Object[] {ARG_CATALOG, ARG_SCHEMA_PATTERN, ARG_TABLE_NAME_PATTERN, ARG_TABLE_TYPES}, + new IValueMeta[] { + COL_TABLE_CAT, + COL_TABLE_SCHEM, + COL_TABLE_NAME, + COL_TABLE_TYPE, + COL_REMARKS, + COL_TYPE_CAT, + COL_TYPE_SCHEM, + COL_TYPE_NAME, + COL_SELF_REFERENCING_COL_NAME, + COL_REF_GENERATION + }, + null + }, + new Object[] { + "getTypeInfo", + new Object[] {}, + new IValueMeta[] { + COL_TYPE_NAME, + COL_DATA_TYPE, + COL_PRECISION, + COL_LITERAL_PREFIX, + COL_LITERAL_SUFFIX, + COL_CREATE_PARAMS, + COL_NULLABLE, + COL_CASE_SENSITIVE, + COL_SEARCHABLE, + COL_UNSIGNED_ATTRIBUTE, + COL_FIXED_PREC_SCALE, + COL_AUTO_INCREMENT, + COL_LOCAL_TYPE_NAME, + COL_MINIMUM_SCALE, + COL_MAXIMUM_SCALE, + COL_SQL_DATA_TYPE, + COL_SQL_DATETIME_SUB, + COL_NUM_PREC_RADIX + }, + null + }, + new Object[] { + "getVersionColumns", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE}, + new IValueMeta[] { + COL_SCOPE, COL_COLUMN_NAME, COL_DATA_TYPE, COL_TYPE_NAME, + COL_COLUMN_SIZE, COL_BUFFER_LENGTH, COL_DECIMAL_DIGITS, COL_PSEUDO_COLUMN + }, + null + } + }; + + private static final String CONNECTION_SOURCE = "connectionSource"; + private static final String CONNECTION_NAME = "connectionName"; + private static final String CONNECTION_FIELD = "connectionField"; + private static final String JDBC_DRIVER_FIELD = "jdbcDriverField"; + private static final String JDBC_URL_FIELD = "jdbcUrlField"; + private static final String JDBC_USER_FIELD = "jdbcUserField"; + private static final String JDBC_PASSWORD_FIELD = "jdbcPasswordField"; + private static final String ALWAYS_PASS_INPUT_ROW = "alwaysPassInputRow"; + private static final String METHOD_NAME = "methodName"; + private static final String REMOVE_ARGUMENT_FIELDS = "removeArgumentFields"; + private static final String ARGUMENT_SOURCE_FIELDS = "argumentSourceFields"; + private static final String ARGUMENT = "argument"; + private static final String ARGUMENTS = ARGUMENT + "s"; + private static final String OUTPUT_FIELD = "outputField"; + private static final String OUTPUT_FIELDS = OUTPUT_FIELD + "s"; + private static final String FIELD_NAME = "name"; + private static final String FIELD_RENAME = "rename"; + + /** + * The PKG member is used when looking up internationalized strings. The properties file with + * localized keys is expected to reside in {the package of the class + * specified}/messages/messages_{locale}.properties + */ + private static Class PKG = JdbcMetadataMeta.class; // for i18n purposes + + public static final String connectionSourceOptionConnection = "Connection"; + public static final String connectionSourceOptionConnectionField = "ConnectionField"; + public static final String connectionSourceOptionJDBC = "JDBC"; + public static final String connectionSourceOptionJDBCFields = "JDBCFields"; + + public static final String[] connectionSourceOptions = + new String[] { + connectionSourceOptionConnection, + connectionSourceOptionConnectionField, + connectionSourceOptionJDBC, + connectionSourceOptionJDBCFields + }; + + /** + * Constructor should call super() to make sure the base class has a chance to initialize + * properly. + */ + @SuppressWarnings("unchecked") + public JdbcMetadataMeta() { + super(); + if (JdbcMetadataMeta.DatabaseMetaDataClass == null) { + try { + JdbcMetadataMeta.DatabaseMetaDataClass = + (Class) Class.forName("java.sql.DatabaseMetaData"); + } catch (Exception exception) { + throw new IllegalArgumentException(exception); + } + } + } + + public void setDefault() { + connectionSource = "Connection"; + connectionName = ""; + jdbcDriverField = ""; + jdbcUrlField = ""; + jdbcUserField = ""; + jdbcPasswordField = ""; + methodName = "getCatalogs"; + argumentSourceFields = false; + outputFields = new ArrayList<>(); + outputFields.add(new OutputField("TABLE_CAT","TABLE_CAT")); + } + + /** Stores the connectionsource. */ + @HopMetadataProperty + private String connectionSource; + /** + * Get the index of the connection source option + * + * @param connectionSourceOption + * @return + */ + public static int getConnectionSourceOptionIndex(String connectionSourceOption) { + for (int i = 0; i < connectionSourceOptions.length; i++) { + if (connectionSourceOptions[i].equals(connectionSourceOption)) return i; + } + return -1; + } + /** + * Getter for the name of the field containing the connection source + * + * @return the source of the connection data + */ + public String getConnectionSource() { + return connectionSource; + } + /** + * Setter for the name of the field containing the connection source + * + * @param connectionSource the source for the connection data + */ + public void setConnectionSource(String connectionSource) { + if (connectionSource == null) connectionSource = connectionSourceOptions[0]; + else if (getConnectionSourceOptionIndex(connectionSource) == -1) + throw new IllegalArgumentException( + connectionSource + " is not a valid value for connectionSource."); + this.connectionSource = connectionSource; + } + + /** Stores the name of connection. */ + @HopMetadataProperty + private String connectionName; + /** + * Getter for the name of the connection + * + * @return the name of the connection + */ + public String getConnectionName() { + return connectionName; + } + /** + * Setter for the name of the connection + * + * @param connectionName the name of the connection + */ + public void setConnectionName(String connectionName) { + this.connectionName = connectionName; + } + + /** Stores the name of field holding the connection name. */ + @HopMetadataProperty + private String connectionField; + /** + * Getter for the name of the field holding the connection name + * + * @return the name of the field holding the connection name + */ + public String getConnectionField() { + return connectionField; + } + /** + * Setter for the name of the field holding the name of the connection + * + * @param connectionField the name of the field holding the connection name + */ + public void setConnectionField(String connectionField) { + this.connectionField = connectionField; + } + /** Stores the name of the field containing the name of the jdbc driver. */ + @HopMetadataProperty + private String jdbcDriverField; + /** + * Getter for the name of the field containing the jdbc driver + * + * @return the name of the field containing the jdbc driver + */ + public String getJdbcDriverField() { + return jdbcDriverField; + } + /** + * Setter for the name of the field containing the jdbc driver + * + * @param jdbcDriverField the name of the field containing the jdbc driver + */ + public void setJdbcDriverField(String jdbcDriverField) { + this.jdbcDriverField = jdbcDriverField; + } + + /** Stores the name of the field containing the url for the jdbc connection. */ + @HopMetadataProperty + private String jdbcUrlField; + /** + * Getter for the name of the field containing the jdbc url + * + * @return the name of the field containing the jdbc url + */ + public String getJdbcUrlField() { + return jdbcUrlField; + } + /** + * Setter for the name of the field containing the jdbc url + * + * @param jdbcUrlField the name of the field containing the jdbc url + */ + public void setJdbcUrlField(String jdbcUrlField) { + this.jdbcUrlField = jdbcUrlField; + } + + /** Stores the name of the field containing the username for the jdbc connection. */ + @HopMetadataProperty + private String jdbcUserField; + /** + * Getter for the name of the field containing the jdbc user + * + * @return the name of the field containing the jdbc user + */ + public String getJdbcUserField() { + return jdbcUserField; + } + /** + * Setter for the name of the field containing the jdbc user + * + * @param jdbcUserField the name of the field containing the jdbc user + */ + public void setJdbcUserField(String jdbcUserField) { + this.jdbcUserField = jdbcUserField; + } + + /** Stores the name of the field containing the password for the jdbc connection. */ + @HopMetadataProperty + private String jdbcPasswordField; + /** + * Getter for the name of the field containing the jdbc password + * + * @return the name of the field containing the jdbc password + */ + public String getJdbcPasswordField() { + return jdbcPasswordField; + } + /** + * Setter for the name of the field containing the jdbc password + * + * @param jdbcPasswordField the name of the field containing the jdbc password + */ + public void setJdbcPasswordField(String jdbcPasswordField) { + this.jdbcPasswordField = jdbcPasswordField; + } + + /** Stores the whether the input row should be returned even if no metadata was found */ + @HopMetadataProperty + private boolean alwaysPassInputRow; + /** + * @return whether fields are to be used as arguments + */ + public boolean isAlwaysPassInputRow() { + return alwaysPassInputRow; + } + + /** + * @param alwaysPassInputRow whether fields should be used as arguments + */ + public void setAlwaysPassInputRow(boolean alwaysPassInputRow) { + this.alwaysPassInputRow = alwaysPassInputRow; + } + + /** Stores the name of the method used to get the metadata */ + @HopMetadataProperty + private String methodName; + /** + * Getter for the name method used to get metadata + * + * @return the name of the method used to get metadata + */ + public String getMethodName() { + return methodName; + } + /** + * Setter for the name of the method used to get metadata + * + * @param methodName the name of the method used to get metadata + */ + public void setMethodName(String methodName) { + this.methodName = methodName; + } + + private static Object[] getMethodDescriptor(String methodName) { + for (Object o : JdbcMetadataMeta.methodDescriptors) { + Object[] oo = (Object[]) o; + if (!oo[0].toString().equals(methodName)) continue; + return oo; + } + return null; + } + + public static Object[] getMethodDescriptor(int index) { + return (Object[]) JdbcMetadataMeta.methodDescriptors[index]; + } + + public Object[] getMethodDescriptor() { + return JdbcMetadataMeta.getMethodDescriptor(getMethodName()); + } + + public IValueMeta[] getMethodResultSetDescriptor() { + Object[] methodDescriptor = getMethodDescriptor(); + return (IValueMeta[]) methodDescriptor[2]; + } + + public static int getMethodDescriptorIndex(String methodName) { + Object[] methods = JdbcMetadataMeta.methodDescriptors; + int n = methods.length; + Object[] methodDescriptor; + String name; + for (int i = 0; i < n; i++) { + methodDescriptor = (Object[]) methods[i]; + name = (String) methodDescriptor[0]; + if (name.equals(methodName)) return i; + } + return -1; + } + + public static String getMethodName(int methodDescriptorIndex) { + Object[] methodDescriptor = + (Object[]) JdbcMetadataMeta.methodDescriptors[methodDescriptorIndex]; + String methodName = (String) methodDescriptor[0]; + return methodName; + } + + public static Class[] getMethodParameterTypes(Object[] methodDescriptor) { + Object[] parameters = (Object[]) methodDescriptor[1]; + Object[] parameter; + int n = parameters.length; + Class[] parameterTypes = new Class[n]; + for (int i = 0; i < n; i++) { + parameter = (Object[]) parameters[i]; + parameterTypes[i] = (Class) parameter[1]; + } + return parameterTypes; + } + + public static Method getMethod(String methodName) throws Exception { + Method method; + Object[] methodDescriptor = getMethodDescriptor(methodName); + method = (Method) methodDescriptor[3]; + if (method != null) return method; + Class dbmd = Class.forName("java.sql.DatabaseMetaData"); + Class[] parameterTypes = getMethodParameterTypes(methodDescriptor); + method = dbmd.getDeclaredMethod(methodName, parameterTypes); + methodDescriptor[3] = method; + return method; + } + + public Method getMethod() throws Exception { + return JdbcMetadataMeta.getMethod(methodName); + } + + /** Stores the whether fields are to be used for method arguments */ + @HopMetadataProperty + private boolean argumentSourceFields; + /** + * @return whether fields are to be used as arguments + */ + public boolean isArgumentSourceFields() { + return argumentSourceFields; + } + /** + * @param argumentSourceFields whether fields should be used as arguments + */ + public void setArgumentSourceFields(boolean argumentSourceFields) { + this.argumentSourceFields = argumentSourceFields; + } + + /** Stores the whether to remove the fields used as arguments from the output row */ + @HopMetadataProperty + private boolean removeArgumentFields; + /** + * @return whether to remove the fields used as arguments from the output row + */ + public boolean isRemoveArgumentFields() { + return removeArgumentFields; + } + /** + * @param removeArgumentFields whether fields used as arguments should be removed from the output + * row + */ + public void setRemoveArgumentFields(boolean removeArgumentFields) { + this.removeArgumentFields = removeArgumentFields; + } + + /** Stores method arguments */ + @HopMetadataProperty(groupKey = "arguments", key = "argument") + private List arguments; + /** + * @return get method arguments + */ + public List getArguments() { + return arguments; + } + + /** + * @param arguments whether fields should be used as arguments + */ + public void setArguments(List arguments) { + this.arguments = arguments; + } + + /** Stores the selection of fields that are added to the stream */ + @HopMetadataProperty(groupKey = "outputFields", key="outputField") + private List outputFields; + /** + * @return the selection of fields added to the stream + */ + public List getOutputFields() { + return outputFields; + } + + /** + * @param outputFields whether fields should be used as arguments + */ + public void setOutputFields(List outputFields) { + this.outputFields = outputFields; + } + + /** + * This method is used when a step is duplicated in Spoon. It needs to return a deep copy of this + * step meta object. Be sure to create proper deep copies if the step configuration is stored in + * modifiable objects. + * + *

See org.pentaho.di.trans.steps.rowgenerator.RowGeneratorMeta.clone() for an example on + * creating a deep copy. + * + * @return a deep copy of this + */ + public Object clone() { + Object retval = super.clone(); + return retval; + } + + public void getFields( + IRowMeta rowMeta, + String origin, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) + throws HopTransformException { + // remove argument source fields coming from the input + if (argumentSourceFields && removeArgumentFields) { + for (int i = 0; i < arguments.size(); i++) { + try { + rowMeta.removeValueMeta(arguments.get(i)); + } catch (HopValueException ex) { + // this probably means the requested field could not be found. + // we can't really handle this here; however, it's not a problem + // because a missing field will be detected before writing rows. + } + } + } + + // add the outputfields added by this step. + List outputFields = getOutputFields(); + OutputField outputField; + int n = outputFields.size(); + + Object[] methodDescriptor = getMethodDescriptor(); + IValueMeta[] fields = (IValueMeta[]) methodDescriptor[2]; + int m = fields.length; + IValueMeta field; + String fieldName; + + for (int i = 0; i < n; i++) { + outputField = outputFields.get(i); + for (int j = 0; j < m; j++) { + field = fields[j]; + fieldName = field.getName(); + if (!fieldName.equals(outputField.getName())) { + continue; + } + field = + new ValueMetaBase(outputField.getRename() == null ? fieldName : outputField.getRename(), field.getType()); + field.setOrigin(origin); + rowMeta.addValueMeta(field); + break; + } + } + } + +} diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/OutputField.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/OutputField.java new file mode 100644 index 00000000000..b3b3d7e8a9c --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/OutputField.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.metadata.api.HopMetadataProperty; + +public class OutputField { + + @HopMetadataProperty + private String name; + + @HopMetadataProperty + private String rename; + + public OutputField() { + } + + public OutputField(String name, String rename) { + this.name = name; + this.rename = rename; + } + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getRename() { + return rename; + } + + public void setRename(String rename) { + this.rename = rename; + } +} diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/jdbcmetadata.svg b/plugins/transforms/jdbc-metadata/src/main/resources/jdbcmetadata.svg new file mode 100644 index 00000000000..600dbc8485f --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/resources/jdbcmetadata.svg @@ -0,0 +1,71 @@ + + + +image/svg+xml \ No newline at end of file diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties new file mode 100644 index 00000000000..b4bd0b07bf7 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +JdbcMetadata.Name=Get JDBC Metadata +JdbcMetadata.Name.Description=Get JDBC Metadata + +JdbcMetadata.connectionSource.Label=Connection Source +JdbcMetadata.connectionSource.Tooltip=Specifies how to establish a database connection +JdbcMetadata.connectionSource.options.Connection=Hop Connection +JdbcMetadata.connectionSource.options.ConnectionField=Hop connection defined by field +JdbcMetadata.connectionSource.options.JDBC=JDBC connection +JdbcMetadata.connectionSource.options.JDBCFields=JDBC connection defined by fields + +JdbcMetadata.ConnectionTab.Label=Connection +JdbcMetadata.ConnectionTab.Tooltip=Use this tab to specify how a database connection will be acquired +JdbcMetadata.MetaDataTab.Label=Method and Arguments +JdbcMetadata.MetaDataTab.Tooltip=Use this tab to specify the kind of metadata and any required parameters +JdbcMetadata.FieldsTab.Label=Output Fields +JdbcMetadata.FieldsTab.Tooltip=Use this tab to define which metadata fields you want to add to the outputstream + +JdbcMetadata.connectionField.Label=Connection field +JdbcMetadata.connectionField.Tooltip=The name of the field from the input stream that specifies a named Hop connection +JdbcMetadata.driverField.Label=Driver class +JdbcMetadata.driverField.Tooltip=The fully qualified classname of the jdbc driver +JdbcMetadata.urlField.Label=URL +JdbcMetadata.urlField.Tooltip=The driver-specific URL to establish a database connection +JdbcMetadata.userField.Label=Username +JdbcMetadata.userField.Tooltip=The database user that establishes the connection +JdbcMetadata.passwordField.Label=Password +JdbcMetadata.passwordField.Tooltip=The password of the database user that establishes the connection +JdbcMetadata.passRow.Label=Always pass the input row? +JdbcMetadata.passRow.Tooltip=Check this if you want to pass the input row even if no metadata was found. (Metadata output fields will be null in this case) +JdbcMetadata.metadataMethod.Label=MetaData Method +JdbcMetadata.metadataMethod.Tooltip=This specifies the kind of metadata to retrieve. +JdbcMetadata.argumentSource.Label=Get Arguments from fields? +JdbcMetadata.argumentSource.Tooltip=Check this if you want to acquire the values for the method arguments from the fields of the input stream +JdbcMetadata.removeArgumentFields.Label=Remove argument fields? +JdbcMetadata.removeArgumentFields.Tooltip=Check this if you do not want to pass on the method argument fields to the output stream +JdbcMetadata.getFieldsButton.Label=Get Fields +JdbcMetadata.getFieldsButton.Tooltip=Click to retrieve all possible output fields for the current metadata method +JdbcMetadata.FieldName.Label=Field name +JdbcMetadata.FieldName.Tooltip=The name of the output field as it is returned by the metadata method +JdbcMetadata.OutputFieldName.Label=Rename to +JdbcMetadata.OutputFieldName.Tooltip=The custom name for this output field + +JdbcMetadata.methods.getBestRowIdentifier=Best row identifier +JdbcMetadata.methods.getCatalogs=Catalogs +JdbcMetadata.methods.getColumnPrivileges=Column privileges +JdbcMetadata.methods.getColumns=Columns +JdbcMetadata.methods.getCrossReference=Cross references +JdbcMetadata.methods.getExportedKeys=Exported key columns +JdbcMetadata.methods.getImportedKeys=Foreign key columns +JdbcMetadata.methods.getPrimaryKeys=Primary key columns +JdbcMetadata.methods.getSchemas=Schemas +JdbcMetadata.methods.getTablePrivileges=Table privileges +JdbcMetadata.methods.getTableTypes=Table types +JdbcMetadata.methods.getTables=Tables +JdbcMetadata.methods.getTypeInfo=Data types +JdbcMetadata.methods.getVersionColumns=Version columns + +JdbcMetadata.arguments.catalog.Label=Catalog +JdbcMetadata.arguments.catalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.parentCatalog.Label=Parent catalog +JdbcMetadata.arguments.parentCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.foreignCatalog.Label=Foreign catalog +JdbcMetadata.arguments.foreignCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.columnNamePattern.Label=Column name pattern +JdbcMetadata.arguments.columnNamePattern.Tooltip=A column name pattern; must match the column name as it is stored in the database +JdbcMetadata.arguments.nullable.Label=Nullable? +JdbcMetadata.arguments.nullable.Tooltip=Include columns that are nullable +JdbcMetadata.arguments.schema.Label=Schema +JdbcMetadata.arguments.schema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.parentSchema.Label=Parent schema +JdbcMetadata.arguments.parentSchema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.foreignSchema.Label=Foreign schema +JdbcMetadata.arguments.foreignSchema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.schemaPattern.Label=Schema pattern +JdbcMetadata.arguments.schemaPattern.Tooltip=A schema pattern name; must match the schema name as it is stored in the database; +JdbcMetadata.arguments.scope.Label=Identity scope +JdbcMetadata.arguments.scope.Tooltip=The scope of interest +JdbcMetadata.arguments.table.Label=Table +JdbcMetadata.arguments.table.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.parentTable.Label=Parent table +JdbcMetadata.arguments.parentTable.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.foreignTable.Label=Foreign table +JdbcMetadata.arguments.foreignTable.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.tableNamePattern.Label=Table name pattern +JdbcMetadata.arguments.tableNamePattern.Tooltip=A table name pattern; may be a fully-qualified name +JdbcMetadata.arguments.tableTypes.Label=Table types +JdbcMetadata.arguments.tableTypes.Tooltip=A list of table types + +JdbcMetaData.CheckResult.ReceivingRows.OK=Transform is receiving input from other transforms. +JdbcMetaData.CheckResult.ReceivingRows.ERROR=No input received from other transforms! +JdbcMetaData.keyword=Utility,JDBC Metadata \ No newline at end of file diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties new file mode 100644 index 00000000000..b4bd0b07bf7 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +JdbcMetadata.Name=Get JDBC Metadata +JdbcMetadata.Name.Description=Get JDBC Metadata + +JdbcMetadata.connectionSource.Label=Connection Source +JdbcMetadata.connectionSource.Tooltip=Specifies how to establish a database connection +JdbcMetadata.connectionSource.options.Connection=Hop Connection +JdbcMetadata.connectionSource.options.ConnectionField=Hop connection defined by field +JdbcMetadata.connectionSource.options.JDBC=JDBC connection +JdbcMetadata.connectionSource.options.JDBCFields=JDBC connection defined by fields + +JdbcMetadata.ConnectionTab.Label=Connection +JdbcMetadata.ConnectionTab.Tooltip=Use this tab to specify how a database connection will be acquired +JdbcMetadata.MetaDataTab.Label=Method and Arguments +JdbcMetadata.MetaDataTab.Tooltip=Use this tab to specify the kind of metadata and any required parameters +JdbcMetadata.FieldsTab.Label=Output Fields +JdbcMetadata.FieldsTab.Tooltip=Use this tab to define which metadata fields you want to add to the outputstream + +JdbcMetadata.connectionField.Label=Connection field +JdbcMetadata.connectionField.Tooltip=The name of the field from the input stream that specifies a named Hop connection +JdbcMetadata.driverField.Label=Driver class +JdbcMetadata.driverField.Tooltip=The fully qualified classname of the jdbc driver +JdbcMetadata.urlField.Label=URL +JdbcMetadata.urlField.Tooltip=The driver-specific URL to establish a database connection +JdbcMetadata.userField.Label=Username +JdbcMetadata.userField.Tooltip=The database user that establishes the connection +JdbcMetadata.passwordField.Label=Password +JdbcMetadata.passwordField.Tooltip=The password of the database user that establishes the connection +JdbcMetadata.passRow.Label=Always pass the input row? +JdbcMetadata.passRow.Tooltip=Check this if you want to pass the input row even if no metadata was found. (Metadata output fields will be null in this case) +JdbcMetadata.metadataMethod.Label=MetaData Method +JdbcMetadata.metadataMethod.Tooltip=This specifies the kind of metadata to retrieve. +JdbcMetadata.argumentSource.Label=Get Arguments from fields? +JdbcMetadata.argumentSource.Tooltip=Check this if you want to acquire the values for the method arguments from the fields of the input stream +JdbcMetadata.removeArgumentFields.Label=Remove argument fields? +JdbcMetadata.removeArgumentFields.Tooltip=Check this if you do not want to pass on the method argument fields to the output stream +JdbcMetadata.getFieldsButton.Label=Get Fields +JdbcMetadata.getFieldsButton.Tooltip=Click to retrieve all possible output fields for the current metadata method +JdbcMetadata.FieldName.Label=Field name +JdbcMetadata.FieldName.Tooltip=The name of the output field as it is returned by the metadata method +JdbcMetadata.OutputFieldName.Label=Rename to +JdbcMetadata.OutputFieldName.Tooltip=The custom name for this output field + +JdbcMetadata.methods.getBestRowIdentifier=Best row identifier +JdbcMetadata.methods.getCatalogs=Catalogs +JdbcMetadata.methods.getColumnPrivileges=Column privileges +JdbcMetadata.methods.getColumns=Columns +JdbcMetadata.methods.getCrossReference=Cross references +JdbcMetadata.methods.getExportedKeys=Exported key columns +JdbcMetadata.methods.getImportedKeys=Foreign key columns +JdbcMetadata.methods.getPrimaryKeys=Primary key columns +JdbcMetadata.methods.getSchemas=Schemas +JdbcMetadata.methods.getTablePrivileges=Table privileges +JdbcMetadata.methods.getTableTypes=Table types +JdbcMetadata.methods.getTables=Tables +JdbcMetadata.methods.getTypeInfo=Data types +JdbcMetadata.methods.getVersionColumns=Version columns + +JdbcMetadata.arguments.catalog.Label=Catalog +JdbcMetadata.arguments.catalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.parentCatalog.Label=Parent catalog +JdbcMetadata.arguments.parentCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.foreignCatalog.Label=Foreign catalog +JdbcMetadata.arguments.foreignCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.columnNamePattern.Label=Column name pattern +JdbcMetadata.arguments.columnNamePattern.Tooltip=A column name pattern; must match the column name as it is stored in the database +JdbcMetadata.arguments.nullable.Label=Nullable? +JdbcMetadata.arguments.nullable.Tooltip=Include columns that are nullable +JdbcMetadata.arguments.schema.Label=Schema +JdbcMetadata.arguments.schema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.parentSchema.Label=Parent schema +JdbcMetadata.arguments.parentSchema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.foreignSchema.Label=Foreign schema +JdbcMetadata.arguments.foreignSchema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.schemaPattern.Label=Schema pattern +JdbcMetadata.arguments.schemaPattern.Tooltip=A schema pattern name; must match the schema name as it is stored in the database; +JdbcMetadata.arguments.scope.Label=Identity scope +JdbcMetadata.arguments.scope.Tooltip=The scope of interest +JdbcMetadata.arguments.table.Label=Table +JdbcMetadata.arguments.table.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.parentTable.Label=Parent table +JdbcMetadata.arguments.parentTable.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.foreignTable.Label=Foreign table +JdbcMetadata.arguments.foreignTable.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.tableNamePattern.Label=Table name pattern +JdbcMetadata.arguments.tableNamePattern.Tooltip=A table name pattern; may be a fully-qualified name +JdbcMetadata.arguments.tableTypes.Label=Table types +JdbcMetadata.arguments.tableTypes.Tooltip=A list of table types + +JdbcMetaData.CheckResult.ReceivingRows.OK=Transform is receiving input from other transforms. +JdbcMetaData.CheckResult.ReceivingRows.ERROR=No input received from other transforms! +JdbcMetaData.keyword=Utility,JDBC Metadata \ No newline at end of file diff --git a/plugins/transforms/pom.xml b/plugins/transforms/pom.xml index 064a539967d..243931b6b11 100644 --- a/plugins/transforms/pom.xml +++ b/plugins/transforms/pom.xml @@ -123,6 +123,7 @@ insertupdate janino javascript + jdbc-metadata joinrows json kafka diff --git a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java index 99a3fb83a4c..e2977fd35b2 100644 --- a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java +++ b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java @@ -225,7 +225,7 @@ public synchronized boolean setFocus() { @Override public void dispose() { - if (wCombo != null && wCombo != null) { + if (wCombo != null) { wCombo.dispose(); } } From 23fae095ddec8313fd48a49a513ce9e68a288b78 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Tue, 21 Mar 2023 10:13:16 +0100 Subject: [PATCH 02/12] fix #2752 ComboVar component not correctly disposed --- .../java/org/apache/hop/ui/core/widget/ComboVar.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java index e2977fd35b2..ab0c8616146 100644 --- a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java +++ b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java @@ -56,6 +56,7 @@ public class ComboVar extends Composite { private IVariables variables; private CCombo wCombo; + private Label wlImage; private ModifyListener modifyListenerTooltipText; @@ -99,7 +100,7 @@ public ComboVar( // Add the variable $ image on the top right of the control // - Label wlImage = new Label(this, SWT.NONE); + wlImage = new Label(this, SWT.NONE); wlImage.setImage(GuiResource.getInstance().getImageVariableMini()); wlImage.setToolTipText(BaseMessages.getString(PKG, "TextVar.tooltip.InsertVariable")); FormData fdlImage = new FormData(); @@ -225,9 +226,13 @@ public synchronized boolean setFocus() { @Override public void dispose() { - if (wCombo != null) { + if (wCombo != null && !wCombo.isDisposed()) { wCombo.dispose(); } + if (wlImage != null && !wlImage.isDisposed()) { + wlImage.dispose(); + } + super.dispose(); } @Override From 7316e6a3fff09607c5d6d145d429c3b3a9665c25 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Tue, 21 Mar 2023 13:27:37 +0100 Subject: [PATCH 03/12] fix #2399 Light refactoring and bugfixing. --- .../transform/jdbcmetadata/JdbcMetadata.java | 115 ++++++------ .../jdbcmetadata/JdbcMetadataData.java | 3 +- .../jdbcmetadata/JdbcMetadataDialog.java | 176 +++++++++--------- 3 files changed, 146 insertions(+), 148 deletions(-) diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java index 36c0e45d202..2df3e35b0fc 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java @@ -29,6 +29,7 @@ import org.apache.hop.pipeline.transform.TransformMeta; import java.lang.reflect.Array; +import java.lang.reflect.Method; import java.sql.*; import java.util.HashMap; import java.util.Iterator; @@ -50,11 +51,11 @@ private Object stringToArgumentValue(String stringValue, Class type) throws IllegalArgumentException { Object argument; if (type == String.class) { - argument = (String) stringValue; + argument = stringValue; } else if (type == Boolean.class) { argument = "Y".equals(stringValue) ? Boolean.TRUE : Boolean.FALSE; } else if (type == Integer.class) { - argument = new Integer(stringValue); + argument = Integer.valueOf(stringValue); } else { throw new IllegalArgumentException("Can't handle valueType " + type.getName()); } @@ -80,8 +81,8 @@ private Object stringListToObjectArray(String stringValue, Class type) { * @throws Exception */ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exception { - // set up the method to call - /* logDebug("Setting up method to call."); + //set up the method to call + logDebug("Setting up method to call."); Method method = meta.getMethod(); data.method = method; @@ -90,16 +91,16 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc Class[] argumentTypes = method.getParameterTypes(); int argc = argumentTypes.length; logDebug("Method has " + argc + " arguments."); - List arguments = meta.getArguments(); - int argSize = arguments.size(); - logDebug("We expected " + argSize + " arguments."); - if (argc != argSize) { - throw new Exception("Method has a " + argc + " arguments, we expected " + argSize); + String[] arguments = new String[meta.getArguments().size()]; + meta.getArguments().toArray(arguments); + logDebug("We expected " + arguments.length + " arguments."); + if (argc != arguments.length) { + throw new Exception("Method has a " + argc + " arguments, we expected " + arguments.length); } logDebug("Allocating arguments array."); - data.arguments = new ArrayList(); + data.arguments = new Object[argc]; String stringArgument; - if (meta.getArgumentSourceFields()) { + if (meta.isArgumentSourceFields()) { //arguments are specified by values coming from fields. //we don't know about the fields yet, so we //initialize with bogus value so we can check if all fields were found @@ -113,10 +114,10 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc //arguments are specified directly by the user in the step. //here we convert the string values into proper argument values Class argumentType; - String argument; + Object argument; for (int i = 0; i < argc; i++) { argumentType = argumentTypes[i]; - stringArgument = arguments.get(i); + stringArgument = arguments[i]; if (stringArgument == null) { argument = null; } @@ -134,9 +135,9 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc argument = stringToArgumentValue(stringArgument, argumentType); } } - data.arguments.add(i, argument); + data.arguments[i] = argument; } - }*/ + } } /** @@ -351,20 +352,20 @@ private void prepareMethodArguments(JdbcMetadataMeta meta, JdbcMetadataData data throws Exception { // if the arguments are not from fields, then we already took care of it in the init phase, so // leave - /* if (!meta.getArgumentSourceFields()) return; - //if the arguments are from fields, then we already stored the right indices to take the values from + if (!meta.isArgumentSourceFields()) return; + // if the arguments are from fields, then we already stored the right indices to take the values + // from Object[] args = data.arguments; int[] indices = data.argumentFieldIndices; int index; Class[] argumentTypes = data.method.getParameterTypes(); Class argumentType; Object argument; - for (int i = 0; i < args.length; i++){ + for (int i = 0; i < args.length; i++) { index = indices[i]; if (index == -2) { argument = null; - } - else { + } else { argument = row[index]; } argumentType = argumentTypes[i]; @@ -372,13 +373,12 @@ private void prepareMethodArguments(JdbcMetadataMeta meta, JdbcMetadataData data if ("".equals(argument)) { logDebug("Converted empty string to null for argument array"); argument = null; - } - else { - argument = stringListToObjectArray((String)argument, argumentType.getComponentType()); + } else { + argument = stringListToObjectArray((String) argument, argumentType.getComponentType()); } } args[i] = argument; - }*/ + } } private Object[] createOutputRow( @@ -506,7 +506,7 @@ public boolean processRow() throws HopException { + "\" found for argument " + j + ": " - + ((String) arg[0])); + + arg[0]); } } else { // this argument points to a valid field. @@ -544,7 +544,7 @@ public boolean processRow() throws HopException { } // clone the input row structure and place it in our data object - data.outputRowMeta = (IRowMeta) inputRowMeta.clone(); + data.outputRowMeta = inputRowMeta.clone(); // use meta.getFields() to change it, so it reflects the output row structure meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); } // end of first @@ -555,7 +555,7 @@ public boolean processRow() throws HopException { prepareMethodArguments(meta, data, r); if (getLogLevel() == LogLevel.ROWLEVEL) { logRowlevel("About to invoke method"); - /* for (int i = 0; i < data.arguments.size(); i++) { + for (int i = 0; i < data.arguments.length; i++) { logRowlevel( "Argument " + i @@ -565,8 +565,9 @@ public boolean processRow() throws HopException { : data.arguments[i].toString() + "; " + data.arguments[i].getClass().getName())); - }*/ + } } + DatabaseMetaData databaseMetaData = connection.getMetaData(); ResultSet resultSet = (ResultSet) data.method.invoke(databaseMetaData, data.arguments); ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); @@ -575,42 +576,21 @@ public boolean processRow() throws HopException { Object value; boolean outputRows = false; int k; - /* while (resultSet.next()) { + while (resultSet.next()) { logRowlevel("Processing 1 output row."); Object[] outputRow = createOutputRow(meta, data, r); for (int i = data.outputRowOffset, j = 0; i < data.outputRowMeta.size(); i++, j++) { k = data.resultSetIndices[j]; if (k > columnCount) continue; - IValueMeta valueMetaInterface = data.outputRowMeta.getValueMeta(i); - switch (valueMetaInterface.getType()) { - case valueMetaInterface - .TYPE_BOOLEAN: // while the JDBC spec prescribes boolean, not all drivers actually - // can deliver. - boolean v; - switch (resultSetMetaData.getColumnType(k)) { - case Types.INTEGER: - case Types.SMALLINT: - case Types.TINYINT: - v = resultSet.getInt(k) == 1 ? true : false; - break; - default: - v = resultSet.getBoolean(k); - } - value = new Boolean(v); - break; - case valueMetaInterface.TYPE_INTEGER: - value = new Long(resultSet.getInt(k)); - break; - default: - value = resultSet.getObject(k); - } + int type = data.outputRowMeta.getValueMeta(i).getType(); + value = getColumnValue(resultSet, resultSetMetaData, k, type); outputRow[i] = value; outputRows = true; } // put the row to the output row stream putRow(data.outputRowMeta, outputRow); logRowlevel("Done processing 1 output row."); - }*/ + } resultSet.close(); if (!outputRows && meta.isAlwaysPassInputRow()) { Object[] outputRow = createOutputRow(meta, data, r); @@ -635,6 +615,35 @@ public boolean processRow() throws HopException { return true; } + private static Object getColumnValue( + ResultSet resultSet, ResultSetMetaData resultSetMetaData, int k, int type) + throws SQLException { + Object value; + switch (type) { + case IValueMeta + .TYPE_BOOLEAN: // while the JDBC spec prescribes boolean, not all drivers actually + // can deliver. + boolean v; + switch (resultSetMetaData.getColumnType(k)) { + case Types.INTEGER: + case Types.SMALLINT: + case Types.TINYINT: + v = resultSet.getInt(k) == 1; + break; + default: + v = resultSet.getBoolean(k); + } + value = Boolean.valueOf(v); + break; + case IValueMeta.TYPE_INTEGER: + value = Long.valueOf(resultSet.getInt(k)); + break; + default: + value = resultSet.getObject(k); + } + return value; + } + @Override public void dispose() { // clean up the database diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java index d9c3ecd0678..423d7a5293e 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java @@ -24,7 +24,6 @@ import java.lang.reflect.Method; import java.sql.Connection; -import java.util.List; import java.util.Map; public class JdbcMetadataData extends BaseTransformData implements ITransformData { @@ -37,7 +36,7 @@ public class JdbcMetadataData extends BaseTransformData implements ITransformDat // used to store the DatabaseMetaData method that generates the data public Method method; // use to store the arguments to the method - public List arguments; + public Object[] arguments; // named kettle connection cache. Used when connection is a named connection specified by input // fields public Map databases = null; diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java index d61fbcc8b53..5598d696c86 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java @@ -50,7 +50,7 @@ public class JdbcMetadataDialog extends BaseTransformDialog implements ITransformDialog { private static final Class PKG = JdbcMetadataMeta.class; // for i18n purposes - private JdbcMetadataMeta meta; + private JdbcMetadataMeta input; private int middle = props.getMiddlePct(); private int margin = PropsUi.getMargin(); @@ -95,7 +95,7 @@ public class JdbcMetadataDialog extends BaseTransformDialog implements ITransfor public JdbcMetadataDialog( Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); - meta = (JdbcMetadataMeta) in; + input = (JdbcMetadataMeta) in; } private final String[] emptyFieldList = new String[0]; @@ -145,9 +145,6 @@ private void connectionSourceUpdated() { /** Remove the UI to enter method arguments The current values are stored and returned. */ private List removeArgumentsUI() { Control[] controls = metadataComposite.getChildren(); - logBasic( - "[removeArgumentsUI] metadataComposite start - Children # -> " - + controls.length); List currentValues = new ArrayList(); for (Control control : controls) { if (control == alwaysPassInputRowLabel @@ -158,25 +155,20 @@ private List removeArgumentsUI() { || control == argumentSourceFields || control == removeArgumentFieldsLabel || control == removeArgumentFieldsButton) continue; - if (control instanceof Combo ) { - if (control.isDisposed()) { - logBasic("removeArgumentsUI - trying to access a disposed control!"); - } - if (!control.isDisposed() && (((Combo) control).getText() != null || ((Combo) control).getText().length()>0)) { - currentValues.add(((Combo) control).getText()); - logBasic("removeArgumentsUI - control.getText(): " + ((Combo) control).getText()); + if (control instanceof ComboVar) { + if (!control.isDisposed() + && (((ComboVar) control).getText() != null + || ((ComboVar) control).getText().length() > 0)) { + currentValues.add(((ComboVar) control).getText()); } } if (!control.isDisposed()) { - logBasic("removeArgumentsUI - Disposing control!"); + logDebug("removeArgumentsUI - Disposing control!"); control.dispose(); - logBasic("removeArgumentsUI - number of children in parent composite: " + controls.length); + logDebug("removeArgumentsUI - number of children in parent composite: " + controls.length); } } - - logBasic( - "[removeArgumentsUI] metadataComposite end - Children # -> " - + metadataComposite.getChildren().length); + ; return currentValues; } @@ -189,7 +181,7 @@ private List removeArgumentsUI() { * @return The combobox where the user enters the argument. */ // private ComboVar createArgumentUI( - private Combo createArgumentUI( + private ComboVar createArgumentUI( Object[] argumentDescriptor, Control lastControl, String[] items) { String argumentName = (String) argumentDescriptor[0]; Label label = new Label(metadataComposite, SWT.RIGHT); @@ -203,10 +195,8 @@ private Combo createArgumentUI( labelFormData.top = new FormAttachment(lastControl, margin); label.setLayoutData(labelFormData); -// ComboVar comboVar = -// new ComboVar(variables, metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - Combo comboVar = - new Combo (metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + ComboVar comboVar = + new ComboVar(variables, metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); props.setLook(comboVar); FormData comboVarFormData = new FormData(); comboVarFormData.left = new FormAttachment(middle, 0); @@ -221,7 +211,7 @@ private Combo createArgumentUI( } /** Create UI to enter arguments. Return a new set of arguments to store in the meta object */ private List createArgumentsUI(Object[] argumentDescriptors, List currentValues) { - logBasic( + logDebug( "createArgumentsUI, currentValues = " + (currentValues == null ? "null" : currentValues.size())); Object[] argumentDescriptor; @@ -231,8 +221,7 @@ private List createArgumentsUI(Object[] argumentDescriptors, List createArgumentsUI(Object[] argumentDescriptors, List outputFields = getOutputFields(); outputFieldsTableView.clearAll(); IValueMeta[] fields = (IValueMeta[]) methodDescriptor[2]; @@ -278,12 +267,12 @@ private void populateFieldsTable(Object[] methodDescriptor) { } private void populateFieldsTable() { - logBasic("populateFieldsTable 2"); + logDebug("populateFieldsTable 2"); populateFieldsTable(JdbcMetadataMeta.getMethodDescriptor(methodCombo.getSelectionIndex())); } private void updateOutputFields(List outputFields) { - logBasic("updateOutputFields " + outputFields); + logDebug("updateOutputFields " + outputFields); if (outputFields == null) return; outputFieldsTableView.clearAll(); OutputField outputField; @@ -311,12 +300,12 @@ private void updateOutputFields(List outputFields) { * takes care of that. */ private void methodUpdated(List argumentValues) { - logBasic( + logDebug( "methodUpdated, argumentValues = " + (argumentValues == null ? "null" : argumentValues.size())); // first, remove the controls for the previous set of arguments List currentValues = removeArgumentsUI(); - logBasic("currentValue = " + currentValues.size()); + logDebug("currentValue = " + currentValues.size()); if (argumentValues == null) { argumentValues = new ArrayList(currentValues.size()); currentValues.addAll(argumentValues); @@ -329,41 +318,33 @@ private void methodUpdated(List argumentValues) { List newArguments = createArgumentsUI(argumentDescriptors, argumentValues); // update the arguments in the meta object - meta.setArguments(newArguments); - // show / hide the argument source ui depending on whether we have arguments - boolean visible = newArguments.size() > 0; - /* (SR) Temporarily removed - argumentSourceFields.setVisible(visible); - argumentSourceLabel.setVisible(visible); - removeArgumentFieldsLabel.setVisible(visible); - removeArgumentFieldsButton.setVisible(visible); - */ + input.setArguments(newArguments); + metadataComposite.layout(); } private void methodUpdated() { - logBasic("Parameterless methodUpdated called."); + logDebug("Parameterless methodUpdated called."); methodUpdated(null); } public String open() { dialogChanged = false; - // store some convenient SWT variables Shell parent = getParent(); // SWT code for preparing the dialog shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MIN | SWT.MAX); props.setLook(shell); - setShellImage(shell, meta); + setShellImage(shell, input); - changed = meta.hasChanged(); + changed = input.hasChanged(); - lsMod = e -> meta.setChanged(); + lsMod = e -> input.setChanged(); SelectionListener lsSelection = new SelectionAdapter() { @Override public void widgetSelected(SelectionEvent e) { - meta.setChanged(); + input.setChanged(); } }; // ------------------------------------------------------- // @@ -439,7 +420,7 @@ public void widgetSelected(SelectionEvent e) { connectionComposite.setLayout(connectionTabLayout); // Connection line - DatabaseMeta databaseMeta = pipelineMeta.findDatabase(meta.getConnectionName(), variables); + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(input.getConnectionName(), variables); wConnectionSource = addConnectionLine( connectionComposite, @@ -602,7 +583,6 @@ public void widgetSelected(SelectionEvent e) { methodCombo = new Combo(metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); props.setLook(methodCombo); - // methodCombo.setEditable(false); methodCombo.addModifyListener(lsMod); FormData methodComboFormData = new FormData(); methodComboFormData.left = new FormAttachment(middle, 0); @@ -625,10 +605,10 @@ public void widgetDefaultSelected(SelectionEvent selectionEvent) {} @Override public void widgetSelected(SelectionEvent selectionEvent) { - logBasic("methodCombo changed, calling parameterless methodUpdated"); + logDebug("methodCombo changed, calling parameterless methodUpdated"); methodUpdated(); populateFieldsTable(); - meta.setChanged(); + input.setChanged(); } }; methodCombo.addSelectionListener(methodComboSelectionListener); @@ -662,14 +642,22 @@ public void widgetDefaultSelected(SelectionEvent selectionEvent) {} public void widgetSelected(SelectionEvent selectionEvent) { Control[] controls = metadataComposite.getChildren(); boolean selection = argumentSourceFields.getSelection(); + removeArgumentFieldsButton.setSelection(selection); removeArgumentFieldsButton.setEnabled(selection); String[] items = selection ? getFieldListForCombo() : emptyFieldList; - for (Control control : controls) { - if (!(control instanceof ComboVar)) continue; - ComboVar comboVar = (ComboVar) control; - comboVar.setItems(items); + + if (!selection) { + removeArgumentsUI(); + } else { + methodUpdated(null); + for (Control control : controls) { + if (!(control instanceof ComboVar)) continue; + ComboVar comboVar = (ComboVar) control; + comboVar.setItems(items); + } } - meta.setChanged(); + + input.setChanged(); } }; argumentSourceFields.addSelectionListener(argumentSourceFieldsSelectionListener); @@ -767,7 +755,7 @@ public void widgetDefaultSelected(SelectionEvent arg0) {} @Override public void widgetSelected(SelectionEvent arg0) { populateFieldsTable(); - meta.setChanged(); + input.setChanged(); } }); @@ -827,7 +815,7 @@ public void shellClosed(ShellEvent e) { setSize(); populateDialog(); - meta.setChanged(changed); + input.setChanged(changed); BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); @@ -844,7 +832,7 @@ private void setMethod(String method) { int index = JdbcMetadataMeta.getMethodDescriptorIndex(method); if (index == -1) throw new IllegalArgumentException("Index for method " + method + " is -1."); methodCombo.select(index); - logBasic("setMethod called, calling parameterless method updated"); + logDebug("setMethod called, calling parameterless method updated"); } /** @@ -855,50 +843,55 @@ private void populateDialog() { wTransformName.selectAll(); String value; - value = meta.getConnectionSource(); + value = input.getConnectionSource(); selectConnectionSource(value); - value = meta.getConnectionName(); + value = input.getConnectionName(); if (value != null) wConnectionSource.setText(value); - value = meta.getConnectionField(); + value = input.getConnectionField(); if (value != null) connectionField.setText(value); - value = meta.getJdbcDriverField(); + value = input.getJdbcDriverField(); if (value != null) jdbcDriverField.setText(value); - value = meta.getJdbcUrlField(); + value = input.getJdbcUrlField(); if (value != null) jdbcUrlField.setText(value); - value = meta.getJdbcUserField(); + value = input.getJdbcUserField(); if (value != null) jdbcUserField.setText(value); - value = meta.getJdbcPasswordField(); + value = input.getJdbcPasswordField(); if (value != null) jdbcPasswordField.setText(value); - alwaysPassInputRowButton.setSelection(meta.isAlwaysPassInputRow()); + alwaysPassInputRowButton.setSelection(input.isAlwaysPassInputRow()); - value = meta.getMethodName(); + value = input.getMethodName(); if (value != null) setMethod(value); - argumentSourceFields.setSelection(meta.isArgumentSourceFields()); - if (meta.getArguments() != null && meta.getArguments().size() > 0) { - methodUpdated(meta.getArguments()); + if (input.isArgumentSourceFields()) { + argumentSourceFields.setSelection(input.isArgumentSourceFields()); + if (input.getArguments() != null && input.getArguments().size() > 0) { + methodUpdated(input.getArguments()); + } else { + // If no arguments' values are saved build the arguments' list without assigning any values + methodUpdated(new ArrayList<>()); + } } - logBasic("Calling methodUpdated from populate dialog."); - if (meta.getOutputFields() != null && meta.getOutputFields().size() > 0) { - updateOutputFields(meta.getOutputFields()); + logDebug("Calling methodUpdated from populate dialog."); + if (input.getOutputFields() != null && input.getOutputFields().size() > 0) { + updateOutputFields(input.getOutputFields()); } - removeArgumentFieldsButton.setSelection(meta.isRemoveArgumentFields()); - removeArgumentFieldsButton.setEnabled(meta.isArgumentSourceFields()); + removeArgumentFieldsButton.setSelection(input.isRemoveArgumentFields()); + removeArgumentFieldsButton.setEnabled(input.isArgumentSourceFields()); } /** Called when the user cancels the dialog. */ private void cancel() { transformName = null; - meta.setChanged(changed); + input.setChanged(changed); dispose(); } @@ -907,28 +900,25 @@ private void ok() { transformName = wTransformName.getText(); // Save settings to the meta object - meta.setConnectionSource( + input.setConnectionSource( JdbcMetadataMeta.connectionSourceOptions[wConnectionSource.getSelectionIndex()]); - meta.setConnectionName(wConnectionSource.getText()); - meta.setConnectionField(connectionField.getText()); - meta.setJdbcDriverField(jdbcDriverField.getText()); - meta.setJdbcUrlField(jdbcUrlField.getText()); - meta.setJdbcUserField(jdbcUserField.getText()); - meta.setJdbcPasswordField(jdbcPasswordField.getText()); - meta.setAlwaysPassInputRow(alwaysPassInputRowButton.getSelection()); - meta.setMethodName(JdbcMetadataMeta.getMethodName(methodCombo.getSelectionIndex())); - meta.setArgumentSourceFields(argumentSourceFields.getSelection()); - meta.setArguments(getArguments()); - meta.setRemoveArgumentFields(removeArgumentFieldsButton.getSelection()); - meta.setOutputFields(getOutputFields()); - - meta.setChanged(dialogChanged || changed); - // close the SWT dialog window + input.setConnectionName(wConnectionSource.getText()); + input.setConnectionField(connectionField.getText()); + input.setJdbcDriverField(jdbcDriverField.getText()); + input.setJdbcUrlField(jdbcUrlField.getText()); + input.setJdbcUserField(jdbcUserField.getText()); + input.setJdbcPasswordField(jdbcPasswordField.getText()); + input.setAlwaysPassInputRow(alwaysPassInputRowButton.getSelection()); + input.setMethodName(JdbcMetadataMeta.getMethodName(methodCombo.getSelectionIndex())); + input.setArgumentSourceFields(argumentSourceFields.getSelection()); + input.setArguments(getArguments()); + input.setRemoveArgumentFields(removeArgumentFieldsButton.getSelection()); + input.setOutputFields(getOutputFields()); + dispose(); } private List getArguments() { - logBasic("getArguments"); List arguments = new ArrayList(); Control[] controls = metadataComposite.getChildren(); String text; From 0954b2531f359595b016c0052d975baaf8574c56 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Thu, 23 Mar 2023 15:28:13 +0100 Subject: [PATCH 04/12] fix #2399 Definitely finished migrating Get JDBC Metadata to Hop --- .../transform/jdbcmetadata/JdbcMetadata.java | 335 +----------------- .../jdbcmetadata/JdbcMetadataData.java | 17 +- .../jdbcmetadata/JdbcMetadataDialog.java | 297 ++-------------- .../jdbcmetadata/JdbcMetadataMeta.java | 191 +--------- .../messages/messages_en_US.properties | 29 +- .../messages/messages_it_IT.properties | 78 ++-- 6 files changed, 82 insertions(+), 865 deletions(-) diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java index 2df3e35b0fc..89ce023ef13 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java @@ -31,10 +31,7 @@ import java.lang.reflect.Array; import java.lang.reflect.Method; import java.sql.*; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.Map; public class JdbcMetadata extends BaseTransform { public JdbcMetadata( @@ -93,9 +90,9 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc logDebug("Method has " + argc + " arguments."); String[] arguments = new String[meta.getArguments().size()]; meta.getArguments().toArray(arguments); - logDebug("We expected " + arguments.length + " arguments."); + logDebug("We expected " + arguments.length + " arguments' values."); if (argc != arguments.length) { - throw new Exception("Method has a " + argc + " arguments, we expected " + arguments.length); + throw new Exception("Method has a " + argc + " arguments, we expected " + arguments.length + " values."); } logDebug("Allocating arguments array."); data.arguments = new Object[argc]; @@ -140,80 +137,6 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc } } - /** - * Utility to create a jdbc connection. (Used when the connection source is JDBC or JDBCFields) - * - * @param driver The fully qualified classname identifying the jdbc driver - * @param url The driver-specific url to create the connection - * @param user The database user that wants to establish a connection - * @param password The password required to establish the connection - * @return A jdbc connection object. - * @throws Exception - */ - private Connection createJdbcConnection(String driver, String url, String user, String password) - throws Exception { - Class.forName(driver); - return DriverManager.getConnection(url, user, password); - } - - /** - * Initialize the connection - * - * @param meta - * @param data - * @throws Exception - */ - private void initConnection(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exception { - // Try to establish a connection in advance - logDebug("Try to establish a connection in advance."); - String connectionSource = meta.getConnectionSource(); - logDebug("Connection source: " + connectionSource); - Connection connection; - - if (JdbcMetadataMeta.connectionSourceOptionConnection.equals(connectionSource)) { - // connection is a named kettle connection - DatabaseMeta dbMeta = getPipelineMeta().findDatabase(meta.getConnectionName(), variables); - Database database = new Database(this, this, dbMeta); - database.connect(); - connection = database.getConnection(); - if (connection == null) { - throw new Exception("Connection returned by database object is null!"); - } - data.database = database; - } else if (JdbcMetadataMeta.connectionSourceOptionJDBC.equals(connectionSource)) { - // connection is a user-entered jdbc connection - String jdbcDriver = variables.resolve(meta.getJdbcDriverField()); - String jdbcUrl = variables.resolve(meta.getJdbcUrlField()); - String jdbcUser = variables.resolve(meta.getJdbcUserField()); - String jdbcPassword = variables.resolve(meta.getJdbcPasswordField()); - logDebug("Attempt to create JDBC connection."); - logDebug("Driver: " + jdbcDriver); - logDebug("Url: " + jdbcUrl); - logDebug("User: " + jdbcUser); - connection = createJdbcConnection(jdbcDriver, jdbcUrl, jdbcUser, jdbcPassword); - } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { - // Connection is a jdbc connection specified by field values - // we don't know the field values yet, but we can initialize a few vars to access their value - // later on - connection = null; - data.jdbcDriverField = -1; - data.jdbcUrlField = -1; - data.jdbcUserField = -1; - data.jdbcPasswordField = -1; - } else if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { - // Connection is a named kettle connection specified by a field value - // we don't know about the field value yet, but we can initialize a var to access its value - // later on - connection = null; - data.connectionField = -1; - } else { - // should not arrive here, just initialize the connection to make the compiler shut up. - connection = null; - } - // if the connection is null at this point, then we need to establish it on a row by row basis - data.connection = connection; - } - private void initOutputFields(JdbcMetadataMeta meta, JdbcMetadataData data) { List outputFields = meta.getOutputFields(); int n = outputFields.size(); @@ -242,11 +165,14 @@ public boolean init() { if (!super.init()) return false; try { - data.databases = new HashMap(); - data.connections = new HashMap(); - data.connectionKey = new String[4]; + DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(meta.getConnection(), variables); + if (databaseMeta == null) { + logError("Database connection is missing for transform " + getTransformName()); + return false; + } + data.db = new Database(this, this, databaseMeta); + data.db.connect(); initMethod(meta, data); - initConnection(meta, data); initOutputFields(meta, data); } catch (Exception exception) { logError( @@ -260,84 +186,7 @@ public boolean init() { return result; } - private String getConnectionNameFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.connectionField]); - } - - private String getJdbcDriverFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.jdbcDriverField]); - } - - private String getJdbcUrlFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.jdbcUrlField]); - } - - private String getJdbcUserFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.jdbcUserField]); - } - - private String getJdbcPasswordFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.jdbcPasswordField]); - } - /** - * This is called in the processRow function to obtain the contain to apply the metadata method - * to. - * - * @param meta - * @param data - * @param row - * @return - * @throws Exception - */ - private Connection getConnection(JdbcMetadataMeta meta, JdbcMetadataData data, Object[] row) - throws Exception { - Connection connection; - if (data.connection == null) { - // connection could not be initialized at init, so we have to obtain it now on a row by row - // basis. - String connectionSource = meta.getConnectionSource(); - if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { - // connection is a named kettle connection specified by a field value - String connectionName = getConnectionNameFromRow(data, row); - // try to get this named connection from the cache - Database database = data.databases.get(connectionName); - if (database == null) { - // we haven't seen this named connection before, try to find it. - DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(connectionName, variables); - database = new Database(this, this, databaseMeta); - connection = database.getConnection(); - if (connection == null) { - throw new IllegalArgumentException("Connection returned by database is null!"); - } - // cache the database for later use - data.databases.put(connectionName, database); - } else { - // database found in cache, get its connection - connection = database.getConnection(); - } - } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { - // database connectin is a jdbc connection defined by field values. - // try to find this connection in the cache - String[] key = data.connectionKey; - key[0] = getJdbcDriverFromRow(data, row); - key[1] = getJdbcUrlFromRow(data, row); - key[2] = getJdbcUserFromRow(data, row); - key[3] = getJdbcPasswordFromRow(data, row); - connection = data.connections.get(key); - if (connection == null) { - // connection not yet in the cache. Let's create it and cahce it. - connection = createJdbcConnection(key[0], key[1], key[2], key[3]); - data.connections.put(key, connection); - } - } else { - throw new Exception("Unexpected error acquiring connection"); - } - } else { - connection = data.connection; - } - return connection; - } /** * This is called in the processRow function to get the actual arguments for the jdbc metadata @@ -414,134 +263,8 @@ public boolean processRow() throws HopException { first = false; IRowMeta inputRowMeta = getInputRowMeta(); data.outputRowOffset = inputRowMeta.size(); - String connectionSource = meta.getConnectionSource(); boolean argumentSourceFields = meta.isArgumentSourceFields(); // check if we need to use fields. - // If so, store the indices so we can easily extract their values during transformation - if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource) - || JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource) - || argumentSourceFields) { - logDebug("Looking up indices of input fields."); - String fieldName; - String[] fieldNames = inputRowMeta.getFieldNames(); - logDebug("We have " + fieldNames.length + " input fields."); - List arguments = meta.getArguments(); - int argc = arguments.size(); - String stringArgument; - for (int i = 0; i < fieldNames.length; i++) { - fieldName = fieldNames[i]; - logDebug("Looking at field: " + fieldName); - // store indices for connection source fields - if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { - if (fieldName.equals(meta.getConnectionField())) { - logDebug("Found the connection field at index: " + i); - data.connectionField = i; - } - } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { - if (fieldName.equals(meta.getJdbcDriverField())) { - logDebug("Found the jdbcDriverField field at index: " + i); - data.jdbcDriverField = i; - } - if (fieldName.equals(meta.getJdbcUrlField())) { - logDebug("Found the jdbcUrlField field at index: " + i); - data.jdbcUrlField = i; - } - if (fieldName.equals(meta.getJdbcUserField())) { - logDebug("Found the jdbcUserField field at index: " + i); - data.jdbcUserField = i; - } - if (fieldName.equals(meta.getJdbcPasswordField())) { - logDebug("Found the jdbcPasswordField field at index: " + i); - data.jdbcPasswordField = i; - } - } - // store indices for argument fields - if (argumentSourceFields) { - logDebug("Trying to match argument fields against: " + fieldName); - for (int j = 0; j < argc; j++) { - stringArgument = arguments.get(j); - logDebug("Found argument " + j + ": " + stringArgument); - if (fieldName.equals(stringArgument)) { - logDebug("Match, storing index " + i); - data.argumentFieldIndices[j] = i; - } - } - } - } // end fields loop - - // ensure that we have all required fields. - if ((JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource) - && (data.jdbcDriverField == -1 - || data.jdbcUrlField == -1 - || data.jdbcUserField == -1 - || data.jdbcPasswordField == -1)) - || (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource) - && data.connectionField == -1)) { - throw new HopException("Not all fields for the connection source were found."); - } - - if (argumentSourceFields) { - // keep track of how many fields we used as args. - // We need this in case remove argument fields is enabled - // as this is the number of fields we need to discard from the input row. - int fieldsUsedAsArgs = 0; - // ensure all argument fields are bound to a valid field - argumentFields: - for (int j = 0; j < argc; j++) { - logDebug("Argument indices at " + j + ": " + data.argumentFieldIndices[j]); - if (data.argumentFieldIndices[j] == -1) { - // this argument does not point to any existing field. - if (arguments.get(j) == null || arguments.get(j).length() == 0) { - // the argument is blank, this is ok: we will pass null instead - data.argumentFieldIndices[j] = -2; - } else { - // this argument is not blank - this means it points to a non-existing field. - // this is an error. - Object[] descriptor = meta.getMethodDescriptor(); - Object[] args = (Object[]) descriptor[1]; - Object[] arg = (Object[]) args[j]; - throw new HopException( - "No field \"" - + arguments.get(j) - + "\" found for argument " - + j - + ": " - + arg[0]); - } - } else { - // this argument points to a valid field. - // let's check if this same field was already used as arg: - for (int i = 0; i < j; i++) { - if (data.argumentFieldIndices[i] == data.argumentFieldIndices[j]) { - // yes, it was used already. Let's check the next argument. - continue argumentFields; - } - } - // this field was not used already, so mark it as used. - fieldsUsedAsArgs++; - } - } - - if (meta.isRemoveArgumentFields()) { - int n = data.outputRowOffset; - data.outputRowOffset -= fieldsUsedAsArgs; - data.inputFieldsToCopy = new int[data.outputRowOffset]; - - inputFieldsToCopy: - for (int i = 0, j = 0; i < n; i++) { // for each field in the input row - for (int k = 0; k < argc; k++) { // for each method argument - if (data.argumentFieldIndices[k] == i) { - // this input field is used as argument. Continue to the next field. - continue inputFieldsToCopy; - } - } - // this field was not used as argument. make sure we copy it to the output. - data.inputFieldsToCopy[j++] = i; - } - } - } - logDebug("Done looking up indices of input fields."); - } // clone the input row structure and place it in our data object data.outputRowMeta = inputRowMeta.clone(); @@ -551,7 +274,6 @@ public boolean processRow() throws HopException { try { logRowlevel("Processing 1 input row"); - Connection connection = getConnection(meta, data, r); prepareMethodArguments(meta, data, r); if (getLogLevel() == LogLevel.ROWLEVEL) { logRowlevel("About to invoke method"); @@ -568,7 +290,7 @@ public boolean processRow() throws HopException { } } - DatabaseMetaData databaseMetaData = connection.getMetaData(); + DatabaseMetaData databaseMetaData = data.db.getConnection().getMetaData(); ResultSet resultSet = (ResultSet) data.method.invoke(databaseMetaData, data.arguments); ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); int columnCount = resultSetMetaData.getColumnCount(); @@ -666,48 +388,11 @@ public void dispose() { logError("Error cleaning up connection: " + ex.getMessage()); } data.connection = null; - - // clean up the database map - Iterator> dbIterator = data.databases.entrySet().iterator(); - while (dbIterator.hasNext()) { - Map.Entry current = dbIterator.next(); - Database database = current.getValue(); - try { - database.disconnect(); - } catch (Exception ex) { - logError("Error cleaning up database " + current.getKey() + ": " + ex.getMessage()); - } - } - data.databases.clear(); - data.databases = null; - - // clean up the connection map - Iterator> connectionIterator = - data.connections.entrySet().iterator(); - while (connectionIterator.hasNext()) { - Connection connection = connectionIterator.next().getValue(); - try { - if (!connection.isClosed()) { - connection.close(); - } - } catch (Exception ex) { - logError("Error cleaning up connection: " + ex.getMessage()); - } - } - data.connections.clear(); - data.connections = null; - data.arguments = null; data.method = null; data.argumentFieldIndices = null; data.inputFieldsToCopy = null; data.resultSetIndices = null; - data.connectionKey = null; - - data.jdbcDriverField = -1; - data.jdbcUrlField = -1; - data.jdbcUserField = -1; - data.jdbcPasswordField = -1; super.dispose(); } diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java index 423d7a5293e..5fd0e09fc16 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java @@ -24,7 +24,6 @@ import java.lang.reflect.Method; import java.sql.Connection; -import java.util.Map; public class JdbcMetadataData extends BaseTransformData implements ITransformData { @@ -37,21 +36,9 @@ public class JdbcMetadataData extends BaseTransformData implements ITransformDat public Method method; // use to store the arguments to the method public Object[] arguments; - // named kettle connection cache. Used when connection is a named connection specified by input - // fields - public Map databases = null; - // connection cache. Used when connection is a jdbc connection specified by input fields - public Map connections = null; + + public Database db; // key to the connection cache. - public String[] connectionKey = null; - // field index for named kettle connection - public int connectionField = -1; - // field indices for jdbc connections - public int jdbcDriverField = -1; - public int jdbcUrlField = -1; - public int jdbcUserField = -1; - public int jdbcPasswordField = -1; - // indices for fields used to specify method arguments public int[] argumentFieldIndices; // the offset in the output row from where we can add our metadata fields. // (we need this in case we're required to remove arguments fields from the input) diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java index 5598d696c86..f88eb2438f1 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java @@ -36,10 +36,12 @@ import org.apache.hop.ui.core.widget.TableView; import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; import org.eclipse.swt.SWT; -import org.eclipse.swt.custom.CCombo; import org.eclipse.swt.custom.CTabFolder; import org.eclipse.swt.custom.CTabItem; -import org.eclipse.swt.events.*; +import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.events.SelectionListener; import org.eclipse.swt.layout.FormAttachment; import org.eclipse.swt.layout.FormData; import org.eclipse.swt.layout.FormLayout; @@ -59,19 +61,8 @@ public class JdbcMetadataDialog extends BaseTransformDialog implements ITransfor private ModifyListener lsMod; private Composite metadataComposite; - // - private MetaSelectionLine - wConnectionSource; // text field holding the name of the field containing the connection name - private CCombo connectionField; - // text field holding the name of the field containing the driver name - private ComboVar jdbcDriverField; - // text field holding the name of the field containing the url - private ComboVar jdbcUrlField; - // text field holding the name of the field containing the user - private ComboVar jdbcUserField; - // text field holding the name of the field containing the password - private ComboVar jdbcPasswordField; - // + + private MetaSelectionLine wConnection; private Label alwaysPassInputRowLabel; // @@ -111,37 +102,6 @@ private String[] getFieldListForCombo() { return items; } - private void connectionSourceUpdated() { - int selectedIndex = wConnectionSource.getSelectionIndex(); - String option = JdbcMetadataMeta.connectionSourceOptions[selectedIndex]; - boolean connectionComboEnabled, connectionFieldEnabled, otherFieldsEnabled; - otherFieldsEnabled = connectionComboEnabled = connectionFieldEnabled = false; - String[] fields = emptyFieldList; - if (JdbcMetadataMeta.connectionSourceOptionConnection.equals(option)) { - connectionComboEnabled = true; - } else { - if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(option)) { - connectionFieldEnabled = true; - connectionField.setItems(getFieldListForCombo()); - } else { - otherFieldsEnabled = true; - if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(option)) { - fields = getFieldListForCombo(); - } - } - } - wConnectionSource.setEnabled(connectionComboEnabled); - connectionField.setEnabled(connectionFieldEnabled); - jdbcDriverField.setEnabled(otherFieldsEnabled); - jdbcDriverField.setItems(fields); - jdbcUrlField.setEnabled(otherFieldsEnabled); - jdbcUrlField.setItems(fields); - jdbcUserField.setEnabled(otherFieldsEnabled); - jdbcUserField.setItems(fields); - jdbcPasswordField.setEnabled(otherFieldsEnabled); - jdbcPasswordField.setItems(fields); - } - /** Remove the UI to enter method arguments The current values are stored and returned. */ private List removeArgumentsUI() { Control[] controls = metadataComposite.getChildren(); @@ -303,6 +263,7 @@ private void methodUpdated(List argumentValues) { logDebug( "methodUpdated, argumentValues = " + (argumentValues == null ? "null" : argumentValues.size())); + // first, remove the controls for the previous set of arguments List currentValues = removeArgumentsUI(); logDebug("currentValue = " + currentValues.size()); @@ -319,7 +280,6 @@ private void methodUpdated(List argumentValues) { List newArguments = createArgumentsUI(argumentDescriptors, argumentValues); // update the arguments in the meta object input.setArguments(newArguments); - metadataComposite.layout(); } @@ -380,6 +340,13 @@ public void widgetSelected(SelectionEvent e) { lastControl = wTransformName; + // Connection line + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(input.getConnection(), variables); + wConnection = addConnectionLine(shell, lastControl, databaseMeta, lsMod); + wConnection.addSelectionListener(lsSelection); + + lastControl = wConnection; + // pass the row checkbox alwaysPassInputRowLabel = new Label(shell, SWT.RIGHT); alwaysPassInputRowLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.passRow.Label")); @@ -406,165 +373,14 @@ public void widgetSelected(SelectionEvent e) { CTabFolder cTabFolder = new CTabFolder(shell, SWT.BORDER); props.setLook(cTabFolder, Props.WIDGET_STYLE_TAB); - // Connection tab - CTabItem connectionTab = new CTabItem(cTabFolder, SWT.NONE); - connectionTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.ConnectionTab.Label")); - connectionTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.ConnectionTab.Tooltip")); - - FormLayout connectionTabLayout = new FormLayout(); - connectionTabLayout.marginWidth = PropsUi.getFormMargin(); - connectionTabLayout.marginHeight = PropsUi.getFormMargin(); - - Composite connectionComposite = new Composite(cTabFolder, SWT.NONE); - props.setLook(connectionComposite); - connectionComposite.setLayout(connectionTabLayout); - - // Connection line - DatabaseMeta databaseMeta = pipelineMeta.findDatabase(input.getConnectionName(), variables); - wConnectionSource = - addConnectionLine( - connectionComposite, - wTransformName, - databaseMeta, - lsMod, - BaseMessages.getString(PKG, "JdbcMetadata.connectionSource.Label"), - BaseMessages.getString(PKG, "JdbcMetadata.connectionSource.Tooltip")); - wConnectionSource.addSelectionListener(lsSelection); - lastControl = wConnectionSource; - - // connection name field - Label connectionFieldLabel = new Label(connectionComposite, SWT.RIGHT); - connectionFieldLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.connectionField.Label")); - connectionFieldLabel.setToolTipText( - BaseMessages.getString(PKG, "JdbcMetadata.connectionField.Tooltip")); - props.setLook(connectionFieldLabel); - FormData connectionFieldLabelFormData = new FormData(); - connectionFieldLabelFormData.left = new FormAttachment(0, 0); - connectionFieldLabelFormData.right = new FormAttachment(middle, -margin); - connectionFieldLabelFormData.top = new FormAttachment(lastControl, margin); - connectionFieldLabel.setLayoutData(connectionFieldLabelFormData); - - connectionField = new CCombo(connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(connectionField); - connectionField.addModifyListener(lsMod); - FormData connectionFieldFormData = new FormData(); - connectionFieldFormData.left = new FormAttachment(middle, 0); - connectionFieldFormData.right = new FormAttachment(100, 0); - connectionFieldFormData.top = new FormAttachment(lastControl, margin); - connectionField.setLayoutData(connectionFieldFormData); - - lastControl = connectionField; - - // jdbc driver field - Label jdbcDriverLabel = new Label(connectionComposite, SWT.RIGHT); - jdbcDriverLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.driverField.Label")); - jdbcDriverLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.driverField.Tooltip")); - props.setLook(jdbcDriverLabel); - FormData jdbcDriverLabelFormData = new FormData(); - jdbcDriverLabelFormData.left = new FormAttachment(0, 0); - jdbcDriverLabelFormData.right = new FormAttachment(middle, -margin); - jdbcDriverLabelFormData.top = new FormAttachment(lastControl, margin); - jdbcDriverLabel.setLayoutData(jdbcDriverLabelFormData); - - jdbcDriverField = - new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(jdbcDriverField); - jdbcDriverField.addModifyListener(lsMod); - FormData jdbcDriverFieldFormData = new FormData(); - jdbcDriverFieldFormData.left = new FormAttachment(middle, 0); - jdbcDriverFieldFormData.right = new FormAttachment(100, 0); - jdbcDriverFieldFormData.top = new FormAttachment(lastControl, margin); - jdbcDriverField.setLayoutData(jdbcDriverFieldFormData); - - lastControl = jdbcDriverField; - - // jdbc url field - Label jdbcUrlLabel = new Label(connectionComposite, SWT.RIGHT); - jdbcUrlLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.urlField.Label")); - jdbcUrlLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.urlField.Tooltip")); - props.setLook(jdbcUrlLabel); - FormData jdbcUrlLabelFormData = new FormData(); - jdbcUrlLabelFormData.left = new FormAttachment(0, 0); - jdbcUrlLabelFormData.right = new FormAttachment(middle, -margin); - jdbcUrlLabelFormData.top = new FormAttachment(lastControl, margin); - jdbcUrlLabel.setLayoutData(jdbcUrlLabelFormData); - - jdbcUrlField = new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(jdbcUrlField); - jdbcUrlField.addModifyListener(lsMod); - FormData jdbcUrlFieldFormData = new FormData(); - jdbcUrlFieldFormData.left = new FormAttachment(middle, 0); - jdbcUrlFieldFormData.right = new FormAttachment(100, 0); - jdbcUrlFieldFormData.top = new FormAttachment(lastControl, margin); - jdbcUrlField.setLayoutData(jdbcUrlFieldFormData); - - lastControl = jdbcUrlField; - - // jdbc user field - Label jdbcUserLabel = new Label(connectionComposite, SWT.RIGHT); - jdbcUserLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.userField.Label")); - jdbcUserLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.userField.Tooltip")); - props.setLook(jdbcUserLabel); - FormData jdbcUserLabelFormData = new FormData(); - jdbcUserLabelFormData.left = new FormAttachment(0, 0); - jdbcUserLabelFormData.right = new FormAttachment(middle, -margin); - jdbcUserLabelFormData.top = new FormAttachment(lastControl, margin); - jdbcUserLabel.setLayoutData(jdbcUserLabelFormData); - - jdbcUserField = - new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(jdbcUserField); - jdbcUserField.addModifyListener(lsMod); - FormData jdbcUserFieldFormData = new FormData(); - jdbcUserFieldFormData.left = new FormAttachment(middle, 0); - jdbcUserFieldFormData.right = new FormAttachment(100, 0); - jdbcUserFieldFormData.top = new FormAttachment(lastControl, margin); - jdbcUserField.setLayoutData(jdbcUserFieldFormData); - - lastControl = jdbcUserField; - - // jdbc password field - Label jdbcPasswordLabel = new Label(connectionComposite, SWT.RIGHT); - jdbcPasswordLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.passwordField.Label")); - jdbcPasswordLabel.setToolTipText( - BaseMessages.getString(PKG, "JdbcMetadata.passwordField.Tooltip")); - props.setLook(jdbcPasswordLabel); - FormData jdbcPasswordLabelFormData = new FormData(); - jdbcPasswordLabelFormData.left = new FormAttachment(0, 0); - jdbcPasswordLabelFormData.right = new FormAttachment(middle, -margin); - jdbcPasswordLabelFormData.top = new FormAttachment(lastControl, margin); - jdbcPasswordLabel.setLayoutData(jdbcPasswordLabelFormData); - - jdbcPasswordField = - new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(jdbcPasswordField); - jdbcPasswordField.addModifyListener(lsMod); - FormData jdbcPasswordFieldFormData = new FormData(); - jdbcPasswordFieldFormData.left = new FormAttachment(middle, 0); - jdbcPasswordFieldFormData.right = new FormAttachment(100, 0); - jdbcPasswordFieldFormData.top = new FormAttachment(lastControl, margin); - jdbcPasswordField.setLayoutData(jdbcPasswordFieldFormData); - - lastControl = jdbcPasswordField; - - // layout the connection tab - FormData connectionTabFormData = new FormData(); - connectionTabFormData.left = new FormAttachment(0, 0); - connectionTabFormData.top = new FormAttachment(0, 0); - connectionTabFormData.right = new FormAttachment(100, 0); - connectionTabFormData.bottom = new FormAttachment(100, 0); - connectionComposite.setLayoutData(connectionTabFormData); - connectionComposite.layout(); - connectionTab.setControl(connectionComposite); - // Metadata tab CTabItem metadataTab = new CTabItem(cTabFolder, SWT.NONE); metadataTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.MetaDataTab.Label")); metadataTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.MetaDataTab.Tooltip")); FormLayout metadataTabLayout = new FormLayout(); - connectionTabLayout.marginWidth = Const.FORM_MARGIN; - connectionTabLayout.marginHeight = Const.FORM_MARGIN; + metadataTabLayout.marginWidth = Const.FORM_MARGIN; + metadataTabLayout.marginHeight = Const.FORM_MARGIN; metadataComposite = new Composite(cTabFolder, SWT.NONE); props.setLook(metadataComposite); @@ -646,15 +462,10 @@ public void widgetSelected(SelectionEvent selectionEvent) { removeArgumentFieldsButton.setEnabled(selection); String[] items = selection ? getFieldListForCombo() : emptyFieldList; - if (!selection) { - removeArgumentsUI(); - } else { - methodUpdated(null); - for (Control control : controls) { - if (!(control instanceof ComboVar)) continue; - ComboVar comboVar = (ComboVar) control; - comboVar.setItems(items); - } + for (Control control : controls) { + if (!(control instanceof ComboVar)) continue; + ComboVar comboVar = (ComboVar) control; + comboVar.setItems(items); } input.setChanged(); @@ -795,23 +606,6 @@ public void widgetSelected(SelectionEvent arg0) { wOk.addListener(SWT.Selection, e -> ok()); wCancel.addListener(SWT.Selection, e -> cancel()); - // default listener (for hitting "enter") - // lsDef = - // new SelectionAdapter() { - // public void widgetDefaultSelected(SelectionEvent e) { - // ok(); - // } - // }; - // wTransformName.addSelectionListener(lsDef); - // jdbcDriverField.addSelectionListener(lsDef); - - // Detect X or ALT-F4 or something that kills this window and cancel the dialog properly - shell.addShellListener( - new ShellAdapter() { - public void shellClosed(ShellEvent e) { - cancel(); - } - }); setSize(); populateDialog(); @@ -822,12 +616,6 @@ public void shellClosed(ShellEvent e) { return transformName; } - private void selectConnectionSource(String connectionSourceOption) { - int index = JdbcMetadataMeta.getConnectionSourceOptionIndex(connectionSourceOption); - wConnectionSource.select(index); - connectionSourceUpdated(); - } - private void setMethod(String method) { int index = JdbcMetadataMeta.getMethodDescriptorIndex(method); if (index == -1) throw new IllegalArgumentException("Index for method " + method + " is -1."); @@ -843,40 +631,20 @@ private void populateDialog() { wTransformName.selectAll(); String value; - value = input.getConnectionSource(); - selectConnectionSource(value); - - value = input.getConnectionName(); - if (value != null) wConnectionSource.setText(value); - - value = input.getConnectionField(); - if (value != null) connectionField.setText(value); - - value = input.getJdbcDriverField(); - if (value != null) jdbcDriverField.setText(value); - - value = input.getJdbcUrlField(); - if (value != null) jdbcUrlField.setText(value); - - value = input.getJdbcUserField(); - if (value != null) jdbcUserField.setText(value); - - value = input.getJdbcPasswordField(); - if (value != null) jdbcPasswordField.setText(value); + value = input.getConnection(); + if (value != null) wConnection.setText(value); alwaysPassInputRowButton.setSelection(input.isAlwaysPassInputRow()); value = input.getMethodName(); if (value != null) setMethod(value); - if (input.isArgumentSourceFields()) { - argumentSourceFields.setSelection(input.isArgumentSourceFields()); - if (input.getArguments() != null && input.getArguments().size() > 0) { - methodUpdated(input.getArguments()); - } else { - // If no arguments' values are saved build the arguments' list without assigning any values - methodUpdated(new ArrayList<>()); - } + argumentSourceFields.setSelection(input.isArgumentSourceFields()); + if (input.getArguments() != null && input.getArguments().size() > 0) { + methodUpdated(input.getArguments()); + } else { + // If no arguments' values are saved build the arguments' list without assigning any values + methodUpdated(); } logDebug("Calling methodUpdated from populate dialog."); @@ -900,14 +668,7 @@ private void ok() { transformName = wTransformName.getText(); // Save settings to the meta object - input.setConnectionSource( - JdbcMetadataMeta.connectionSourceOptions[wConnectionSource.getSelectionIndex()]); - input.setConnectionName(wConnectionSource.getText()); - input.setConnectionField(connectionField.getText()); - input.setJdbcDriverField(jdbcDriverField.getText()); - input.setJdbcUrlField(jdbcUrlField.getText()); - input.setJdbcUserField(jdbcUserField.getText()); - input.setJdbcPasswordField(jdbcPasswordField.getText()); + input.setConnection(wConnection.getText()); input.setAlwaysPassInputRow(alwaysPassInputRowButton.getSelection()); input.setMethodName(JdbcMetadataMeta.getMethodName(methodCombo.getSelectionIndex())); input.setArgumentSourceFields(argumentSourceFields.getSelection()); diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java index b9a92ec7cff..efabbfd6db3 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java @@ -365,24 +365,6 @@ public class JdbcMetadataMeta extends BaseTransformMeta PKG = JdbcMetadataMeta.class; // for i18n purposes - public static final String connectionSourceOptionConnection = "Connection"; - public static final String connectionSourceOptionConnectionField = "ConnectionField"; - public static final String connectionSourceOptionJDBC = "JDBC"; - public static final String connectionSourceOptionJDBCFields = "JDBCFields"; - - public static final String[] connectionSourceOptions = - new String[] { - connectionSourceOptionConnection, - connectionSourceOptionConnectionField, - connectionSourceOptionJDBC, - connectionSourceOptionJDBCFields - }; - /** * Constructor should call super() to make sure the base class has a chance to initialize * properly. @@ -421,171 +390,21 @@ public JdbcMetadataMeta() { } public void setDefault() { - connectionSource = "Connection"; - connectionName = ""; - jdbcDriverField = ""; - jdbcUrlField = ""; - jdbcUserField = ""; - jdbcPasswordField = ""; methodName = "getCatalogs"; argumentSourceFields = false; outputFields = new ArrayList<>(); outputFields.add(new OutputField("TABLE_CAT","TABLE_CAT")); } - /** Stores the connectionsource. */ - @HopMetadataProperty - private String connectionSource; - /** - * Get the index of the connection source option - * - * @param connectionSourceOption - * @return - */ - public static int getConnectionSourceOptionIndex(String connectionSourceOption) { - for (int i = 0; i < connectionSourceOptions.length; i++) { - if (connectionSourceOptions[i].equals(connectionSourceOption)) return i; - } - return -1; - } - /** - * Getter for the name of the field containing the connection source - * - * @return the source of the connection data - */ - public String getConnectionSource() { - return connectionSource; - } - /** - * Setter for the name of the field containing the connection source - * - * @param connectionSource the source for the connection data - */ - public void setConnectionSource(String connectionSource) { - if (connectionSource == null) connectionSource = connectionSourceOptions[0]; - else if (getConnectionSourceOptionIndex(connectionSource) == -1) - throw new IllegalArgumentException( - connectionSource + " is not a valid value for connectionSource."); - this.connectionSource = connectionSource; - } - - /** Stores the name of connection. */ - @HopMetadataProperty - private String connectionName; - /** - * Getter for the name of the connection - * - * @return the name of the connection - */ - public String getConnectionName() { - return connectionName; - } - /** - * Setter for the name of the connection - * - * @param connectionName the name of the connection - */ - public void setConnectionName(String connectionName) { - this.connectionName = connectionName; - } - - /** Stores the name of field holding the connection name. */ - @HopMetadataProperty - private String connectionField; - /** - * Getter for the name of the field holding the connection name - * - * @return the name of the field holding the connection name - */ - public String getConnectionField() { - return connectionField; - } - /** - * Setter for the name of the field holding the name of the connection - * - * @param connectionField the name of the field holding the connection name - */ - public void setConnectionField(String connectionField) { - this.connectionField = connectionField; - } - /** Stores the name of the field containing the name of the jdbc driver. */ - @HopMetadataProperty - private String jdbcDriverField; - /** - * Getter for the name of the field containing the jdbc driver - * - * @return the name of the field containing the jdbc driver - */ - public String getJdbcDriverField() { - return jdbcDriverField; - } - /** - * Setter for the name of the field containing the jdbc driver - * - * @param jdbcDriverField the name of the field containing the jdbc driver - */ - public void setJdbcDriverField(String jdbcDriverField) { - this.jdbcDriverField = jdbcDriverField; - } - - /** Stores the name of the field containing the url for the jdbc connection. */ @HopMetadataProperty - private String jdbcUrlField; - /** - * Getter for the name of the field containing the jdbc url - * - * @return the name of the field containing the jdbc url - */ - public String getJdbcUrlField() { - return jdbcUrlField; - } - /** - * Setter for the name of the field containing the jdbc url - * - * @param jdbcUrlField the name of the field containing the jdbc url - */ - public void setJdbcUrlField(String jdbcUrlField) { - this.jdbcUrlField = jdbcUrlField; - } + private String connection; - /** Stores the name of the field containing the username for the jdbc connection. */ - @HopMetadataProperty - private String jdbcUserField; - /** - * Getter for the name of the field containing the jdbc user - * - * @return the name of the field containing the jdbc user - */ - public String getJdbcUserField() { - return jdbcUserField; - } - /** - * Setter for the name of the field containing the jdbc user - * - * @param jdbcUserField the name of the field containing the jdbc user - */ - public void setJdbcUserField(String jdbcUserField) { - this.jdbcUserField = jdbcUserField; + public String getConnection() { + return connection; } - /** Stores the name of the field containing the password for the jdbc connection. */ - @HopMetadataProperty - private String jdbcPasswordField; - /** - * Getter for the name of the field containing the jdbc password - * - * @return the name of the field containing the jdbc password - */ - public String getJdbcPasswordField() { - return jdbcPasswordField; - } - /** - * Setter for the name of the field containing the jdbc password - * - * @param jdbcPasswordField the name of the field containing the jdbc password - */ - public void setJdbcPasswordField(String jdbcPasswordField) { - this.jdbcPasswordField = jdbcPasswordField; + public void setConnection(String connection) { + this.connection = connection; } /** Stores the whether the input row should be returned even if no metadata was found */ diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties index b4bd0b07bf7..bfdfef7c57a 100644 --- a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties +++ b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties @@ -18,39 +18,22 @@ JdbcMetadata.Name=Get JDBC Metadata JdbcMetadata.Name.Description=Get JDBC Metadata -JdbcMetadata.connectionSource.Label=Connection Source -JdbcMetadata.connectionSource.Tooltip=Specifies how to establish a database connection -JdbcMetadata.connectionSource.options.Connection=Hop Connection -JdbcMetadata.connectionSource.options.ConnectionField=Hop connection defined by field -JdbcMetadata.connectionSource.options.JDBC=JDBC connection -JdbcMetadata.connectionSource.options.JDBCFields=JDBC connection defined by fields -JdbcMetadata.ConnectionTab.Label=Connection -JdbcMetadata.ConnectionTab.Tooltip=Use this tab to specify how a database connection will be acquired -JdbcMetadata.MetaDataTab.Label=Method and Arguments +JdbcMetadata.MetaDataTab.Label=Method and arguments JdbcMetadata.MetaDataTab.Tooltip=Use this tab to specify the kind of metadata and any required parameters -JdbcMetadata.FieldsTab.Label=Output Fields +JdbcMetadata.FieldsTab.Label=Output fields JdbcMetadata.FieldsTab.Tooltip=Use this tab to define which metadata fields you want to add to the outputstream -JdbcMetadata.connectionField.Label=Connection field -JdbcMetadata.connectionField.Tooltip=The name of the field from the input stream that specifies a named Hop connection -JdbcMetadata.driverField.Label=Driver class -JdbcMetadata.driverField.Tooltip=The fully qualified classname of the jdbc driver -JdbcMetadata.urlField.Label=URL -JdbcMetadata.urlField.Tooltip=The driver-specific URL to establish a database connection -JdbcMetadata.userField.Label=Username -JdbcMetadata.userField.Tooltip=The database user that establishes the connection -JdbcMetadata.passwordField.Label=Password -JdbcMetadata.passwordField.Tooltip=The password of the database user that establishes the connection + JdbcMetadata.passRow.Label=Always pass the input row? JdbcMetadata.passRow.Tooltip=Check this if you want to pass the input row even if no metadata was found. (Metadata output fields will be null in this case) -JdbcMetadata.metadataMethod.Label=MetaData Method +JdbcMetadata.metadataMethod.Label=Metadata method JdbcMetadata.metadataMethod.Tooltip=This specifies the kind of metadata to retrieve. -JdbcMetadata.argumentSource.Label=Get Arguments from fields? +JdbcMetadata.argumentSource.Label=Get arguments from fields? JdbcMetadata.argumentSource.Tooltip=Check this if you want to acquire the values for the method arguments from the fields of the input stream JdbcMetadata.removeArgumentFields.Label=Remove argument fields? JdbcMetadata.removeArgumentFields.Tooltip=Check this if you do not want to pass on the method argument fields to the output stream -JdbcMetadata.getFieldsButton.Label=Get Fields +JdbcMetadata.getFieldsButton.Label=Get fields JdbcMetadata.getFieldsButton.Tooltip=Click to retrieve all possible output fields for the current metadata method JdbcMetadata.FieldName.Label=Field name JdbcMetadata.FieldName.Tooltip=The name of the output field as it is returned by the metadata method diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties index b4bd0b07bf7..72e2b4b0d54 100644 --- a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties +++ b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties @@ -18,44 +18,26 @@ JdbcMetadata.Name=Get JDBC Metadata JdbcMetadata.Name.Description=Get JDBC Metadata -JdbcMetadata.connectionSource.Label=Connection Source -JdbcMetadata.connectionSource.Tooltip=Specifies how to establish a database connection -JdbcMetadata.connectionSource.options.Connection=Hop Connection -JdbcMetadata.connectionSource.options.ConnectionField=Hop connection defined by field -JdbcMetadata.connectionSource.options.JDBC=JDBC connection -JdbcMetadata.connectionSource.options.JDBCFields=JDBC connection defined by fields +JdbcMetadata.MetaDataTab.Label=Metodi ed argomenti +JdbcMetadata.MetaDataTab.Tooltip=Usa questo tab per specificare il tipo dei metadati ed inserire i parametri richiesti +JdbcMetadata.FieldsTab.Label=Campi di output +JdbcMetadata.FieldsTab.Tooltip=Usa questo tab per definire quali campi dei metadati di output vuoi aggiungere allo stream di uscita -JdbcMetadata.ConnectionTab.Label=Connection -JdbcMetadata.ConnectionTab.Tooltip=Use this tab to specify how a database connection will be acquired -JdbcMetadata.MetaDataTab.Label=Method and Arguments -JdbcMetadata.MetaDataTab.Tooltip=Use this tab to specify the kind of metadata and any required parameters -JdbcMetadata.FieldsTab.Label=Output Fields -JdbcMetadata.FieldsTab.Tooltip=Use this tab to define which metadata fields you want to add to the outputstream -JdbcMetadata.connectionField.Label=Connection field -JdbcMetadata.connectionField.Tooltip=The name of the field from the input stream that specifies a named Hop connection -JdbcMetadata.driverField.Label=Driver class -JdbcMetadata.driverField.Tooltip=The fully qualified classname of the jdbc driver -JdbcMetadata.urlField.Label=URL -JdbcMetadata.urlField.Tooltip=The driver-specific URL to establish a database connection -JdbcMetadata.userField.Label=Username -JdbcMetadata.userField.Tooltip=The database user that establishes the connection -JdbcMetadata.passwordField.Label=Password -JdbcMetadata.passwordField.Tooltip=The password of the database user that establishes the connection -JdbcMetadata.passRow.Label=Always pass the input row? -JdbcMetadata.passRow.Tooltip=Check this if you want to pass the input row even if no metadata was found. (Metadata output fields will be null in this case) -JdbcMetadata.metadataMethod.Label=MetaData Method -JdbcMetadata.metadataMethod.Tooltip=This specifies the kind of metadata to retrieve. -JdbcMetadata.argumentSource.Label=Get Arguments from fields? -JdbcMetadata.argumentSource.Tooltip=Check this if you want to acquire the values for the method arguments from the fields of the input stream -JdbcMetadata.removeArgumentFields.Label=Remove argument fields? -JdbcMetadata.removeArgumentFields.Tooltip=Check this if you do not want to pass on the method argument fields to the output stream -JdbcMetadata.getFieldsButton.Label=Get Fields -JdbcMetadata.getFieldsButton.Tooltip=Click to retrieve all possible output fields for the current metadata method -JdbcMetadata.FieldName.Label=Field name -JdbcMetadata.FieldName.Tooltip=The name of the output field as it is returned by the metadata method -JdbcMetadata.OutputFieldName.Label=Rename to -JdbcMetadata.OutputFieldName.Tooltip=The custom name for this output field +JdbcMetadata.passRow.Label=Fai sempre passare una riga? +JdbcMetadata.passRow.Tooltip=Seleziona l''opzione se vuoi fare passare sempre le righe di input anche se non ci sono metadati in output (i campi in output dovuti ai metadati saranno tutti null in questo caso) +JdbcMetadata.metadataMethod.Label=Metodo da richiamare +JdbcMetadata.metadataMethod.Tooltip=Specifica il tipo di metadati da recuperare. +JdbcMetadata.argumentSource.Label=Valore degli argomenti dai campi? +JdbcMetadata.argumentSource.Tooltip=Seleziona l''opzione se vuoi acquisire i valori per gli argomenti dei metodi da campi dello stream di input +JdbcMetadata.removeArgumentFields.Label=Rimuove i campi degli argomenti? +JdbcMetadata.removeArgumentFields.Tooltip=Seleziona l''opzione se non vuoi passare gli argomenti della chiamata al metodo nello stream di output +JdbcMetadata.getFieldsButton.Label=Estrai campi +JdbcMetadata.getFieldsButton.Tooltip=Premi per estrarre tutti i possibili campi di output per il metodo di DatabaseMetadata selezionato +JdbcMetadata.FieldName.Label=Nome campo +JdbcMetadata.FieldName.Tooltip=Nome del campo di output come viene restituito dal metodo di DatabaseMetadata +JdbcMetadata.OutputFieldName.Label=Rinomina a +JdbcMetadata.OutputFieldName.Tooltip=Nome custom il campo di output JdbcMetadata.methods.getBestRowIdentifier=Best row identifier JdbcMetadata.methods.getCatalogs=Catalogs @@ -72,35 +54,35 @@ JdbcMetadata.methods.getTables=Tables JdbcMetadata.methods.getTypeInfo=Data types JdbcMetadata.methods.getVersionColumns=Version columns -JdbcMetadata.arguments.catalog.Label=Catalog +JdbcMetadata.arguments.catalog.Label=Catalogo JdbcMetadata.arguments.catalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database -JdbcMetadata.arguments.parentCatalog.Label=Parent catalog +JdbcMetadata.arguments.parentCatalog.Label=Catalogo parent JdbcMetadata.arguments.parentCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database -JdbcMetadata.arguments.foreignCatalog.Label=Foreign catalog +JdbcMetadata.arguments.foreignCatalog.Label=Catalogo foreign JdbcMetadata.arguments.foreignCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database -JdbcMetadata.arguments.columnNamePattern.Label=Column name pattern +JdbcMetadata.arguments.columnNamePattern.Label=Colonna (pattern) JdbcMetadata.arguments.columnNamePattern.Tooltip=A column name pattern; must match the column name as it is stored in the database JdbcMetadata.arguments.nullable.Label=Nullable? JdbcMetadata.arguments.nullable.Tooltip=Include columns that are nullable JdbcMetadata.arguments.schema.Label=Schema JdbcMetadata.arguments.schema.Tooltip=A schema name; must match the schema name as it is stored in the database -JdbcMetadata.arguments.parentSchema.Label=Parent schema +JdbcMetadata.arguments.parentSchema.Label=Schema parent JdbcMetadata.arguments.parentSchema.Tooltip=A schema name; must match the schema name as it is stored in the database -JdbcMetadata.arguments.foreignSchema.Label=Foreign schema +JdbcMetadata.arguments.foreignSchema.Label=Schema foreign JdbcMetadata.arguments.foreignSchema.Tooltip=A schema name; must match the schema name as it is stored in the database -JdbcMetadata.arguments.schemaPattern.Label=Schema pattern +JdbcMetadata.arguments.schemaPattern.Label=Schema (pattern) JdbcMetadata.arguments.schemaPattern.Tooltip=A schema pattern name; must match the schema name as it is stored in the database; JdbcMetadata.arguments.scope.Label=Identity scope JdbcMetadata.arguments.scope.Tooltip=The scope of interest -JdbcMetadata.arguments.table.Label=Table +JdbcMetadata.arguments.table.Label=Tabella JdbcMetadata.arguments.table.Tooltip=A table name; must match the table name as it is stored in the database -JdbcMetadata.arguments.parentTable.Label=Parent table +JdbcMetadata.arguments.parentTable.Label=Tabella padre JdbcMetadata.arguments.parentTable.Tooltip=A table name; must match the table name as it is stored in the database -JdbcMetadata.arguments.foreignTable.Label=Foreign table +JdbcMetadata.arguments.foreignTable.Label=Tabella foreign JdbcMetadata.arguments.foreignTable.Tooltip=A table name; must match the table name as it is stored in the database -JdbcMetadata.arguments.tableNamePattern.Label=Table name pattern +JdbcMetadata.arguments.tableNamePattern.Label=Tabella (pattern) JdbcMetadata.arguments.tableNamePattern.Tooltip=A table name pattern; may be a fully-qualified name -JdbcMetadata.arguments.tableTypes.Label=Table types +JdbcMetadata.arguments.tableTypes.Label=Tipi di tabella JdbcMetadata.arguments.tableTypes.Tooltip=A list of table types JdbcMetaData.CheckResult.ReceivingRows.OK=Transform is receiving input from other transforms. From 99c2f8b1e8bbfba552136378ba8834afdb91a0ad Mon Sep 17 00:00:00 2001 From: sramazzina Date: Mon, 20 Mar 2023 16:44:45 +0100 Subject: [PATCH 05/12] fix #2399 Initial conversion of JDBCMetaData plugin from PDI to Hop --- .../plugins/transforms/jdbc-metadata/pom.xml | 44 + .../jdbc-metadata/src/assembly/assembly.xml | 50 + .../src/main/resources/version.xml | 20 + assemblies/plugins/transforms/pom.xml | 1 + plugins/transforms/jdbc-metadata/pom.xml | 34 + .../transform/jdbcmetadata/JdbcMetadata.java | 705 +++++++++++++ .../jdbcmetadata/JdbcMetadataData.java | 68 ++ .../jdbcmetadata/JdbcMetadataDialog.java | 955 ++++++++++++++++++ .../jdbcmetadata/JdbcMetadataMeta.java | 829 +++++++++++++++ .../transform/jdbcmetadata/OutputField.java | 54 + .../src/main/resources/jdbcmetadata.svg | 71 ++ .../messages/messages_en_US.properties | 108 ++ .../messages/messages_it_IT.properties | 108 ++ plugins/transforms/pom.xml | 1 + .../apache/hop/ui/core/widget/ComboVar.java | 2 +- 15 files changed, 3049 insertions(+), 1 deletion(-) create mode 100644 assemblies/plugins/transforms/jdbc-metadata/pom.xml create mode 100644 assemblies/plugins/transforms/jdbc-metadata/src/assembly/assembly.xml create mode 100644 assemblies/plugins/transforms/jdbc-metadata/src/main/resources/version.xml create mode 100644 plugins/transforms/jdbc-metadata/pom.xml create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/OutputField.java create mode 100644 plugins/transforms/jdbc-metadata/src/main/resources/jdbcmetadata.svg create mode 100644 plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties create mode 100644 plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties diff --git a/assemblies/plugins/transforms/jdbc-metadata/pom.xml b/assemblies/plugins/transforms/jdbc-metadata/pom.xml new file mode 100644 index 00000000000..564eac05d76 --- /dev/null +++ b/assemblies/plugins/transforms/jdbc-metadata/pom.xml @@ -0,0 +1,44 @@ + + + + + 4.0.0 + + + org.apache.hop + hop-assemblies-plugins-transforms + 2.4.0-SNAPSHOT + + + + hop-assemblies-plugins-transforms-jdbc-metadata + 2.4.0-SNAPSHOT + pom + + Hop Assemblies Plugins Transforms JDBC Metadata + + + + + org.apache.hop + hop-transform-jdbc-metadata + ${project.version} + + + \ No newline at end of file diff --git a/assemblies/plugins/transforms/jdbc-metadata/src/assembly/assembly.xml b/assemblies/plugins/transforms/jdbc-metadata/src/assembly/assembly.xml new file mode 100644 index 00000000000..5d090b1cb2d --- /dev/null +++ b/assemblies/plugins/transforms/jdbc-metadata/src/assembly/assembly.xml @@ -0,0 +1,50 @@ + + + + hop-assemblies-plugins-transforms-jdbc-metadata + + zip + + transforms/jdbc-metadata + + + ${project.basedir}/src/main/resources/version.xml + . + true + + + + + lib + + **/* + + + + + + false + + org.apache.hop:hop-transform-jdbc-metadata:jar + + + + \ No newline at end of file diff --git a/assemblies/plugins/transforms/jdbc-metadata/src/main/resources/version.xml b/assemblies/plugins/transforms/jdbc-metadata/src/main/resources/version.xml new file mode 100644 index 00000000000..6be576acae9 --- /dev/null +++ b/assemblies/plugins/transforms/jdbc-metadata/src/main/resources/version.xml @@ -0,0 +1,20 @@ + + + +${project.version} \ No newline at end of file diff --git a/assemblies/plugins/transforms/pom.xml b/assemblies/plugins/transforms/pom.xml index 4bf3a649c83..6223c13a1fe 100644 --- a/assemblies/plugins/transforms/pom.xml +++ b/assemblies/plugins/transforms/pom.xml @@ -92,6 +92,7 @@ ifnull janino javascript + jdbc-metadata joinrows json kafka diff --git a/plugins/transforms/jdbc-metadata/pom.xml b/plugins/transforms/jdbc-metadata/pom.xml new file mode 100644 index 00000000000..e68fd34c6ae --- /dev/null +++ b/plugins/transforms/jdbc-metadata/pom.xml @@ -0,0 +1,34 @@ + + + + + 4.0.0 + + + org.apache.hop + hop-plugins-transforms + 2.4.0-SNAPSHOT + + + hop-transform-jdbc-metadata + jar + + Hop Plugins JDBC Metadata + + \ No newline at end of file diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java new file mode 100644 index 00000000000..36c0e45d202 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java @@ -0,0 +1,705 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.core.database.Database; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.LogLevel; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +import java.lang.reflect.Array; +import java.sql.*; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class JdbcMetadata extends BaseTransform { + public JdbcMetadata( + TransformMeta transformMeta, + JdbcMetadataMeta meta, + JdbcMetadataData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + private Object stringToArgumentValue(String stringValue, Class type) + throws IllegalArgumentException { + Object argument; + if (type == String.class) { + argument = (String) stringValue; + } else if (type == Boolean.class) { + argument = "Y".equals(stringValue) ? Boolean.TRUE : Boolean.FALSE; + } else if (type == Integer.class) { + argument = new Integer(stringValue); + } else { + throw new IllegalArgumentException("Can't handle valueType " + type.getName()); + } + return argument; + } + + private Object stringListToObjectArray(String stringValue, Class type) { + if (stringValue == null) return null; + String[] stringValues = stringValue.split(","); + int n = stringValues.length; + Object result = Array.newInstance(type, n); + for (int i = 0; i < n; i++) { + Array.set(result, i, stringToArgumentValue(stringValues[i], type)); + } + return result; + } + + /** + * Utility to set up the method and its arguments + * + * @param meta + * @param data + * @throws Exception + */ + private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exception { + // set up the method to call + /* logDebug("Setting up method to call."); + Method method = meta.getMethod(); + data.method = method; + + //Try to set up the arguments for the method + logDebug("Setting up method arguments."); + Class[] argumentTypes = method.getParameterTypes(); + int argc = argumentTypes.length; + logDebug("Method has " + argc + " arguments."); + List arguments = meta.getArguments(); + int argSize = arguments.size(); + logDebug("We expected " + argSize + " arguments."); + if (argc != argSize) { + throw new Exception("Method has a " + argc + " arguments, we expected " + argSize); + } + logDebug("Allocating arguments array."); + data.arguments = new ArrayList(); + String stringArgument; + if (meta.getArgumentSourceFields()) { + //arguments are specified by values coming from fields. + //we don't know about the fields yet, so we + //initialize with bogus value so we can check if all fields were found + logDebug("Allocating fieldindices array for arguments."); + data.argumentFieldIndices = new int[argc]; + for (int i = 0; i < argc; i++) { + data.argumentFieldIndices[i] = -1; + } + } + else { + //arguments are specified directly by the user in the step. + //here we convert the string values into proper argument values + Class argumentType; + String argument; + for (int i = 0; i < argc; i++) { + argumentType = argumentTypes[i]; + stringArgument = arguments.get(i); + if (stringArgument == null) { + argument = null; + } + else { + stringArgument = variables.resolve(stringArgument); + if (argumentType.isArray()) { + if (stringArgument.length() == 0) { + argument = null; + } + else { + argument = stringListToObjectArray(stringArgument, argumentType.getComponentType()); + } + } + else { + argument = stringToArgumentValue(stringArgument, argumentType); + } + } + data.arguments.add(i, argument); + } + }*/ + } + + /** + * Utility to create a jdbc connection. (Used when the connection source is JDBC or JDBCFields) + * + * @param driver The fully qualified classname identifying the jdbc driver + * @param url The driver-specific url to create the connection + * @param user The database user that wants to establish a connection + * @param password The password required to establish the connection + * @return A jdbc connection object. + * @throws Exception + */ + private Connection createJdbcConnection(String driver, String url, String user, String password) + throws Exception { + Class.forName(driver); + return DriverManager.getConnection(url, user, password); + } + + /** + * Initialize the connection + * + * @param meta + * @param data + * @throws Exception + */ + private void initConnection(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exception { + // Try to establish a connection in advance + logDebug("Try to establish a connection in advance."); + String connectionSource = meta.getConnectionSource(); + logDebug("Connection source: " + connectionSource); + Connection connection; + + if (JdbcMetadataMeta.connectionSourceOptionConnection.equals(connectionSource)) { + // connection is a named kettle connection + DatabaseMeta dbMeta = getPipelineMeta().findDatabase(meta.getConnectionName(), variables); + Database database = new Database(this, this, dbMeta); + database.connect(); + connection = database.getConnection(); + if (connection == null) { + throw new Exception("Connection returned by database object is null!"); + } + data.database = database; + } else if (JdbcMetadataMeta.connectionSourceOptionJDBC.equals(connectionSource)) { + // connection is a user-entered jdbc connection + String jdbcDriver = variables.resolve(meta.getJdbcDriverField()); + String jdbcUrl = variables.resolve(meta.getJdbcUrlField()); + String jdbcUser = variables.resolve(meta.getJdbcUserField()); + String jdbcPassword = variables.resolve(meta.getJdbcPasswordField()); + logDebug("Attempt to create JDBC connection."); + logDebug("Driver: " + jdbcDriver); + logDebug("Url: " + jdbcUrl); + logDebug("User: " + jdbcUser); + connection = createJdbcConnection(jdbcDriver, jdbcUrl, jdbcUser, jdbcPassword); + } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { + // Connection is a jdbc connection specified by field values + // we don't know the field values yet, but we can initialize a few vars to access their value + // later on + connection = null; + data.jdbcDriverField = -1; + data.jdbcUrlField = -1; + data.jdbcUserField = -1; + data.jdbcPasswordField = -1; + } else if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { + // Connection is a named kettle connection specified by a field value + // we don't know about the field value yet, but we can initialize a var to access its value + // later on + connection = null; + data.connectionField = -1; + } else { + // should not arrive here, just initialize the connection to make the compiler shut up. + connection = null; + } + // if the connection is null at this point, then we need to establish it on a row by row basis + data.connection = connection; + } + + private void initOutputFields(JdbcMetadataMeta meta, JdbcMetadataData data) { + List outputFields = meta.getOutputFields(); + int n = outputFields.size(); + data.resultSetIndices = new int[n]; + + IValueMeta[] fields = meta.getMethodResultSetDescriptor(); + int m = fields.length; + + for (int i = 0; i < n; i++) { + String fieldName = outputFields.get(i).getName(); + if (fieldName == null) { + continue; + } + for (int j = 0; j < m; j++) { + IValueMeta field = fields[j]; + if (!fieldName.equals(field.getName())) continue; + data.resultSetIndices[i] = j + 1; + break; + } + } + } + + @Override + public boolean init() { + boolean result = true; + if (!super.init()) return false; + + try { + data.databases = new HashMap(); + data.connections = new HashMap(); + data.connectionKey = new String[4]; + initMethod(meta, data); + initConnection(meta, data); + initOutputFields(meta, data); + } catch (Exception exception) { + logError( + "Unexpected " + + exception.getClass().getName() + + " initializing step: " + + exception.getMessage()); + exception.printStackTrace(); + result = false; + } + return result; + } + + private String getConnectionNameFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.connectionField]); + } + + private String getJdbcDriverFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.jdbcDriverField]); + } + + private String getJdbcUrlFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.jdbcUrlField]); + } + + private String getJdbcUserFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.jdbcUserField]); + } + + private String getJdbcPasswordFromRow(JdbcMetadataData data, Object[] row) { + return (String) (row[data.jdbcPasswordField]); + } + + /** + * This is called in the processRow function to obtain the contain to apply the metadata method + * to. + * + * @param meta + * @param data + * @param row + * @return + * @throws Exception + */ + private Connection getConnection(JdbcMetadataMeta meta, JdbcMetadataData data, Object[] row) + throws Exception { + Connection connection; + if (data.connection == null) { + // connection could not be initialized at init, so we have to obtain it now on a row by row + // basis. + String connectionSource = meta.getConnectionSource(); + if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { + // connection is a named kettle connection specified by a field value + String connectionName = getConnectionNameFromRow(data, row); + // try to get this named connection from the cache + Database database = data.databases.get(connectionName); + if (database == null) { + // we haven't seen this named connection before, try to find it. + DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(connectionName, variables); + database = new Database(this, this, databaseMeta); + connection = database.getConnection(); + if (connection == null) { + throw new IllegalArgumentException("Connection returned by database is null!"); + } + // cache the database for later use + data.databases.put(connectionName, database); + } else { + // database found in cache, get its connection + connection = database.getConnection(); + } + } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { + // database connectin is a jdbc connection defined by field values. + // try to find this connection in the cache + String[] key = data.connectionKey; + key[0] = getJdbcDriverFromRow(data, row); + key[1] = getJdbcUrlFromRow(data, row); + key[2] = getJdbcUserFromRow(data, row); + key[3] = getJdbcPasswordFromRow(data, row); + connection = data.connections.get(key); + if (connection == null) { + // connection not yet in the cache. Let's create it and cahce it. + connection = createJdbcConnection(key[0], key[1], key[2], key[3]); + data.connections.put(key, connection); + } + } else { + throw new Exception("Unexpected error acquiring connection"); + } + } else { + connection = data.connection; + } + return connection; + } + + /** + * This is called in the processRow function to get the actual arguments for the jdbc metadata + * method + * + * @param meta + * @param data + * @param row + * @throws Exception + */ + private void prepareMethodArguments(JdbcMetadataMeta meta, JdbcMetadataData data, Object[] row) + throws Exception { + // if the arguments are not from fields, then we already took care of it in the init phase, so + // leave + /* if (!meta.getArgumentSourceFields()) return; + //if the arguments are from fields, then we already stored the right indices to take the values from + Object[] args = data.arguments; + int[] indices = data.argumentFieldIndices; + int index; + Class[] argumentTypes = data.method.getParameterTypes(); + Class argumentType; + Object argument; + for (int i = 0; i < args.length; i++){ + index = indices[i]; + if (index == -2) { + argument = null; + } + else { + argument = row[index]; + } + argumentType = argumentTypes[i]; + if (argumentType.isArray() && argument != null) { + if ("".equals(argument)) { + logDebug("Converted empty string to null for argument array"); + argument = null; + } + else { + argument = stringListToObjectArray((String)argument, argumentType.getComponentType()); + } + } + args[i] = argument; + }*/ + } + + private Object[] createOutputRow( + JdbcMetadataMeta meta, JdbcMetadataData data, Object[] inputRow) { + Object[] outputRow = new Object[data.outputRowMeta.size()]; + if (data.inputFieldsToCopy == null) { + System.arraycopy(inputRow, 0, outputRow, 0, getInputRowMeta().size()); + } else { + for (int i = 0; i < data.inputFieldsToCopy.length; i++) { + outputRow[i] = inputRow[data.inputFieldsToCopy[i]]; + } + } + return outputRow; + } + + public boolean processRow() throws HopException { + + // get incoming row, getRow() potentially blocks waiting for more rows, returns null if no more + // rows expected + Object[] r = getRow(); + + // if no more rows are expected, indicate step is finished and processRow() should not be called + // again + if (r == null) { + setOutputDone(); + return false; + } + + // the "first" flag is inherited from the base step implementation + // it is used to guard some processing tasks, like figuring out field indexes + // in the row structure that only need to be done once + if (first) { + first = false; + IRowMeta inputRowMeta = getInputRowMeta(); + data.outputRowOffset = inputRowMeta.size(); + String connectionSource = meta.getConnectionSource(); + boolean argumentSourceFields = meta.isArgumentSourceFields(); + // check if we need to use fields. + // If so, store the indices so we can easily extract their values during transformation + if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource) + || JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource) + || argumentSourceFields) { + logDebug("Looking up indices of input fields."); + String fieldName; + String[] fieldNames = inputRowMeta.getFieldNames(); + logDebug("We have " + fieldNames.length + " input fields."); + List arguments = meta.getArguments(); + int argc = arguments.size(); + String stringArgument; + for (int i = 0; i < fieldNames.length; i++) { + fieldName = fieldNames[i]; + logDebug("Looking at field: " + fieldName); + // store indices for connection source fields + if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { + if (fieldName.equals(meta.getConnectionField())) { + logDebug("Found the connection field at index: " + i); + data.connectionField = i; + } + } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { + if (fieldName.equals(meta.getJdbcDriverField())) { + logDebug("Found the jdbcDriverField field at index: " + i); + data.jdbcDriverField = i; + } + if (fieldName.equals(meta.getJdbcUrlField())) { + logDebug("Found the jdbcUrlField field at index: " + i); + data.jdbcUrlField = i; + } + if (fieldName.equals(meta.getJdbcUserField())) { + logDebug("Found the jdbcUserField field at index: " + i); + data.jdbcUserField = i; + } + if (fieldName.equals(meta.getJdbcPasswordField())) { + logDebug("Found the jdbcPasswordField field at index: " + i); + data.jdbcPasswordField = i; + } + } + // store indices for argument fields + if (argumentSourceFields) { + logDebug("Trying to match argument fields against: " + fieldName); + for (int j = 0; j < argc; j++) { + stringArgument = arguments.get(j); + logDebug("Found argument " + j + ": " + stringArgument); + if (fieldName.equals(stringArgument)) { + logDebug("Match, storing index " + i); + data.argumentFieldIndices[j] = i; + } + } + } + } // end fields loop + + // ensure that we have all required fields. + if ((JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource) + && (data.jdbcDriverField == -1 + || data.jdbcUrlField == -1 + || data.jdbcUserField == -1 + || data.jdbcPasswordField == -1)) + || (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource) + && data.connectionField == -1)) { + throw new HopException("Not all fields for the connection source were found."); + } + + if (argumentSourceFields) { + // keep track of how many fields we used as args. + // We need this in case remove argument fields is enabled + // as this is the number of fields we need to discard from the input row. + int fieldsUsedAsArgs = 0; + // ensure all argument fields are bound to a valid field + argumentFields: + for (int j = 0; j < argc; j++) { + logDebug("Argument indices at " + j + ": " + data.argumentFieldIndices[j]); + if (data.argumentFieldIndices[j] == -1) { + // this argument does not point to any existing field. + if (arguments.get(j) == null || arguments.get(j).length() == 0) { + // the argument is blank, this is ok: we will pass null instead + data.argumentFieldIndices[j] = -2; + } else { + // this argument is not blank - this means it points to a non-existing field. + // this is an error. + Object[] descriptor = meta.getMethodDescriptor(); + Object[] args = (Object[]) descriptor[1]; + Object[] arg = (Object[]) args[j]; + throw new HopException( + "No field \"" + + arguments.get(j) + + "\" found for argument " + + j + + ": " + + ((String) arg[0])); + } + } else { + // this argument points to a valid field. + // let's check if this same field was already used as arg: + for (int i = 0; i < j; i++) { + if (data.argumentFieldIndices[i] == data.argumentFieldIndices[j]) { + // yes, it was used already. Let's check the next argument. + continue argumentFields; + } + } + // this field was not used already, so mark it as used. + fieldsUsedAsArgs++; + } + } + + if (meta.isRemoveArgumentFields()) { + int n = data.outputRowOffset; + data.outputRowOffset -= fieldsUsedAsArgs; + data.inputFieldsToCopy = new int[data.outputRowOffset]; + + inputFieldsToCopy: + for (int i = 0, j = 0; i < n; i++) { // for each field in the input row + for (int k = 0; k < argc; k++) { // for each method argument + if (data.argumentFieldIndices[k] == i) { + // this input field is used as argument. Continue to the next field. + continue inputFieldsToCopy; + } + } + // this field was not used as argument. make sure we copy it to the output. + data.inputFieldsToCopy[j++] = i; + } + } + } + logDebug("Done looking up indices of input fields."); + } + + // clone the input row structure and place it in our data object + data.outputRowMeta = (IRowMeta) inputRowMeta.clone(); + // use meta.getFields() to change it, so it reflects the output row structure + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); + } // end of first + + try { + logRowlevel("Processing 1 input row"); + Connection connection = getConnection(meta, data, r); + prepareMethodArguments(meta, data, r); + if (getLogLevel() == LogLevel.ROWLEVEL) { + logRowlevel("About to invoke method"); + /* for (int i = 0; i < data.arguments.size(); i++) { + logRowlevel( + "Argument " + + i + + "; " + + (data.arguments[i] == null + ? "null" + : data.arguments[i].toString() + + "; " + + data.arguments[i].getClass().getName())); + }*/ + } + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet resultSet = (ResultSet) data.method.invoke(databaseMetaData, data.arguments); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + int columnCount = resultSetMetaData.getColumnCount(); + IValueMeta IValueMeta; + Object value; + boolean outputRows = false; + int k; + /* while (resultSet.next()) { + logRowlevel("Processing 1 output row."); + Object[] outputRow = createOutputRow(meta, data, r); + for (int i = data.outputRowOffset, j = 0; i < data.outputRowMeta.size(); i++, j++) { + k = data.resultSetIndices[j]; + if (k > columnCount) continue; + IValueMeta valueMetaInterface = data.outputRowMeta.getValueMeta(i); + switch (valueMetaInterface.getType()) { + case valueMetaInterface + .TYPE_BOOLEAN: // while the JDBC spec prescribes boolean, not all drivers actually + // can deliver. + boolean v; + switch (resultSetMetaData.getColumnType(k)) { + case Types.INTEGER: + case Types.SMALLINT: + case Types.TINYINT: + v = resultSet.getInt(k) == 1 ? true : false; + break; + default: + v = resultSet.getBoolean(k); + } + value = new Boolean(v); + break; + case valueMetaInterface.TYPE_INTEGER: + value = new Long(resultSet.getInt(k)); + break; + default: + value = resultSet.getObject(k); + } + outputRow[i] = value; + outputRows = true; + } + // put the row to the output row stream + putRow(data.outputRowMeta, outputRow); + logRowlevel("Done processing 1 output row."); + }*/ + resultSet.close(); + if (!outputRows && meta.isAlwaysPassInputRow()) { + Object[] outputRow = createOutputRow(meta, data, r); + putRow(data.outputRowMeta, outputRow); + } + logRowlevel("Done processing 1 input row."); + } catch (Exception exception) { + exception.printStackTrace(); + if (exception instanceof HopException) { + throw (HopException) exception; + } else { + throw new HopException(exception); + } + } + + // log progress if it is time to to so + if (checkFeedback(getLinesRead())) { + logBasic("Linenr " + getLinesRead()); // Some basic logging + } + + // indicate that processRow() should be called again + return true; + } + + @Override + public void dispose() { + // clean up the database + try { + if (data.database != null) { + data.database.disconnect(); + data.connection = null; + data.database = null; + } + } catch (Exception ex) { + logError("Error cleaning up database: " + ex.getMessage()); + } + + // clean up the connection + try { + if (data.connection != null && !data.connection.isClosed()) { + data.connection.close(); + } + } catch (Exception ex) { + logError("Error cleaning up connection: " + ex.getMessage()); + } + data.connection = null; + + // clean up the database map + Iterator> dbIterator = data.databases.entrySet().iterator(); + while (dbIterator.hasNext()) { + Map.Entry current = dbIterator.next(); + Database database = current.getValue(); + try { + database.disconnect(); + } catch (Exception ex) { + logError("Error cleaning up database " + current.getKey() + ": " + ex.getMessage()); + } + } + data.databases.clear(); + data.databases = null; + + // clean up the connection map + Iterator> connectionIterator = + data.connections.entrySet().iterator(); + while (connectionIterator.hasNext()) { + Connection connection = connectionIterator.next().getValue(); + try { + if (!connection.isClosed()) { + connection.close(); + } + } catch (Exception ex) { + logError("Error cleaning up connection: " + ex.getMessage()); + } + } + data.connections.clear(); + data.connections = null; + + data.arguments = null; + data.method = null; + data.argumentFieldIndices = null; + data.inputFieldsToCopy = null; + data.resultSetIndices = null; + data.connectionKey = null; + + data.jdbcDriverField = -1; + data.jdbcUrlField = -1; + data.jdbcUserField = -1; + data.jdbcPasswordField = -1; + + super.dispose(); + } +} diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java new file mode 100644 index 00000000000..d9c3ecd0678 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.core.database.Database; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.ITransformData; + +import java.lang.reflect.Method; +import java.sql.Connection; +import java.util.List; +import java.util.Map; + +public class JdbcMetadataData extends BaseTransformData implements ITransformData { + + public IRowMeta outputRowMeta; + // used to store the named connection + public Database database; + // used to store the actual jdbc connection + public Connection connection; + // used to store the DatabaseMetaData method that generates the data + public Method method; + // use to store the arguments to the method + public List arguments; + // named kettle connection cache. Used when connection is a named connection specified by input + // fields + public Map databases = null; + // connection cache. Used when connection is a jdbc connection specified by input fields + public Map connections = null; + // key to the connection cache. + public String[] connectionKey = null; + // field index for named kettle connection + public int connectionField = -1; + // field indices for jdbc connections + public int jdbcDriverField = -1; + public int jdbcUrlField = -1; + public int jdbcUserField = -1; + public int jdbcPasswordField = -1; + // indices for fields used to specify method arguments + public int[] argumentFieldIndices; + // the offset in the output row from where we can add our metadata fields. + // (we need this in case we're required to remove arguments fields from the input) + public int outputRowOffset = -1; + // + public int[] inputFieldsToCopy; + // the indices of the columns in the resultset + public int[] resultSetIndices; + + public JdbcMetadataData() { + super(); + } +} diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java new file mode 100644 index 00000000000..d61fbcc8b53 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java @@ -0,0 +1,955 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.core.Const; +import org.apache.hop.core.Props; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.ITransformDialog; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.widget.ColumnInfo; +import org.apache.hop.ui.core.widget.ComboVar; +import org.apache.hop.ui.core.widget.MetaSelectionLine; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; +import org.eclipse.swt.events.*; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.*; + +import java.util.ArrayList; +import java.util.List; + +public class JdbcMetadataDialog extends BaseTransformDialog implements ITransformDialog { + private static final Class PKG = JdbcMetadataMeta.class; // for i18n purposes + private JdbcMetadataMeta meta; + + private int middle = props.getMiddlePct(); + private int margin = PropsUi.getMargin(); + + private boolean dialogChanged; + private ModifyListener lsMod; + + private Composite metadataComposite; + // + private MetaSelectionLine + wConnectionSource; // text field holding the name of the field containing the connection name + private CCombo connectionField; + // text field holding the name of the field containing the driver name + private ComboVar jdbcDriverField; + // text field holding the name of the field containing the url + private ComboVar jdbcUrlField; + // text field holding the name of the field containing the user + private ComboVar jdbcUserField; + // text field holding the name of the field containing the password + private ComboVar jdbcPasswordField; + // + + private Label alwaysPassInputRowLabel; + // + private Button alwaysPassInputRowButton; + // + private Label methodLabel; + // + // private CCombo methodCombo; + private Combo methodCombo; + + private Label argumentSourceLabel; + // + private Button argumentSourceFields; + // + private Label removeArgumentFieldsLabel; + // + private Button removeArgumentFieldsButton; + // + private TableView outputFieldsTableView; + + public JdbcMetadataDialog( + Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { + super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); + meta = (JdbcMetadataMeta) in; + } + + private final String[] emptyFieldList = new String[0]; + + private String[] getFieldListForCombo() { + String[] items; + try { + IRowMeta r = pipelineMeta.getPrevTransformFields(variables, transformName); + items = r.getFieldNames(); + } catch (HopException exception) { + items = emptyFieldList; + } + return items; + } + + private void connectionSourceUpdated() { + int selectedIndex = wConnectionSource.getSelectionIndex(); + String option = JdbcMetadataMeta.connectionSourceOptions[selectedIndex]; + boolean connectionComboEnabled, connectionFieldEnabled, otherFieldsEnabled; + otherFieldsEnabled = connectionComboEnabled = connectionFieldEnabled = false; + String[] fields = emptyFieldList; + if (JdbcMetadataMeta.connectionSourceOptionConnection.equals(option)) { + connectionComboEnabled = true; + } else { + if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(option)) { + connectionFieldEnabled = true; + connectionField.setItems(getFieldListForCombo()); + } else { + otherFieldsEnabled = true; + if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(option)) { + fields = getFieldListForCombo(); + } + } + } + wConnectionSource.setEnabled(connectionComboEnabled); + connectionField.setEnabled(connectionFieldEnabled); + jdbcDriverField.setEnabled(otherFieldsEnabled); + jdbcDriverField.setItems(fields); + jdbcUrlField.setEnabled(otherFieldsEnabled); + jdbcUrlField.setItems(fields); + jdbcUserField.setEnabled(otherFieldsEnabled); + jdbcUserField.setItems(fields); + jdbcPasswordField.setEnabled(otherFieldsEnabled); + jdbcPasswordField.setItems(fields); + } + + /** Remove the UI to enter method arguments The current values are stored and returned. */ + private List removeArgumentsUI() { + Control[] controls = metadataComposite.getChildren(); + logBasic( + "[removeArgumentsUI] metadataComposite start - Children # -> " + + controls.length); + List currentValues = new ArrayList(); + for (Control control : controls) { + if (control == alwaysPassInputRowLabel + || control == alwaysPassInputRowButton + || control == methodLabel + || control == methodCombo + || control == argumentSourceLabel + || control == argumentSourceFields + || control == removeArgumentFieldsLabel + || control == removeArgumentFieldsButton) continue; + if (control instanceof Combo ) { + if (control.isDisposed()) { + logBasic("removeArgumentsUI - trying to access a disposed control!"); + } + if (!control.isDisposed() && (((Combo) control).getText() != null || ((Combo) control).getText().length()>0)) { + currentValues.add(((Combo) control).getText()); + logBasic("removeArgumentsUI - control.getText(): " + ((Combo) control).getText()); + } + } + if (!control.isDisposed()) { + logBasic("removeArgumentsUI - Disposing control!"); + control.dispose(); + logBasic("removeArgumentsUI - number of children in parent composite: " + controls.length); + } + } + + logBasic( + "[removeArgumentsUI] metadataComposite end - Children # -> " + + metadataComposite.getChildren().length); + return currentValues; + } + + /** + * Create the UI to enter one argument. + * + * @param argumentDescriptor + * @param lastControl + * @param items + * @return The combobox where the user enters the argument. + */ + // private ComboVar createArgumentUI( + private Combo createArgumentUI( + Object[] argumentDescriptor, Control lastControl, String[] items) { + String argumentName = (String) argumentDescriptor[0]; + Label label = new Label(metadataComposite, SWT.RIGHT); + label.setText(BaseMessages.getString(PKG, "JdbcMetadata.arguments." + argumentName + ".Label")); + label.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.arguments." + argumentName + ".Tooltip")); + props.setLook(label); + FormData labelFormData = new FormData(); + labelFormData.left = new FormAttachment(0, 0); + labelFormData.right = new FormAttachment(middle, -margin); + labelFormData.top = new FormAttachment(lastControl, margin); + label.setLayoutData(labelFormData); + +// ComboVar comboVar = +// new ComboVar(variables, metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + Combo comboVar = + new Combo (metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(comboVar); + FormData comboVarFormData = new FormData(); + comboVarFormData.left = new FormAttachment(middle, 0); + comboVarFormData.right = new FormAttachment(100, 0); + comboVarFormData.top = new FormAttachment(lastControl, margin); + comboVar.setLayoutData(comboVarFormData); + comboVar.setItems(items); + + // comboVar.addModifyListener(lsMod); + + return comboVar; + } + /** Create UI to enter arguments. Return a new set of arguments to store in the meta object */ + private List createArgumentsUI(Object[] argumentDescriptors, List currentValues) { + logBasic( + "createArgumentsUI, currentValues = " + + (currentValues == null ? "null" : currentValues.size())); + Object[] argumentDescriptor; + int argc = argumentDescriptors.length; + List newArguments = new ArrayList(argc); + Control lastControl = removeArgumentFieldsButton; + String[] items = argumentSourceFields.getSelection() ? getFieldListForCombo() : emptyFieldList; + for (int i = 0; i < argc; i++) { + argumentDescriptor = (Object[]) argumentDescriptors[i]; + // ComboVar comboVar = createArgumentUI(argumentDescriptor, lastControl, items); + Combo comboVar = createArgumentUI(argumentDescriptor, lastControl, items); + lastControl = comboVar; + + // copy the old argument values to the new arguments array + if (i >= currentValues.size()) continue; + String argumentValue = currentValues.get(i); + newArguments.add(i, argumentValue); + if (argumentValue == null) continue; + comboVar.setText(argumentValue); + } + return newArguments; + } + + /** fill the fields table with output fields. */ + private void populateFieldsTable(Object[] methodDescriptor) { + logBasic("populateFieldsTable 1"); + List outputFields = getOutputFields(); + outputFieldsTableView.clearAll(); + IValueMeta[] fields = (IValueMeta[]) methodDescriptor[2]; + int n = fields.length; + Table table = outputFieldsTableView.table; + table.setItemCount(n); + TableItem tableItem; + String fieldName; + IValueMeta field; + outputFieldsTableView.optWidth(true, n); + int m = (outputFields == null) ? 0 : outputFields.size(); + OutputField outputField; + for (int i = 0; i < n; i++) { + field = fields[i]; + tableItem = table.getItem(i); + fieldName = field.getName(); + tableItem.setText(1, fieldName); + // initially, the field is renamed unto itself. + tableItem.setText(2, fieldName); + // now see if the meta object renamed this field. + for (int j = 0; j < m; j++) { + outputField = outputFields.get(j); + if (!fieldName.equals(outputField.getName())) continue; + tableItem.setText(2, outputField.getRename()); + break; + } + } + } + + private void populateFieldsTable() { + logBasic("populateFieldsTable 2"); + populateFieldsTable(JdbcMetadataMeta.getMethodDescriptor(methodCombo.getSelectionIndex())); + } + + private void updateOutputFields(List outputFields) { + logBasic("updateOutputFields " + outputFields); + if (outputFields == null) return; + outputFieldsTableView.clearAll(); + OutputField outputField; + int n = outputFields.size(); + Table table = outputFieldsTableView.table; + table.setItemCount(n); + TableItem tableItem; + String text; + for (int i = 0; i < n; i++) { + outputField = outputFields.get(i); + tableItem = table.getItem(i); + + if (outputField == null) continue; + text = outputField.getName(); + tableItem.setText(1, text == null ? "" : text); + + if (outputField.getRename() == null) continue; + text = outputField.getRename(); + tableItem.setText(2, text == null ? "" : text); + } + } + + /** + * When the method is updated, we need the ui to change to allow the user to enter arguments. This + * takes care of that. + */ + private void methodUpdated(List argumentValues) { + logBasic( + "methodUpdated, argumentValues = " + + (argumentValues == null ? "null" : argumentValues.size())); + // first, remove the controls for the previous set of arguments + List currentValues = removeArgumentsUI(); + logBasic("currentValue = " + currentValues.size()); + if (argumentValues == null) { + argumentValues = new ArrayList(currentValues.size()); + currentValues.addAll(argumentValues); + } + + // setup controls for the current set of arguments + int index = methodCombo.getSelectionIndex(); + Object[] methodDescriptor = (Object[]) JdbcMetadataMeta.methodDescriptors[index]; + Object[] argumentDescriptors = (Object[]) methodDescriptor[1]; + + List newArguments = createArgumentsUI(argumentDescriptors, argumentValues); + // update the arguments in the meta object + meta.setArguments(newArguments); + // show / hide the argument source ui depending on whether we have arguments + boolean visible = newArguments.size() > 0; + /* (SR) Temporarily removed + argumentSourceFields.setVisible(visible); + argumentSourceLabel.setVisible(visible); + removeArgumentFieldsLabel.setVisible(visible); + removeArgumentFieldsButton.setVisible(visible); + */ + metadataComposite.layout(); + } + + private void methodUpdated() { + logBasic("Parameterless methodUpdated called."); + methodUpdated(null); + } + + public String open() { + dialogChanged = false; + // store some convenient SWT variables + Shell parent = getParent(); + + // SWT code for preparing the dialog + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MIN | SWT.MAX); + props.setLook(shell); + setShellImage(shell, meta); + + changed = meta.hasChanged(); + + lsMod = e -> meta.setChanged(); + SelectionListener lsSelection = + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + meta.setChanged(); + } + }; + // ------------------------------------------------------- // + // SWT code for building the actual settings dialog // + // ------------------------------------------------------- // + Control lastControl; + + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = PropsUi.getFormMargin(); + formLayout.marginHeight = PropsUi.getFormMargin(); + + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "JdbcMetadata.Name")); + + wlTransformName = new Label(shell, SWT.RIGHT); + wlTransformName.setText(BaseMessages.getString(PKG, "System.Label.TransformName")); + props.setLook(wlTransformName); + fdlTransformName = new FormData(); + fdlTransformName.left = new FormAttachment(0, 0); + fdlTransformName.right = new FormAttachment(middle, -margin); + fdlTransformName.top = new FormAttachment(0, margin); + wlTransformName.setLayoutData(fdlTransformName); + + wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wTransformName.setText(transformName); + props.setLook(wTransformName); + wTransformName.addModifyListener(lsMod); + fdTransformName = new FormData(); + fdTransformName.left = new FormAttachment(middle, 0); + fdTransformName.top = new FormAttachment(0, margin); + fdTransformName.right = new FormAttachment(100, 0); + wTransformName.setLayoutData(fdTransformName); + + lastControl = wTransformName; + + // pass the row checkbox + alwaysPassInputRowLabel = new Label(shell, SWT.RIGHT); + alwaysPassInputRowLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.passRow.Label")); + alwaysPassInputRowLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.passRow.Tooltip")); + props.setLook(alwaysPassInputRowLabel); + FormData alwaysPassInputRowLabelFormData = new FormData(); + alwaysPassInputRowLabelFormData.left = new FormAttachment(0, 0); + alwaysPassInputRowLabelFormData.right = new FormAttachment(middle, -margin); + alwaysPassInputRowLabelFormData.top = new FormAttachment(lastControl, margin); + alwaysPassInputRowLabel.setLayoutData(alwaysPassInputRowLabelFormData); + + alwaysPassInputRowButton = new Button(shell, SWT.CHECK); + props.setLook(alwaysPassInputRowButton); + FormData alwaysPassInputRowButtonFormData = new FormData(); + alwaysPassInputRowButtonFormData.left = new FormAttachment(middle, 0); + alwaysPassInputRowButtonFormData.right = new FormAttachment(100, 0); + alwaysPassInputRowButtonFormData.top = new FormAttachment(lastControl, margin); + alwaysPassInputRowButton.setLayoutData(alwaysPassInputRowButtonFormData); + + lastControl = alwaysPassInputRowButton; + + // + CTabFolder cTabFolder = new CTabFolder(shell, SWT.BORDER); + props.setLook(cTabFolder, Props.WIDGET_STYLE_TAB); + + // Connection tab + CTabItem connectionTab = new CTabItem(cTabFolder, SWT.NONE); + connectionTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.ConnectionTab.Label")); + connectionTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.ConnectionTab.Tooltip")); + + FormLayout connectionTabLayout = new FormLayout(); + connectionTabLayout.marginWidth = PropsUi.getFormMargin(); + connectionTabLayout.marginHeight = PropsUi.getFormMargin(); + + Composite connectionComposite = new Composite(cTabFolder, SWT.NONE); + props.setLook(connectionComposite); + connectionComposite.setLayout(connectionTabLayout); + + // Connection line + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(meta.getConnectionName(), variables); + wConnectionSource = + addConnectionLine( + connectionComposite, + wTransformName, + databaseMeta, + lsMod, + BaseMessages.getString(PKG, "JdbcMetadata.connectionSource.Label"), + BaseMessages.getString(PKG, "JdbcMetadata.connectionSource.Tooltip")); + wConnectionSource.addSelectionListener(lsSelection); + lastControl = wConnectionSource; + + // connection name field + Label connectionFieldLabel = new Label(connectionComposite, SWT.RIGHT); + connectionFieldLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.connectionField.Label")); + connectionFieldLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.connectionField.Tooltip")); + props.setLook(connectionFieldLabel); + FormData connectionFieldLabelFormData = new FormData(); + connectionFieldLabelFormData.left = new FormAttachment(0, 0); + connectionFieldLabelFormData.right = new FormAttachment(middle, -margin); + connectionFieldLabelFormData.top = new FormAttachment(lastControl, margin); + connectionFieldLabel.setLayoutData(connectionFieldLabelFormData); + + connectionField = new CCombo(connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(connectionField); + connectionField.addModifyListener(lsMod); + FormData connectionFieldFormData = new FormData(); + connectionFieldFormData.left = new FormAttachment(middle, 0); + connectionFieldFormData.right = new FormAttachment(100, 0); + connectionFieldFormData.top = new FormAttachment(lastControl, margin); + connectionField.setLayoutData(connectionFieldFormData); + + lastControl = connectionField; + + // jdbc driver field + Label jdbcDriverLabel = new Label(connectionComposite, SWT.RIGHT); + jdbcDriverLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.driverField.Label")); + jdbcDriverLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.driverField.Tooltip")); + props.setLook(jdbcDriverLabel); + FormData jdbcDriverLabelFormData = new FormData(); + jdbcDriverLabelFormData.left = new FormAttachment(0, 0); + jdbcDriverLabelFormData.right = new FormAttachment(middle, -margin); + jdbcDriverLabelFormData.top = new FormAttachment(lastControl, margin); + jdbcDriverLabel.setLayoutData(jdbcDriverLabelFormData); + + jdbcDriverField = + new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(jdbcDriverField); + jdbcDriverField.addModifyListener(lsMod); + FormData jdbcDriverFieldFormData = new FormData(); + jdbcDriverFieldFormData.left = new FormAttachment(middle, 0); + jdbcDriverFieldFormData.right = new FormAttachment(100, 0); + jdbcDriverFieldFormData.top = new FormAttachment(lastControl, margin); + jdbcDriverField.setLayoutData(jdbcDriverFieldFormData); + + lastControl = jdbcDriverField; + + // jdbc url field + Label jdbcUrlLabel = new Label(connectionComposite, SWT.RIGHT); + jdbcUrlLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.urlField.Label")); + jdbcUrlLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.urlField.Tooltip")); + props.setLook(jdbcUrlLabel); + FormData jdbcUrlLabelFormData = new FormData(); + jdbcUrlLabelFormData.left = new FormAttachment(0, 0); + jdbcUrlLabelFormData.right = new FormAttachment(middle, -margin); + jdbcUrlLabelFormData.top = new FormAttachment(lastControl, margin); + jdbcUrlLabel.setLayoutData(jdbcUrlLabelFormData); + + jdbcUrlField = new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(jdbcUrlField); + jdbcUrlField.addModifyListener(lsMod); + FormData jdbcUrlFieldFormData = new FormData(); + jdbcUrlFieldFormData.left = new FormAttachment(middle, 0); + jdbcUrlFieldFormData.right = new FormAttachment(100, 0); + jdbcUrlFieldFormData.top = new FormAttachment(lastControl, margin); + jdbcUrlField.setLayoutData(jdbcUrlFieldFormData); + + lastControl = jdbcUrlField; + + // jdbc user field + Label jdbcUserLabel = new Label(connectionComposite, SWT.RIGHT); + jdbcUserLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.userField.Label")); + jdbcUserLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.userField.Tooltip")); + props.setLook(jdbcUserLabel); + FormData jdbcUserLabelFormData = new FormData(); + jdbcUserLabelFormData.left = new FormAttachment(0, 0); + jdbcUserLabelFormData.right = new FormAttachment(middle, -margin); + jdbcUserLabelFormData.top = new FormAttachment(lastControl, margin); + jdbcUserLabel.setLayoutData(jdbcUserLabelFormData); + + jdbcUserField = + new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(jdbcUserField); + jdbcUserField.addModifyListener(lsMod); + FormData jdbcUserFieldFormData = new FormData(); + jdbcUserFieldFormData.left = new FormAttachment(middle, 0); + jdbcUserFieldFormData.right = new FormAttachment(100, 0); + jdbcUserFieldFormData.top = new FormAttachment(lastControl, margin); + jdbcUserField.setLayoutData(jdbcUserFieldFormData); + + lastControl = jdbcUserField; + + // jdbc password field + Label jdbcPasswordLabel = new Label(connectionComposite, SWT.RIGHT); + jdbcPasswordLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.passwordField.Label")); + jdbcPasswordLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.passwordField.Tooltip")); + props.setLook(jdbcPasswordLabel); + FormData jdbcPasswordLabelFormData = new FormData(); + jdbcPasswordLabelFormData.left = new FormAttachment(0, 0); + jdbcPasswordLabelFormData.right = new FormAttachment(middle, -margin); + jdbcPasswordLabelFormData.top = new FormAttachment(lastControl, margin); + jdbcPasswordLabel.setLayoutData(jdbcPasswordLabelFormData); + + jdbcPasswordField = + new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(jdbcPasswordField); + jdbcPasswordField.addModifyListener(lsMod); + FormData jdbcPasswordFieldFormData = new FormData(); + jdbcPasswordFieldFormData.left = new FormAttachment(middle, 0); + jdbcPasswordFieldFormData.right = new FormAttachment(100, 0); + jdbcPasswordFieldFormData.top = new FormAttachment(lastControl, margin); + jdbcPasswordField.setLayoutData(jdbcPasswordFieldFormData); + + lastControl = jdbcPasswordField; + + // layout the connection tab + FormData connectionTabFormData = new FormData(); + connectionTabFormData.left = new FormAttachment(0, 0); + connectionTabFormData.top = new FormAttachment(0, 0); + connectionTabFormData.right = new FormAttachment(100, 0); + connectionTabFormData.bottom = new FormAttachment(100, 0); + connectionComposite.setLayoutData(connectionTabFormData); + connectionComposite.layout(); + connectionTab.setControl(connectionComposite); + + // Metadata tab + CTabItem metadataTab = new CTabItem(cTabFolder, SWT.NONE); + metadataTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.MetaDataTab.Label")); + metadataTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.MetaDataTab.Tooltip")); + + FormLayout metadataTabLayout = new FormLayout(); + connectionTabLayout.marginWidth = Const.FORM_MARGIN; + connectionTabLayout.marginHeight = Const.FORM_MARGIN; + + metadataComposite = new Composite(cTabFolder, SWT.NONE); + props.setLook(metadataComposite); + metadataComposite.setLayout(metadataTabLayout); + + // method + methodLabel = new Label(metadataComposite, SWT.RIGHT); + methodLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.metadataMethod.Label")); + methodLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.metadataMethod.Tooltip")); + props.setLook(methodLabel); + FormData methodLabelFormData = new FormData(); + methodLabelFormData.left = new FormAttachment(0, 0); + methodLabelFormData.right = new FormAttachment(middle, -margin); + methodLabelFormData.top = new FormAttachment(0, margin); + methodLabel.setLayoutData(methodLabelFormData); + + methodCombo = new Combo(metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(methodCombo); + // methodCombo.setEditable(false); + methodCombo.addModifyListener(lsMod); + FormData methodComboFormData = new FormData(); + methodComboFormData.left = new FormAttachment(middle, 0); + methodComboFormData.right = new FormAttachment(100, 0); + methodComboFormData.top = new FormAttachment(lastControl, margin); + methodCombo.setLayoutData(methodComboFormData); + + Object[] methodDescriptor; + String methodName; + for (int i = 0; i < JdbcMetadataMeta.methodDescriptors.length; i++) { + methodDescriptor = (Object[]) JdbcMetadataMeta.methodDescriptors[i]; + methodName = (String) methodDescriptor[0]; + methodCombo.add(BaseMessages.getString(PKG, "JdbcMetadata.methods." + methodName)); + } + + SelectionListener methodComboSelectionListener = + new SelectionListener() { + @Override + public void widgetDefaultSelected(SelectionEvent selectionEvent) {} + + @Override + public void widgetSelected(SelectionEvent selectionEvent) { + logBasic("methodCombo changed, calling parameterless methodUpdated"); + methodUpdated(); + populateFieldsTable(); + meta.setChanged(); + } + }; + methodCombo.addSelectionListener(methodComboSelectionListener); + lastControl = methodCombo; + + // argument source + argumentSourceLabel = new Label(metadataComposite, SWT.RIGHT); + argumentSourceLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.argumentSource.Label")); + argumentSourceLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.argumentSource.Tooltip")); + props.setLook(argumentSourceLabel); + FormData argumentSourceLabelFormData = new FormData(); + argumentSourceLabelFormData.left = new FormAttachment(0, 0); + argumentSourceLabelFormData.right = new FormAttachment(middle, -margin); + argumentSourceLabelFormData.top = new FormAttachment(lastControl, margin); + argumentSourceLabel.setLayoutData(argumentSourceLabelFormData); + + argumentSourceFields = new Button(metadataComposite, SWT.CHECK); + props.setLook(argumentSourceFields); + FormData argumentSourceFieldsFormData = new FormData(); + argumentSourceFieldsFormData.left = new FormAttachment(middle, 0); + argumentSourceFieldsFormData.right = new FormAttachment(100, 0); + argumentSourceFieldsFormData.top = new FormAttachment(lastControl, margin); + argumentSourceFields.setLayoutData(argumentSourceFieldsFormData); + SelectionListener argumentSourceFieldsSelectionListener = + new SelectionListener() { + @Override + public void widgetDefaultSelected(SelectionEvent selectionEvent) {} + + @Override + public void widgetSelected(SelectionEvent selectionEvent) { + Control[] controls = metadataComposite.getChildren(); + boolean selection = argumentSourceFields.getSelection(); + removeArgumentFieldsButton.setEnabled(selection); + String[] items = selection ? getFieldListForCombo() : emptyFieldList; + for (Control control : controls) { + if (!(control instanceof ComboVar)) continue; + ComboVar comboVar = (ComboVar) control; + comboVar.setItems(items); + } + meta.setChanged(); + } + }; + argumentSourceFields.addSelectionListener(argumentSourceFieldsSelectionListener); + + lastControl = argumentSourceFields; + + // remove arguments + removeArgumentFieldsLabel = new Label(metadataComposite, SWT.RIGHT); + removeArgumentFieldsLabel.setText( + BaseMessages.getString(PKG, "JdbcMetadata.removeArgumentFields.Label")); + removeArgumentFieldsLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.removeArgumentFields.Tooltip")); + props.setLook(removeArgumentFieldsLabel); + FormData removeArgumentFieldsLabelFormData = new FormData(); + removeArgumentFieldsLabelFormData.left = new FormAttachment(0, 0); + removeArgumentFieldsLabelFormData.right = new FormAttachment(middle, -margin); + removeArgumentFieldsLabelFormData.top = new FormAttachment(lastControl, margin); + removeArgumentFieldsLabel.setLayoutData(removeArgumentFieldsLabelFormData); + + removeArgumentFieldsButton = new Button(metadataComposite, SWT.CHECK); + props.setLook(removeArgumentFieldsButton); + FormData removeArgumentFieldsButtonFormData = new FormData(); + removeArgumentFieldsButtonFormData.left = new FormAttachment(middle, 0); + removeArgumentFieldsButtonFormData.right = new FormAttachment(100, 0); + removeArgumentFieldsButtonFormData.top = new FormAttachment(lastControl, margin); + removeArgumentFieldsButton.setLayoutData(removeArgumentFieldsButtonFormData); + + // layout the metdata tab + FormData metadataTabFormData = new FormData(); + metadataTabFormData.left = new FormAttachment(0, 0); + metadataTabFormData.top = new FormAttachment(0, 0); + metadataTabFormData.right = new FormAttachment(100, 0); + metadataTabFormData.bottom = new FormAttachment(100, 0); + metadataComposite.setLayoutData(metadataTabFormData); + metadataComposite.layout(); + metadataTab.setControl(metadataComposite); + + // Fields tab + CTabItem fieldsTab = new CTabItem(cTabFolder, SWT.NONE); + fieldsTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.FieldsTab.Label")); + fieldsTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.FieldsTab.Tooltip")); + + FormLayout fieldsTabLayout = new FormLayout(); + fieldsTabLayout.marginWidth = Const.FORM_MARGIN; + fieldsTabLayout.marginHeight = Const.FORM_MARGIN; + + Composite fieldsComposite = new Composite(cTabFolder, SWT.NONE); + props.setLook(fieldsComposite); + fieldsComposite.setLayout(fieldsTabLayout); + + // add UI for the fields tab. + Label outputFieldsTableViewLabel = new Label(fieldsComposite, SWT.NONE); + outputFieldsTableViewLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.FieldsTab.Label")); + outputFieldsTableViewLabel.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.FieldsTab.Tooltip")); + props.setLook(outputFieldsTableViewLabel); + FormData outputFieldsTableViewLabelFormData = new FormData(); + outputFieldsTableViewLabelFormData.left = new FormAttachment(0, 0); + outputFieldsTableViewLabelFormData.top = new FormAttachment(0, margin); + outputFieldsTableViewLabel.setLayoutData(outputFieldsTableViewLabelFormData); + + ColumnInfo[] columnInfo = + new ColumnInfo[] { + new ColumnInfo( + BaseMessages.getString(PKG, "JdbcMetadata.FieldName.Label"), + ColumnInfo.COLUMN_TYPE_NONE), + new ColumnInfo( + BaseMessages.getString(PKG, "JdbcMetadata.OutputFieldName.Label"), + ColumnInfo.COLUMN_TYPE_TEXT) + }; + outputFieldsTableView = + new TableView( + variables, + fieldsComposite, + SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL, + columnInfo, + 10, + lsMod, + props); + + Button getFieldsButton = new Button(fieldsComposite, SWT.PUSH); + getFieldsButton.setText(BaseMessages.getString(PKG, "JdbcMetadata.getFieldsButton.Label")); + getFieldsButton.setToolTipText( + BaseMessages.getString(PKG, "JdbcMetadata.getFieldsButton.Tooltip")); + FormData getFieldsButtonFormData = new FormData(); + getFieldsButtonFormData.top = new FormAttachment(outputFieldsTableViewLabel, margin); + getFieldsButtonFormData.right = new FormAttachment(100, 0); + getFieldsButton.setLayoutData(getFieldsButtonFormData); + getFieldsButton.addSelectionListener( + new SelectionListener() { + + @Override + public void widgetDefaultSelected(SelectionEvent arg0) {} + + @Override + public void widgetSelected(SelectionEvent arg0) { + populateFieldsTable(); + meta.setChanged(); + } + }); + + FormData outputFieldsTableViewFormData = new FormData(); + outputFieldsTableViewFormData.left = new FormAttachment(0, 0); + outputFieldsTableViewFormData.top = new FormAttachment(outputFieldsTableViewLabel, margin); + outputFieldsTableViewFormData.right = new FormAttachment(getFieldsButton, -margin); + outputFieldsTableViewFormData.bottom = new FormAttachment(100, -2 * margin); + outputFieldsTableView.setLayoutData(outputFieldsTableViewFormData); + + // layout the fields tab + FormData fieldsTabFormData = new FormData(); + fieldsTabFormData.left = new FormAttachment(0, 0); + fieldsTabFormData.top = new FormAttachment(0, 0); + fieldsTabFormData.right = new FormAttachment(100, 0); + fieldsTabFormData.bottom = new FormAttachment(100, 0); + fieldsComposite.setLayoutData(metadataTabFormData); + fieldsComposite.layout(); + fieldsTab.setControl(fieldsComposite); + + // OK and cancel buttons + wOk = new Button(shell, SWT.PUSH); + wOk.setText(BaseMessages.getString(PKG, "System.Button.OK")); + wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); + + setButtonPositions(new Button[] {wOk, wCancel}, margin, null); + + // Tabfolder + FormData cTabFolderFormData = new FormData(); + cTabFolderFormData.left = new FormAttachment(0, 0); + cTabFolderFormData.top = new FormAttachment(alwaysPassInputRowButton, margin); + cTabFolderFormData.right = new FormAttachment(100, 0); + cTabFolderFormData.bottom = new FormAttachment(wOk, -margin); + cTabFolder.setLayoutData(cTabFolderFormData); + cTabFolder.setSelection(0); + + wOk.addListener(SWT.Selection, e -> ok()); + wCancel.addListener(SWT.Selection, e -> cancel()); + // default listener (for hitting "enter") + // lsDef = + // new SelectionAdapter() { + // public void widgetDefaultSelected(SelectionEvent e) { + // ok(); + // } + // }; + // wTransformName.addSelectionListener(lsDef); + // jdbcDriverField.addSelectionListener(lsDef); + + // Detect X or ALT-F4 or something that kills this window and cancel the dialog properly + shell.addShellListener( + new ShellAdapter() { + public void shellClosed(ShellEvent e) { + cancel(); + } + }); + + setSize(); + populateDialog(); + meta.setChanged(changed); + + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + + return transformName; + } + + private void selectConnectionSource(String connectionSourceOption) { + int index = JdbcMetadataMeta.getConnectionSourceOptionIndex(connectionSourceOption); + wConnectionSource.select(index); + connectionSourceUpdated(); + } + + private void setMethod(String method) { + int index = JdbcMetadataMeta.getMethodDescriptorIndex(method); + if (index == -1) throw new IllegalArgumentException("Index for method " + method + " is -1."); + methodCombo.select(index); + logBasic("setMethod called, calling parameterless method updated"); + } + + /** + * This helper method puts the step configuration stored in the meta object and puts it into the + * dialog controls. + */ + private void populateDialog() { + wTransformName.selectAll(); + String value; + + value = meta.getConnectionSource(); + selectConnectionSource(value); + + value = meta.getConnectionName(); + if (value != null) wConnectionSource.setText(value); + + value = meta.getConnectionField(); + if (value != null) connectionField.setText(value); + + value = meta.getJdbcDriverField(); + if (value != null) jdbcDriverField.setText(value); + + value = meta.getJdbcUrlField(); + if (value != null) jdbcUrlField.setText(value); + + value = meta.getJdbcUserField(); + if (value != null) jdbcUserField.setText(value); + + value = meta.getJdbcPasswordField(); + if (value != null) jdbcPasswordField.setText(value); + + alwaysPassInputRowButton.setSelection(meta.isAlwaysPassInputRow()); + + value = meta.getMethodName(); + if (value != null) setMethod(value); + + argumentSourceFields.setSelection(meta.isArgumentSourceFields()); + if (meta.getArguments() != null && meta.getArguments().size() > 0) { + methodUpdated(meta.getArguments()); + } + + logBasic("Calling methodUpdated from populate dialog."); + if (meta.getOutputFields() != null && meta.getOutputFields().size() > 0) { + updateOutputFields(meta.getOutputFields()); + } + + removeArgumentFieldsButton.setSelection(meta.isRemoveArgumentFields()); + removeArgumentFieldsButton.setEnabled(meta.isArgumentSourceFields()); + } + + /** Called when the user cancels the dialog. */ + private void cancel() { + transformName = null; + meta.setChanged(changed); + dispose(); + } + + /** Called when the user confirms the dialog */ + private void ok() { + + transformName = wTransformName.getText(); + // Save settings to the meta object + meta.setConnectionSource( + JdbcMetadataMeta.connectionSourceOptions[wConnectionSource.getSelectionIndex()]); + meta.setConnectionName(wConnectionSource.getText()); + meta.setConnectionField(connectionField.getText()); + meta.setJdbcDriverField(jdbcDriverField.getText()); + meta.setJdbcUrlField(jdbcUrlField.getText()); + meta.setJdbcUserField(jdbcUserField.getText()); + meta.setJdbcPasswordField(jdbcPasswordField.getText()); + meta.setAlwaysPassInputRow(alwaysPassInputRowButton.getSelection()); + meta.setMethodName(JdbcMetadataMeta.getMethodName(methodCombo.getSelectionIndex())); + meta.setArgumentSourceFields(argumentSourceFields.getSelection()); + meta.setArguments(getArguments()); + meta.setRemoveArgumentFields(removeArgumentFieldsButton.getSelection()); + meta.setOutputFields(getOutputFields()); + + meta.setChanged(dialogChanged || changed); + // close the SWT dialog window + dispose(); + } + + private List getArguments() { + logBasic("getArguments"); + List arguments = new ArrayList(); + Control[] controls = metadataComposite.getChildren(); + String text; + for (Control control : controls) { + if (!(control instanceof ComboVar)) continue; + ComboVar comboVar = (ComboVar) control; + text = comboVar.getText(); + arguments.add(text); + } + return arguments; + } + + private List getOutputFields() { + Table table = outputFieldsTableView.table; + List outputFields = new ArrayList<>(); + TableItem tableItem; + for (int i = 0; i < table.getItemCount(); i++) { + tableItem = table.getItem(i); + OutputField item = new OutputField(tableItem.getText(1), tableItem.getText(2)); + outputFields.add(i, item); + } + return outputFields; + } +} diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java new file mode 100644 index 00000000000..b9a92ec7cff --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java @@ -0,0 +1,829 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +import java.lang.reflect.Method; +import java.sql.DatabaseMetaData; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Transform( + id = "JdbcMetadata", + image = "jdbcmetadata.svg", + name = "i18n::JdbcMetadata.Name", + description = "i18n::JdbcMetaData.Description", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Utility", + keywords = "i18n::JdbcMetaData.keyword", + documentationUrl = "/pipeline/transforms/jdbcmetadata.html") +public class JdbcMetadataMeta extends BaseTransformMeta { + + public static Class DatabaseMetaDataClass; + + private static final Map OPTIONS_SCOPE = new HashMap(); + + static { + OPTIONS_SCOPE.put(DatabaseMetaData.bestRowSession, "bestRowSession"); + OPTIONS_SCOPE.put(DatabaseMetaData.bestRowTemporary, "bestRowTemporary"); + OPTIONS_SCOPE.put(DatabaseMetaData.bestRowTransaction, "bestRowTransaction"); + } + + // following list of COL_ static members represent columns of metadata result sets + private static final IValueMeta COL_TABLE_CAT = new ValueMetaString("TABLE_CAT"); + private static final IValueMeta COL_PKTABLE_CAT = new ValueMetaString("PKTABLE_CAT"); + private static final IValueMeta COL_FKTABLE_CAT = new ValueMetaString("FKTABLE_CAT"); + private static final IValueMeta COL_TABLE_CATALOG = new ValueMetaString("TABLE_CATALOG"); + private static final IValueMeta COL_TABLE_SCHEM = new ValueMetaString("TABLE_SCHEM"); + private static final IValueMeta COL_PKTABLE_SCHEM = new ValueMetaString("PKTABLE_SCHEM"); + private static final IValueMeta COL_FKTABLE_SCHEM = new ValueMetaString("FKTABLE_SCHEM"); + private static final IValueMeta COL_TABLE_NAME = new ValueMetaString("TABLE_NAME"); + private static final IValueMeta COL_PKTABLE_NAME = new ValueMetaString("PKTABLE_NAME"); + private static final IValueMeta COL_FKTABLE_NAME = new ValueMetaString("FKTABLE_NAME"); + private static final IValueMeta COL_TABLE_TYPE = new ValueMetaString("TABLE_TYPE"); + private static final IValueMeta COL_COLUMN_NAME = new ValueMetaString("COLUMN_NAME"); + private static final IValueMeta COL_PKCOLUMN_NAME = new ValueMetaString("PKCOLUMN_NAME"); + private static final IValueMeta COL_FKCOLUMN_NAME = new ValueMetaString("FKCOLUMN_NAME"); + private static final IValueMeta COL_PK_NAME = new ValueMetaString("PK_NAME"); + private static final IValueMeta COL_FK_NAME = new ValueMetaString("FK_NAME"); + private static final IValueMeta COL_KEY_SEQ = new ValueMetaInteger("KEY_SEQ"); + private static final IValueMeta COL_UPDATE_RULE = new ValueMetaInteger("UPDATE_RULE"); + private static final IValueMeta COL_DELETE_RULE = new ValueMetaInteger("DELETE_RULE"); + private static final IValueMeta COL_DEFERRABILITY = new ValueMetaInteger("DEFERRABILITY"); + private static final IValueMeta COL_TYPE_NAME = new ValueMetaString("TYPE_NAME"); + private static final IValueMeta COL_DATA_TYPE = new ValueMetaInteger("DATA_TYPE"); + private static final IValueMeta COL_PRECISION = new ValueMetaInteger("PRECISION"); + private static final IValueMeta COL_COLUMN_SIZE = new ValueMetaInteger("COLUMN_SIZE"); + private static final IValueMeta COL_BUFFER_LENGTH = new ValueMetaString("BUFFER_LENGTH"); + private static final IValueMeta COL_LITERAL_PREFIX = new ValueMetaString("LITERAL_PREFIX"); + private static final IValueMeta COL_LITERAL_SUFFIX = new ValueMetaString("LITERAL_SUFFIX"); + private static final IValueMeta COL_CREATE_PARAMS = new ValueMetaString("CREATE_PARAMS"); + private static final IValueMeta COL_NULLABLE = new ValueMetaInteger("NULLABLE"); + private static final IValueMeta COL_CASE_SENSITIVE = new ValueMetaBoolean("CASE_SENSITIVE"); + private static final IValueMeta COL_SEARCHABLE = new ValueMetaInteger("SEARCHABLE"); + private static final IValueMeta COL_UNSIGNED_ATTRIBUTE = + new ValueMetaBoolean("UNSIGNED_ATTRIBUTE"); + private static final IValueMeta COL_FIXED_PREC_SCALE = new ValueMetaBoolean("FIXED_PREC_SCALE"); + private static final IValueMeta COL_AUTO_INCREMENT = new ValueMetaBoolean("AUTO_INCREMENT"); + private static final IValueMeta COL_LOCAL_TYPE_NAME = new ValueMetaString("LOCAL_TYPE_NAME"); + private static final IValueMeta COL_MINIMUM_SCALE = new ValueMetaInteger("MINIMUM_SCALE"); + private static final IValueMeta COL_MAXIMUM_SCALE = new ValueMetaInteger("MAXIMUM_SCALE"); + private static final IValueMeta COL_DECIMAL_DIGITS = new ValueMetaInteger("DECIMAL_DIGITS"); + private static final IValueMeta COL_SQL_DATA_TYPE = new ValueMetaInteger("SQL_DATA_TYPE"); + private static final IValueMeta COL_SQL_DATETIME_SUB = new ValueMetaInteger("SQL_DATETIME_SUB"); + private static final IValueMeta COL_SOURCE_DATA_TYPE = new ValueMetaInteger("SOURCE_DATA_TYPE"); + private static final IValueMeta COL_NUM_PREC_RADIX = new ValueMetaInteger("NUM_PREC_RADIX"); + private static final IValueMeta COL_REMARKS = new ValueMetaString("REMARKS"); + private static final IValueMeta COL_TYPE_CAT = new ValueMetaString("TYPE_CAT"); + private static final IValueMeta COL_TYPE_SCHEM = new ValueMetaString("TYPE_SCHEM"); + private static final IValueMeta COL_SELF_REFERENCING_COL_NAME = + new ValueMetaString("SELF_REFERENCING_COL_NAME"); + private static final IValueMeta COL_REF_GENERATION = new ValueMetaString("REF_GENERATION"); + private static final IValueMeta COL_SCOPE = new ValueMetaInteger("SCOPE"); + private static final IValueMeta COL_PSEUDO_COLUMN = new ValueMetaInteger("COL_PSEUDO_COLUMN"); + private static final IValueMeta COL_GRANTOR = new ValueMetaString("GRANTOR"); + private static final IValueMeta COL_GRANTEE = new ValueMetaString("GRANTEE"); + private static final IValueMeta COL_PRIVILEGE = new ValueMetaString("PRIVILEGE"); + private static final IValueMeta COL_IS_GRANTABLE = new ValueMetaString("IS_GRANTABLE"); + private static final IValueMeta COL_COLUMN_DEF = new ValueMetaString("COLUMN_DEF"); + private static final IValueMeta COL_CHAR_OCTET_LENGTH = new ValueMetaInteger("CHAR_OCTET_LENGTH"); + private static final IValueMeta COL_ORDINAL_POSITION = new ValueMetaInteger("ORDINAL_POSITION"); + private static final IValueMeta COL_IS_NULLABLE = new ValueMetaString("IS_NULLABLE"); + private static final IValueMeta COL_SCOPE_CATALOG = new ValueMetaString("SCOPE_CATALOG"); + private static final IValueMeta COL_SCOPE_SCHEMA = new ValueMetaString("SCOPE_SCHEMA"); + private static final IValueMeta COL_SCOPE_TABLE = new ValueMetaString("SCOPE_TABLE"); + private static final IValueMeta COL_IS_AUTOINCREMENT = new ValueMetaString("IS_AUTOINCREMENT"); + private static final IValueMeta COL_IS_GENERATEDCOLUMN = + new ValueMetaString("IS_GENERATEDCOLUMN"); + + // following of argument descriptors describe arguments to metdata methods + // 1) name of the argument + // 2) java type of the argument. + private static final Object[] ARG_CATALOG = new Object[] {"catalog", String.class}; + private static final Object[] ARG_SCHEMA = new Object[] {"schema", String.class}; + private static final Object[] ARG_TABLE = new Object[] {"table", String.class}; + private static final Object[] ARG_COLUMN_NAME_PATTERN = + new Object[] {"columnNamePattern", String.class}; + private static final Object[] ARG_NULLABLE = + new Object[] {"nullable", Boolean.class, new Object[] {}}; + private static final Object[] ARG_SCHEMA_PATTERN = new Object[] {"schemaPattern", String.class}; + private static final Object[] ARG_SCOPE = new Object[] {"scope", Integer.class, OPTIONS_SCOPE}; + private static final Object[] ARG_TABLE_TYPES = new Object[] {"tableTypes", String[].class}; + private static final Object[] ARG_TABLE_NAME_PATTERN = + new Object[] {"tableNamePattern", String.class}; + private static final Object[] ARG_PARENT_CATALOG = new Object[] {"parentCatalog", String.class}; + private static final Object[] ARG_PARENT_SCHEMA = new Object[] {"parentSchema", String.class}; + private static final Object[] ARG_PARENT_TABLE = new Object[] {"parentTable", String.class}; + private static final Object[] ARG_FOREIGN_CATALOG = new Object[] {"foreignCatalog", String.class}; + private static final Object[] ARG_FOREIGN_SCHEMA = new Object[] {"foreignSchema", String.class}; + private static final Object[] ARG_FOREIGN_TABLE = new Object[] {"foreignTable", String.class}; + + // this is a map of the methods we can get metadata from. + // 1) name of the java.sql.DatabaseMetaData method + // 2) array of argument descriptors + // 3) array of return fields + // 4) initially empty slot where the actual Method object is lazily stored. + public static final Object[] methodDescriptors = + new Object[] { + new Object[] {"getCatalogs", new Object[] {}, new IValueMeta[] {COL_TABLE_CAT}, null}, + new Object[] { + "getBestRowIdentifier", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE, ARG_SCOPE, ARG_NULLABLE}, + new IValueMeta[] { + COL_SCOPE, COL_COLUMN_NAME, COL_DATA_TYPE, COL_TYPE_NAME, + COL_COLUMN_SIZE, COL_BUFFER_LENGTH, COL_DECIMAL_DIGITS, COL_PSEUDO_COLUMN + }, + null + }, + new Object[] { + "getColumnPrivileges", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE, ARG_COLUMN_NAME_PATTERN}, + new IValueMeta[] { + COL_TABLE_CAT, COL_TABLE_SCHEM, COL_TABLE_NAME, COL_COLUMN_NAME, + COL_GRANTOR, COL_GRANTEE, COL_PRIVILEGE, COL_IS_GRANTABLE + }, + null + }, + new Object[] { + "getColumns", + new Object[] { + ARG_CATALOG, ARG_SCHEMA_PATTERN, ARG_TABLE_NAME_PATTERN, ARG_COLUMN_NAME_PATTERN + }, + new IValueMeta[] { + COL_TABLE_CAT, + COL_TABLE_SCHEM, + COL_TABLE_NAME, + COL_COLUMN_NAME, + COL_DATA_TYPE, + COL_TYPE_NAME, + COL_COLUMN_SIZE, + COL_BUFFER_LENGTH, + COL_DECIMAL_DIGITS, + COL_NUM_PREC_RADIX, + COL_NULLABLE, + COL_REMARKS, + COL_COLUMN_DEF, + COL_SQL_DATA_TYPE, + COL_SQL_DATETIME_SUB, + COL_CHAR_OCTET_LENGTH, + COL_ORDINAL_POSITION, + COL_IS_NULLABLE, + COL_SCOPE_CATALOG, + COL_SCOPE_SCHEMA, + COL_SCOPE_TABLE, + COL_SOURCE_DATA_TYPE, + COL_IS_AUTOINCREMENT, + COL_IS_GENERATEDCOLUMN + }, + null + }, + new Object[] { + "getCrossReference", + new Object[] { + ARG_PARENT_CATALOG, ARG_PARENT_SCHEMA, ARG_PARENT_TABLE, + ARG_FOREIGN_CATALOG, ARG_FOREIGN_SCHEMA, ARG_FOREIGN_TABLE, + }, + new IValueMeta[] { + COL_PKTABLE_CAT, + COL_PKTABLE_SCHEM, + COL_PKTABLE_NAME, + COL_PKCOLUMN_NAME, + COL_FKTABLE_CAT, + COL_FKTABLE_SCHEM, + COL_FKTABLE_NAME, + COL_FKCOLUMN_NAME, + COL_KEY_SEQ, + COL_UPDATE_RULE, + COL_DELETE_RULE, + COL_FK_NAME, + COL_PK_NAME, + COL_DEFERRABILITY + }, + null + }, + new Object[] { + "getExportedKeys", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE}, + new IValueMeta[] { + COL_PKTABLE_CAT, + COL_PKTABLE_SCHEM, + COL_PKTABLE_NAME, + COL_PKCOLUMN_NAME, + COL_FKTABLE_CAT, + COL_FKTABLE_SCHEM, + COL_FKTABLE_NAME, + COL_FKCOLUMN_NAME, + COL_KEY_SEQ, + COL_UPDATE_RULE, + COL_DELETE_RULE, + COL_FK_NAME, + COL_PK_NAME, + COL_DEFERRABILITY + }, + null + }, + new Object[] { + "getImportedKeys", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE}, + new IValueMeta[] { + COL_PKTABLE_CAT, + COL_PKTABLE_SCHEM, + COL_PKTABLE_NAME, + COL_PKCOLUMN_NAME, + COL_FKTABLE_CAT, + COL_FKTABLE_SCHEM, + COL_FKTABLE_NAME, + COL_FKCOLUMN_NAME, + COL_KEY_SEQ, + COL_UPDATE_RULE, + COL_DELETE_RULE, + COL_FK_NAME, + COL_PK_NAME, + COL_DEFERRABILITY + }, + null + }, + new Object[] { + "getPrimaryKeys", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE}, + new IValueMeta[] { + COL_TABLE_CAT, + COL_TABLE_SCHEM, + COL_TABLE_NAME, + COL_COLUMN_NAME, + COL_KEY_SEQ, + COL_PK_NAME + }, + null + }, + new Object[] { + "getSchemas", new Object[] {}, new IValueMeta[] {COL_TABLE_SCHEM, COL_TABLE_CATALOG}, null + }, + /* We'd love to use this version of getSchemas, but we found that calling it throws AbstractMethodError in h2 and sqlite (possibly others) + new Object[]{ + "getSchemas", + new Object[]{ARG_CATALOG, ARG_SCHEMA_PATTERN}, + new IValueMeta[]{COL_TABLE_SCHEM, COL_TABLE_CATALOG}, + null + }, + */ + new Object[] { + "getTablePrivileges", + new Object[] {ARG_CATALOG, ARG_SCHEMA_PATTERN, ARG_TABLE_NAME_PATTERN}, + new IValueMeta[] { + COL_TABLE_CAT, + COL_TABLE_SCHEM, + COL_TABLE_NAME, + COL_GRANTOR, + COL_GRANTEE, + COL_PRIVILEGE, + COL_IS_GRANTABLE + }, + null + }, + new Object[] {"getTableTypes", new Object[] {}, new IValueMeta[] {COL_TABLE_TYPE}, null}, + new Object[] { + "getTables", + new Object[] {ARG_CATALOG, ARG_SCHEMA_PATTERN, ARG_TABLE_NAME_PATTERN, ARG_TABLE_TYPES}, + new IValueMeta[] { + COL_TABLE_CAT, + COL_TABLE_SCHEM, + COL_TABLE_NAME, + COL_TABLE_TYPE, + COL_REMARKS, + COL_TYPE_CAT, + COL_TYPE_SCHEM, + COL_TYPE_NAME, + COL_SELF_REFERENCING_COL_NAME, + COL_REF_GENERATION + }, + null + }, + new Object[] { + "getTypeInfo", + new Object[] {}, + new IValueMeta[] { + COL_TYPE_NAME, + COL_DATA_TYPE, + COL_PRECISION, + COL_LITERAL_PREFIX, + COL_LITERAL_SUFFIX, + COL_CREATE_PARAMS, + COL_NULLABLE, + COL_CASE_SENSITIVE, + COL_SEARCHABLE, + COL_UNSIGNED_ATTRIBUTE, + COL_FIXED_PREC_SCALE, + COL_AUTO_INCREMENT, + COL_LOCAL_TYPE_NAME, + COL_MINIMUM_SCALE, + COL_MAXIMUM_SCALE, + COL_SQL_DATA_TYPE, + COL_SQL_DATETIME_SUB, + COL_NUM_PREC_RADIX + }, + null + }, + new Object[] { + "getVersionColumns", + new Object[] {ARG_CATALOG, ARG_SCHEMA, ARG_TABLE}, + new IValueMeta[] { + COL_SCOPE, COL_COLUMN_NAME, COL_DATA_TYPE, COL_TYPE_NAME, + COL_COLUMN_SIZE, COL_BUFFER_LENGTH, COL_DECIMAL_DIGITS, COL_PSEUDO_COLUMN + }, + null + } + }; + + private static final String CONNECTION_SOURCE = "connectionSource"; + private static final String CONNECTION_NAME = "connectionName"; + private static final String CONNECTION_FIELD = "connectionField"; + private static final String JDBC_DRIVER_FIELD = "jdbcDriverField"; + private static final String JDBC_URL_FIELD = "jdbcUrlField"; + private static final String JDBC_USER_FIELD = "jdbcUserField"; + private static final String JDBC_PASSWORD_FIELD = "jdbcPasswordField"; + private static final String ALWAYS_PASS_INPUT_ROW = "alwaysPassInputRow"; + private static final String METHOD_NAME = "methodName"; + private static final String REMOVE_ARGUMENT_FIELDS = "removeArgumentFields"; + private static final String ARGUMENT_SOURCE_FIELDS = "argumentSourceFields"; + private static final String ARGUMENT = "argument"; + private static final String ARGUMENTS = ARGUMENT + "s"; + private static final String OUTPUT_FIELD = "outputField"; + private static final String OUTPUT_FIELDS = OUTPUT_FIELD + "s"; + private static final String FIELD_NAME = "name"; + private static final String FIELD_RENAME = "rename"; + + /** + * The PKG member is used when looking up internationalized strings. The properties file with + * localized keys is expected to reside in {the package of the class + * specified}/messages/messages_{locale}.properties + */ + private static Class PKG = JdbcMetadataMeta.class; // for i18n purposes + + public static final String connectionSourceOptionConnection = "Connection"; + public static final String connectionSourceOptionConnectionField = "ConnectionField"; + public static final String connectionSourceOptionJDBC = "JDBC"; + public static final String connectionSourceOptionJDBCFields = "JDBCFields"; + + public static final String[] connectionSourceOptions = + new String[] { + connectionSourceOptionConnection, + connectionSourceOptionConnectionField, + connectionSourceOptionJDBC, + connectionSourceOptionJDBCFields + }; + + /** + * Constructor should call super() to make sure the base class has a chance to initialize + * properly. + */ + @SuppressWarnings("unchecked") + public JdbcMetadataMeta() { + super(); + if (JdbcMetadataMeta.DatabaseMetaDataClass == null) { + try { + JdbcMetadataMeta.DatabaseMetaDataClass = + (Class) Class.forName("java.sql.DatabaseMetaData"); + } catch (Exception exception) { + throw new IllegalArgumentException(exception); + } + } + } + + public void setDefault() { + connectionSource = "Connection"; + connectionName = ""; + jdbcDriverField = ""; + jdbcUrlField = ""; + jdbcUserField = ""; + jdbcPasswordField = ""; + methodName = "getCatalogs"; + argumentSourceFields = false; + outputFields = new ArrayList<>(); + outputFields.add(new OutputField("TABLE_CAT","TABLE_CAT")); + } + + /** Stores the connectionsource. */ + @HopMetadataProperty + private String connectionSource; + /** + * Get the index of the connection source option + * + * @param connectionSourceOption + * @return + */ + public static int getConnectionSourceOptionIndex(String connectionSourceOption) { + for (int i = 0; i < connectionSourceOptions.length; i++) { + if (connectionSourceOptions[i].equals(connectionSourceOption)) return i; + } + return -1; + } + /** + * Getter for the name of the field containing the connection source + * + * @return the source of the connection data + */ + public String getConnectionSource() { + return connectionSource; + } + /** + * Setter for the name of the field containing the connection source + * + * @param connectionSource the source for the connection data + */ + public void setConnectionSource(String connectionSource) { + if (connectionSource == null) connectionSource = connectionSourceOptions[0]; + else if (getConnectionSourceOptionIndex(connectionSource) == -1) + throw new IllegalArgumentException( + connectionSource + " is not a valid value for connectionSource."); + this.connectionSource = connectionSource; + } + + /** Stores the name of connection. */ + @HopMetadataProperty + private String connectionName; + /** + * Getter for the name of the connection + * + * @return the name of the connection + */ + public String getConnectionName() { + return connectionName; + } + /** + * Setter for the name of the connection + * + * @param connectionName the name of the connection + */ + public void setConnectionName(String connectionName) { + this.connectionName = connectionName; + } + + /** Stores the name of field holding the connection name. */ + @HopMetadataProperty + private String connectionField; + /** + * Getter for the name of the field holding the connection name + * + * @return the name of the field holding the connection name + */ + public String getConnectionField() { + return connectionField; + } + /** + * Setter for the name of the field holding the name of the connection + * + * @param connectionField the name of the field holding the connection name + */ + public void setConnectionField(String connectionField) { + this.connectionField = connectionField; + } + /** Stores the name of the field containing the name of the jdbc driver. */ + @HopMetadataProperty + private String jdbcDriverField; + /** + * Getter for the name of the field containing the jdbc driver + * + * @return the name of the field containing the jdbc driver + */ + public String getJdbcDriverField() { + return jdbcDriverField; + } + /** + * Setter for the name of the field containing the jdbc driver + * + * @param jdbcDriverField the name of the field containing the jdbc driver + */ + public void setJdbcDriverField(String jdbcDriverField) { + this.jdbcDriverField = jdbcDriverField; + } + + /** Stores the name of the field containing the url for the jdbc connection. */ + @HopMetadataProperty + private String jdbcUrlField; + /** + * Getter for the name of the field containing the jdbc url + * + * @return the name of the field containing the jdbc url + */ + public String getJdbcUrlField() { + return jdbcUrlField; + } + /** + * Setter for the name of the field containing the jdbc url + * + * @param jdbcUrlField the name of the field containing the jdbc url + */ + public void setJdbcUrlField(String jdbcUrlField) { + this.jdbcUrlField = jdbcUrlField; + } + + /** Stores the name of the field containing the username for the jdbc connection. */ + @HopMetadataProperty + private String jdbcUserField; + /** + * Getter for the name of the field containing the jdbc user + * + * @return the name of the field containing the jdbc user + */ + public String getJdbcUserField() { + return jdbcUserField; + } + /** + * Setter for the name of the field containing the jdbc user + * + * @param jdbcUserField the name of the field containing the jdbc user + */ + public void setJdbcUserField(String jdbcUserField) { + this.jdbcUserField = jdbcUserField; + } + + /** Stores the name of the field containing the password for the jdbc connection. */ + @HopMetadataProperty + private String jdbcPasswordField; + /** + * Getter for the name of the field containing the jdbc password + * + * @return the name of the field containing the jdbc password + */ + public String getJdbcPasswordField() { + return jdbcPasswordField; + } + /** + * Setter for the name of the field containing the jdbc password + * + * @param jdbcPasswordField the name of the field containing the jdbc password + */ + public void setJdbcPasswordField(String jdbcPasswordField) { + this.jdbcPasswordField = jdbcPasswordField; + } + + /** Stores the whether the input row should be returned even if no metadata was found */ + @HopMetadataProperty + private boolean alwaysPassInputRow; + /** + * @return whether fields are to be used as arguments + */ + public boolean isAlwaysPassInputRow() { + return alwaysPassInputRow; + } + + /** + * @param alwaysPassInputRow whether fields should be used as arguments + */ + public void setAlwaysPassInputRow(boolean alwaysPassInputRow) { + this.alwaysPassInputRow = alwaysPassInputRow; + } + + /** Stores the name of the method used to get the metadata */ + @HopMetadataProperty + private String methodName; + /** + * Getter for the name method used to get metadata + * + * @return the name of the method used to get metadata + */ + public String getMethodName() { + return methodName; + } + /** + * Setter for the name of the method used to get metadata + * + * @param methodName the name of the method used to get metadata + */ + public void setMethodName(String methodName) { + this.methodName = methodName; + } + + private static Object[] getMethodDescriptor(String methodName) { + for (Object o : JdbcMetadataMeta.methodDescriptors) { + Object[] oo = (Object[]) o; + if (!oo[0].toString().equals(methodName)) continue; + return oo; + } + return null; + } + + public static Object[] getMethodDescriptor(int index) { + return (Object[]) JdbcMetadataMeta.methodDescriptors[index]; + } + + public Object[] getMethodDescriptor() { + return JdbcMetadataMeta.getMethodDescriptor(getMethodName()); + } + + public IValueMeta[] getMethodResultSetDescriptor() { + Object[] methodDescriptor = getMethodDescriptor(); + return (IValueMeta[]) methodDescriptor[2]; + } + + public static int getMethodDescriptorIndex(String methodName) { + Object[] methods = JdbcMetadataMeta.methodDescriptors; + int n = methods.length; + Object[] methodDescriptor; + String name; + for (int i = 0; i < n; i++) { + methodDescriptor = (Object[]) methods[i]; + name = (String) methodDescriptor[0]; + if (name.equals(methodName)) return i; + } + return -1; + } + + public static String getMethodName(int methodDescriptorIndex) { + Object[] methodDescriptor = + (Object[]) JdbcMetadataMeta.methodDescriptors[methodDescriptorIndex]; + String methodName = (String) methodDescriptor[0]; + return methodName; + } + + public static Class[] getMethodParameterTypes(Object[] methodDescriptor) { + Object[] parameters = (Object[]) methodDescriptor[1]; + Object[] parameter; + int n = parameters.length; + Class[] parameterTypes = new Class[n]; + for (int i = 0; i < n; i++) { + parameter = (Object[]) parameters[i]; + parameterTypes[i] = (Class) parameter[1]; + } + return parameterTypes; + } + + public static Method getMethod(String methodName) throws Exception { + Method method; + Object[] methodDescriptor = getMethodDescriptor(methodName); + method = (Method) methodDescriptor[3]; + if (method != null) return method; + Class dbmd = Class.forName("java.sql.DatabaseMetaData"); + Class[] parameterTypes = getMethodParameterTypes(methodDescriptor); + method = dbmd.getDeclaredMethod(methodName, parameterTypes); + methodDescriptor[3] = method; + return method; + } + + public Method getMethod() throws Exception { + return JdbcMetadataMeta.getMethod(methodName); + } + + /** Stores the whether fields are to be used for method arguments */ + @HopMetadataProperty + private boolean argumentSourceFields; + /** + * @return whether fields are to be used as arguments + */ + public boolean isArgumentSourceFields() { + return argumentSourceFields; + } + /** + * @param argumentSourceFields whether fields should be used as arguments + */ + public void setArgumentSourceFields(boolean argumentSourceFields) { + this.argumentSourceFields = argumentSourceFields; + } + + /** Stores the whether to remove the fields used as arguments from the output row */ + @HopMetadataProperty + private boolean removeArgumentFields; + /** + * @return whether to remove the fields used as arguments from the output row + */ + public boolean isRemoveArgumentFields() { + return removeArgumentFields; + } + /** + * @param removeArgumentFields whether fields used as arguments should be removed from the output + * row + */ + public void setRemoveArgumentFields(boolean removeArgumentFields) { + this.removeArgumentFields = removeArgumentFields; + } + + /** Stores method arguments */ + @HopMetadataProperty(groupKey = "arguments", key = "argument") + private List arguments; + /** + * @return get method arguments + */ + public List getArguments() { + return arguments; + } + + /** + * @param arguments whether fields should be used as arguments + */ + public void setArguments(List arguments) { + this.arguments = arguments; + } + + /** Stores the selection of fields that are added to the stream */ + @HopMetadataProperty(groupKey = "outputFields", key="outputField") + private List outputFields; + /** + * @return the selection of fields added to the stream + */ + public List getOutputFields() { + return outputFields; + } + + /** + * @param outputFields whether fields should be used as arguments + */ + public void setOutputFields(List outputFields) { + this.outputFields = outputFields; + } + + /** + * This method is used when a step is duplicated in Spoon. It needs to return a deep copy of this + * step meta object. Be sure to create proper deep copies if the step configuration is stored in + * modifiable objects. + * + *

See org.pentaho.di.trans.steps.rowgenerator.RowGeneratorMeta.clone() for an example on + * creating a deep copy. + * + * @return a deep copy of this + */ + public Object clone() { + Object retval = super.clone(); + return retval; + } + + public void getFields( + IRowMeta rowMeta, + String origin, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) + throws HopTransformException { + // remove argument source fields coming from the input + if (argumentSourceFields && removeArgumentFields) { + for (int i = 0; i < arguments.size(); i++) { + try { + rowMeta.removeValueMeta(arguments.get(i)); + } catch (HopValueException ex) { + // this probably means the requested field could not be found. + // we can't really handle this here; however, it's not a problem + // because a missing field will be detected before writing rows. + } + } + } + + // add the outputfields added by this step. + List outputFields = getOutputFields(); + OutputField outputField; + int n = outputFields.size(); + + Object[] methodDescriptor = getMethodDescriptor(); + IValueMeta[] fields = (IValueMeta[]) methodDescriptor[2]; + int m = fields.length; + IValueMeta field; + String fieldName; + + for (int i = 0; i < n; i++) { + outputField = outputFields.get(i); + for (int j = 0; j < m; j++) { + field = fields[j]; + fieldName = field.getName(); + if (!fieldName.equals(outputField.getName())) { + continue; + } + field = + new ValueMetaBase(outputField.getRename() == null ? fieldName : outputField.getRename(), field.getType()); + field.setOrigin(origin); + rowMeta.addValueMeta(field); + break; + } + } + } + +} diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/OutputField.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/OutputField.java new file mode 100644 index 00000000000..b3b3d7e8a9c --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/OutputField.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pipeline.transform.jdbcmetadata; + +import org.apache.hop.metadata.api.HopMetadataProperty; + +public class OutputField { + + @HopMetadataProperty + private String name; + + @HopMetadataProperty + private String rename; + + public OutputField() { + } + + public OutputField(String name, String rename) { + this.name = name; + this.rename = rename; + } + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getRename() { + return rename; + } + + public void setRename(String rename) { + this.rename = rename; + } +} diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/jdbcmetadata.svg b/plugins/transforms/jdbc-metadata/src/main/resources/jdbcmetadata.svg new file mode 100644 index 00000000000..600dbc8485f --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/resources/jdbcmetadata.svg @@ -0,0 +1,71 @@ + + + +image/svg+xml \ No newline at end of file diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties new file mode 100644 index 00000000000..b4bd0b07bf7 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +JdbcMetadata.Name=Get JDBC Metadata +JdbcMetadata.Name.Description=Get JDBC Metadata + +JdbcMetadata.connectionSource.Label=Connection Source +JdbcMetadata.connectionSource.Tooltip=Specifies how to establish a database connection +JdbcMetadata.connectionSource.options.Connection=Hop Connection +JdbcMetadata.connectionSource.options.ConnectionField=Hop connection defined by field +JdbcMetadata.connectionSource.options.JDBC=JDBC connection +JdbcMetadata.connectionSource.options.JDBCFields=JDBC connection defined by fields + +JdbcMetadata.ConnectionTab.Label=Connection +JdbcMetadata.ConnectionTab.Tooltip=Use this tab to specify how a database connection will be acquired +JdbcMetadata.MetaDataTab.Label=Method and Arguments +JdbcMetadata.MetaDataTab.Tooltip=Use this tab to specify the kind of metadata and any required parameters +JdbcMetadata.FieldsTab.Label=Output Fields +JdbcMetadata.FieldsTab.Tooltip=Use this tab to define which metadata fields you want to add to the outputstream + +JdbcMetadata.connectionField.Label=Connection field +JdbcMetadata.connectionField.Tooltip=The name of the field from the input stream that specifies a named Hop connection +JdbcMetadata.driverField.Label=Driver class +JdbcMetadata.driverField.Tooltip=The fully qualified classname of the jdbc driver +JdbcMetadata.urlField.Label=URL +JdbcMetadata.urlField.Tooltip=The driver-specific URL to establish a database connection +JdbcMetadata.userField.Label=Username +JdbcMetadata.userField.Tooltip=The database user that establishes the connection +JdbcMetadata.passwordField.Label=Password +JdbcMetadata.passwordField.Tooltip=The password of the database user that establishes the connection +JdbcMetadata.passRow.Label=Always pass the input row? +JdbcMetadata.passRow.Tooltip=Check this if you want to pass the input row even if no metadata was found. (Metadata output fields will be null in this case) +JdbcMetadata.metadataMethod.Label=MetaData Method +JdbcMetadata.metadataMethod.Tooltip=This specifies the kind of metadata to retrieve. +JdbcMetadata.argumentSource.Label=Get Arguments from fields? +JdbcMetadata.argumentSource.Tooltip=Check this if you want to acquire the values for the method arguments from the fields of the input stream +JdbcMetadata.removeArgumentFields.Label=Remove argument fields? +JdbcMetadata.removeArgumentFields.Tooltip=Check this if you do not want to pass on the method argument fields to the output stream +JdbcMetadata.getFieldsButton.Label=Get Fields +JdbcMetadata.getFieldsButton.Tooltip=Click to retrieve all possible output fields for the current metadata method +JdbcMetadata.FieldName.Label=Field name +JdbcMetadata.FieldName.Tooltip=The name of the output field as it is returned by the metadata method +JdbcMetadata.OutputFieldName.Label=Rename to +JdbcMetadata.OutputFieldName.Tooltip=The custom name for this output field + +JdbcMetadata.methods.getBestRowIdentifier=Best row identifier +JdbcMetadata.methods.getCatalogs=Catalogs +JdbcMetadata.methods.getColumnPrivileges=Column privileges +JdbcMetadata.methods.getColumns=Columns +JdbcMetadata.methods.getCrossReference=Cross references +JdbcMetadata.methods.getExportedKeys=Exported key columns +JdbcMetadata.methods.getImportedKeys=Foreign key columns +JdbcMetadata.methods.getPrimaryKeys=Primary key columns +JdbcMetadata.methods.getSchemas=Schemas +JdbcMetadata.methods.getTablePrivileges=Table privileges +JdbcMetadata.methods.getTableTypes=Table types +JdbcMetadata.methods.getTables=Tables +JdbcMetadata.methods.getTypeInfo=Data types +JdbcMetadata.methods.getVersionColumns=Version columns + +JdbcMetadata.arguments.catalog.Label=Catalog +JdbcMetadata.arguments.catalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.parentCatalog.Label=Parent catalog +JdbcMetadata.arguments.parentCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.foreignCatalog.Label=Foreign catalog +JdbcMetadata.arguments.foreignCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.columnNamePattern.Label=Column name pattern +JdbcMetadata.arguments.columnNamePattern.Tooltip=A column name pattern; must match the column name as it is stored in the database +JdbcMetadata.arguments.nullable.Label=Nullable? +JdbcMetadata.arguments.nullable.Tooltip=Include columns that are nullable +JdbcMetadata.arguments.schema.Label=Schema +JdbcMetadata.arguments.schema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.parentSchema.Label=Parent schema +JdbcMetadata.arguments.parentSchema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.foreignSchema.Label=Foreign schema +JdbcMetadata.arguments.foreignSchema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.schemaPattern.Label=Schema pattern +JdbcMetadata.arguments.schemaPattern.Tooltip=A schema pattern name; must match the schema name as it is stored in the database; +JdbcMetadata.arguments.scope.Label=Identity scope +JdbcMetadata.arguments.scope.Tooltip=The scope of interest +JdbcMetadata.arguments.table.Label=Table +JdbcMetadata.arguments.table.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.parentTable.Label=Parent table +JdbcMetadata.arguments.parentTable.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.foreignTable.Label=Foreign table +JdbcMetadata.arguments.foreignTable.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.tableNamePattern.Label=Table name pattern +JdbcMetadata.arguments.tableNamePattern.Tooltip=A table name pattern; may be a fully-qualified name +JdbcMetadata.arguments.tableTypes.Label=Table types +JdbcMetadata.arguments.tableTypes.Tooltip=A list of table types + +JdbcMetaData.CheckResult.ReceivingRows.OK=Transform is receiving input from other transforms. +JdbcMetaData.CheckResult.ReceivingRows.ERROR=No input received from other transforms! +JdbcMetaData.keyword=Utility,JDBC Metadata \ No newline at end of file diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties new file mode 100644 index 00000000000..b4bd0b07bf7 --- /dev/null +++ b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +JdbcMetadata.Name=Get JDBC Metadata +JdbcMetadata.Name.Description=Get JDBC Metadata + +JdbcMetadata.connectionSource.Label=Connection Source +JdbcMetadata.connectionSource.Tooltip=Specifies how to establish a database connection +JdbcMetadata.connectionSource.options.Connection=Hop Connection +JdbcMetadata.connectionSource.options.ConnectionField=Hop connection defined by field +JdbcMetadata.connectionSource.options.JDBC=JDBC connection +JdbcMetadata.connectionSource.options.JDBCFields=JDBC connection defined by fields + +JdbcMetadata.ConnectionTab.Label=Connection +JdbcMetadata.ConnectionTab.Tooltip=Use this tab to specify how a database connection will be acquired +JdbcMetadata.MetaDataTab.Label=Method and Arguments +JdbcMetadata.MetaDataTab.Tooltip=Use this tab to specify the kind of metadata and any required parameters +JdbcMetadata.FieldsTab.Label=Output Fields +JdbcMetadata.FieldsTab.Tooltip=Use this tab to define which metadata fields you want to add to the outputstream + +JdbcMetadata.connectionField.Label=Connection field +JdbcMetadata.connectionField.Tooltip=The name of the field from the input stream that specifies a named Hop connection +JdbcMetadata.driverField.Label=Driver class +JdbcMetadata.driverField.Tooltip=The fully qualified classname of the jdbc driver +JdbcMetadata.urlField.Label=URL +JdbcMetadata.urlField.Tooltip=The driver-specific URL to establish a database connection +JdbcMetadata.userField.Label=Username +JdbcMetadata.userField.Tooltip=The database user that establishes the connection +JdbcMetadata.passwordField.Label=Password +JdbcMetadata.passwordField.Tooltip=The password of the database user that establishes the connection +JdbcMetadata.passRow.Label=Always pass the input row? +JdbcMetadata.passRow.Tooltip=Check this if you want to pass the input row even if no metadata was found. (Metadata output fields will be null in this case) +JdbcMetadata.metadataMethod.Label=MetaData Method +JdbcMetadata.metadataMethod.Tooltip=This specifies the kind of metadata to retrieve. +JdbcMetadata.argumentSource.Label=Get Arguments from fields? +JdbcMetadata.argumentSource.Tooltip=Check this if you want to acquire the values for the method arguments from the fields of the input stream +JdbcMetadata.removeArgumentFields.Label=Remove argument fields? +JdbcMetadata.removeArgumentFields.Tooltip=Check this if you do not want to pass on the method argument fields to the output stream +JdbcMetadata.getFieldsButton.Label=Get Fields +JdbcMetadata.getFieldsButton.Tooltip=Click to retrieve all possible output fields for the current metadata method +JdbcMetadata.FieldName.Label=Field name +JdbcMetadata.FieldName.Tooltip=The name of the output field as it is returned by the metadata method +JdbcMetadata.OutputFieldName.Label=Rename to +JdbcMetadata.OutputFieldName.Tooltip=The custom name for this output field + +JdbcMetadata.methods.getBestRowIdentifier=Best row identifier +JdbcMetadata.methods.getCatalogs=Catalogs +JdbcMetadata.methods.getColumnPrivileges=Column privileges +JdbcMetadata.methods.getColumns=Columns +JdbcMetadata.methods.getCrossReference=Cross references +JdbcMetadata.methods.getExportedKeys=Exported key columns +JdbcMetadata.methods.getImportedKeys=Foreign key columns +JdbcMetadata.methods.getPrimaryKeys=Primary key columns +JdbcMetadata.methods.getSchemas=Schemas +JdbcMetadata.methods.getTablePrivileges=Table privileges +JdbcMetadata.methods.getTableTypes=Table types +JdbcMetadata.methods.getTables=Tables +JdbcMetadata.methods.getTypeInfo=Data types +JdbcMetadata.methods.getVersionColumns=Version columns + +JdbcMetadata.arguments.catalog.Label=Catalog +JdbcMetadata.arguments.catalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.parentCatalog.Label=Parent catalog +JdbcMetadata.arguments.parentCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.foreignCatalog.Label=Foreign catalog +JdbcMetadata.arguments.foreignCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database +JdbcMetadata.arguments.columnNamePattern.Label=Column name pattern +JdbcMetadata.arguments.columnNamePattern.Tooltip=A column name pattern; must match the column name as it is stored in the database +JdbcMetadata.arguments.nullable.Label=Nullable? +JdbcMetadata.arguments.nullable.Tooltip=Include columns that are nullable +JdbcMetadata.arguments.schema.Label=Schema +JdbcMetadata.arguments.schema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.parentSchema.Label=Parent schema +JdbcMetadata.arguments.parentSchema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.foreignSchema.Label=Foreign schema +JdbcMetadata.arguments.foreignSchema.Tooltip=A schema name; must match the schema name as it is stored in the database +JdbcMetadata.arguments.schemaPattern.Label=Schema pattern +JdbcMetadata.arguments.schemaPattern.Tooltip=A schema pattern name; must match the schema name as it is stored in the database; +JdbcMetadata.arguments.scope.Label=Identity scope +JdbcMetadata.arguments.scope.Tooltip=The scope of interest +JdbcMetadata.arguments.table.Label=Table +JdbcMetadata.arguments.table.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.parentTable.Label=Parent table +JdbcMetadata.arguments.parentTable.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.foreignTable.Label=Foreign table +JdbcMetadata.arguments.foreignTable.Tooltip=A table name; must match the table name as it is stored in the database +JdbcMetadata.arguments.tableNamePattern.Label=Table name pattern +JdbcMetadata.arguments.tableNamePattern.Tooltip=A table name pattern; may be a fully-qualified name +JdbcMetadata.arguments.tableTypes.Label=Table types +JdbcMetadata.arguments.tableTypes.Tooltip=A list of table types + +JdbcMetaData.CheckResult.ReceivingRows.OK=Transform is receiving input from other transforms. +JdbcMetaData.CheckResult.ReceivingRows.ERROR=No input received from other transforms! +JdbcMetaData.keyword=Utility,JDBC Metadata \ No newline at end of file diff --git a/plugins/transforms/pom.xml b/plugins/transforms/pom.xml index 3777cf58665..039cda85377 100644 --- a/plugins/transforms/pom.xml +++ b/plugins/transforms/pom.xml @@ -123,6 +123,7 @@ insertupdate janino javascript + jdbc-metadata joinrows json kafka diff --git a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java index 99a3fb83a4c..e2977fd35b2 100644 --- a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java +++ b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java @@ -225,7 +225,7 @@ public synchronized boolean setFocus() { @Override public void dispose() { - if (wCombo != null && wCombo != null) { + if (wCombo != null) { wCombo.dispose(); } } From d91bed99903a1def4941391f40f39c73009c7122 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Tue, 21 Mar 2023 10:13:16 +0100 Subject: [PATCH 06/12] fix #2752 ComboVar component not correctly disposed --- .../java/org/apache/hop/ui/core/widget/ComboVar.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java index e2977fd35b2..ab0c8616146 100644 --- a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java +++ b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java @@ -56,6 +56,7 @@ public class ComboVar extends Composite { private IVariables variables; private CCombo wCombo; + private Label wlImage; private ModifyListener modifyListenerTooltipText; @@ -99,7 +100,7 @@ public ComboVar( // Add the variable $ image on the top right of the control // - Label wlImage = new Label(this, SWT.NONE); + wlImage = new Label(this, SWT.NONE); wlImage.setImage(GuiResource.getInstance().getImageVariableMini()); wlImage.setToolTipText(BaseMessages.getString(PKG, "TextVar.tooltip.InsertVariable")); FormData fdlImage = new FormData(); @@ -225,9 +226,13 @@ public synchronized boolean setFocus() { @Override public void dispose() { - if (wCombo != null) { + if (wCombo != null && !wCombo.isDisposed()) { wCombo.dispose(); } + if (wlImage != null && !wlImage.isDisposed()) { + wlImage.dispose(); + } + super.dispose(); } @Override From f92ffe05f059c39dad10cdf1ae037a51e17f6539 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Tue, 21 Mar 2023 13:27:37 +0100 Subject: [PATCH 07/12] fix #2399 Light refactoring and bugfixing. --- .../transform/jdbcmetadata/JdbcMetadata.java | 115 ++++++------ .../jdbcmetadata/JdbcMetadataData.java | 3 +- .../jdbcmetadata/JdbcMetadataDialog.java | 176 +++++++++--------- 3 files changed, 146 insertions(+), 148 deletions(-) diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java index 36c0e45d202..2df3e35b0fc 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java @@ -29,6 +29,7 @@ import org.apache.hop.pipeline.transform.TransformMeta; import java.lang.reflect.Array; +import java.lang.reflect.Method; import java.sql.*; import java.util.HashMap; import java.util.Iterator; @@ -50,11 +51,11 @@ private Object stringToArgumentValue(String stringValue, Class type) throws IllegalArgumentException { Object argument; if (type == String.class) { - argument = (String) stringValue; + argument = stringValue; } else if (type == Boolean.class) { argument = "Y".equals(stringValue) ? Boolean.TRUE : Boolean.FALSE; } else if (type == Integer.class) { - argument = new Integer(stringValue); + argument = Integer.valueOf(stringValue); } else { throw new IllegalArgumentException("Can't handle valueType " + type.getName()); } @@ -80,8 +81,8 @@ private Object stringListToObjectArray(String stringValue, Class type) { * @throws Exception */ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exception { - // set up the method to call - /* logDebug("Setting up method to call."); + //set up the method to call + logDebug("Setting up method to call."); Method method = meta.getMethod(); data.method = method; @@ -90,16 +91,16 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc Class[] argumentTypes = method.getParameterTypes(); int argc = argumentTypes.length; logDebug("Method has " + argc + " arguments."); - List arguments = meta.getArguments(); - int argSize = arguments.size(); - logDebug("We expected " + argSize + " arguments."); - if (argc != argSize) { - throw new Exception("Method has a " + argc + " arguments, we expected " + argSize); + String[] arguments = new String[meta.getArguments().size()]; + meta.getArguments().toArray(arguments); + logDebug("We expected " + arguments.length + " arguments."); + if (argc != arguments.length) { + throw new Exception("Method has a " + argc + " arguments, we expected " + arguments.length); } logDebug("Allocating arguments array."); - data.arguments = new ArrayList(); + data.arguments = new Object[argc]; String stringArgument; - if (meta.getArgumentSourceFields()) { + if (meta.isArgumentSourceFields()) { //arguments are specified by values coming from fields. //we don't know about the fields yet, so we //initialize with bogus value so we can check if all fields were found @@ -113,10 +114,10 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc //arguments are specified directly by the user in the step. //here we convert the string values into proper argument values Class argumentType; - String argument; + Object argument; for (int i = 0; i < argc; i++) { argumentType = argumentTypes[i]; - stringArgument = arguments.get(i); + stringArgument = arguments[i]; if (stringArgument == null) { argument = null; } @@ -134,9 +135,9 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc argument = stringToArgumentValue(stringArgument, argumentType); } } - data.arguments.add(i, argument); + data.arguments[i] = argument; } - }*/ + } } /** @@ -351,20 +352,20 @@ private void prepareMethodArguments(JdbcMetadataMeta meta, JdbcMetadataData data throws Exception { // if the arguments are not from fields, then we already took care of it in the init phase, so // leave - /* if (!meta.getArgumentSourceFields()) return; - //if the arguments are from fields, then we already stored the right indices to take the values from + if (!meta.isArgumentSourceFields()) return; + // if the arguments are from fields, then we already stored the right indices to take the values + // from Object[] args = data.arguments; int[] indices = data.argumentFieldIndices; int index; Class[] argumentTypes = data.method.getParameterTypes(); Class argumentType; Object argument; - for (int i = 0; i < args.length; i++){ + for (int i = 0; i < args.length; i++) { index = indices[i]; if (index == -2) { argument = null; - } - else { + } else { argument = row[index]; } argumentType = argumentTypes[i]; @@ -372,13 +373,12 @@ private void prepareMethodArguments(JdbcMetadataMeta meta, JdbcMetadataData data if ("".equals(argument)) { logDebug("Converted empty string to null for argument array"); argument = null; - } - else { - argument = stringListToObjectArray((String)argument, argumentType.getComponentType()); + } else { + argument = stringListToObjectArray((String) argument, argumentType.getComponentType()); } } args[i] = argument; - }*/ + } } private Object[] createOutputRow( @@ -506,7 +506,7 @@ public boolean processRow() throws HopException { + "\" found for argument " + j + ": " - + ((String) arg[0])); + + arg[0]); } } else { // this argument points to a valid field. @@ -544,7 +544,7 @@ public boolean processRow() throws HopException { } // clone the input row structure and place it in our data object - data.outputRowMeta = (IRowMeta) inputRowMeta.clone(); + data.outputRowMeta = inputRowMeta.clone(); // use meta.getFields() to change it, so it reflects the output row structure meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); } // end of first @@ -555,7 +555,7 @@ public boolean processRow() throws HopException { prepareMethodArguments(meta, data, r); if (getLogLevel() == LogLevel.ROWLEVEL) { logRowlevel("About to invoke method"); - /* for (int i = 0; i < data.arguments.size(); i++) { + for (int i = 0; i < data.arguments.length; i++) { logRowlevel( "Argument " + i @@ -565,8 +565,9 @@ public boolean processRow() throws HopException { : data.arguments[i].toString() + "; " + data.arguments[i].getClass().getName())); - }*/ + } } + DatabaseMetaData databaseMetaData = connection.getMetaData(); ResultSet resultSet = (ResultSet) data.method.invoke(databaseMetaData, data.arguments); ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); @@ -575,42 +576,21 @@ public boolean processRow() throws HopException { Object value; boolean outputRows = false; int k; - /* while (resultSet.next()) { + while (resultSet.next()) { logRowlevel("Processing 1 output row."); Object[] outputRow = createOutputRow(meta, data, r); for (int i = data.outputRowOffset, j = 0; i < data.outputRowMeta.size(); i++, j++) { k = data.resultSetIndices[j]; if (k > columnCount) continue; - IValueMeta valueMetaInterface = data.outputRowMeta.getValueMeta(i); - switch (valueMetaInterface.getType()) { - case valueMetaInterface - .TYPE_BOOLEAN: // while the JDBC spec prescribes boolean, not all drivers actually - // can deliver. - boolean v; - switch (resultSetMetaData.getColumnType(k)) { - case Types.INTEGER: - case Types.SMALLINT: - case Types.TINYINT: - v = resultSet.getInt(k) == 1 ? true : false; - break; - default: - v = resultSet.getBoolean(k); - } - value = new Boolean(v); - break; - case valueMetaInterface.TYPE_INTEGER: - value = new Long(resultSet.getInt(k)); - break; - default: - value = resultSet.getObject(k); - } + int type = data.outputRowMeta.getValueMeta(i).getType(); + value = getColumnValue(resultSet, resultSetMetaData, k, type); outputRow[i] = value; outputRows = true; } // put the row to the output row stream putRow(data.outputRowMeta, outputRow); logRowlevel("Done processing 1 output row."); - }*/ + } resultSet.close(); if (!outputRows && meta.isAlwaysPassInputRow()) { Object[] outputRow = createOutputRow(meta, data, r); @@ -635,6 +615,35 @@ public boolean processRow() throws HopException { return true; } + private static Object getColumnValue( + ResultSet resultSet, ResultSetMetaData resultSetMetaData, int k, int type) + throws SQLException { + Object value; + switch (type) { + case IValueMeta + .TYPE_BOOLEAN: // while the JDBC spec prescribes boolean, not all drivers actually + // can deliver. + boolean v; + switch (resultSetMetaData.getColumnType(k)) { + case Types.INTEGER: + case Types.SMALLINT: + case Types.TINYINT: + v = resultSet.getInt(k) == 1; + break; + default: + v = resultSet.getBoolean(k); + } + value = Boolean.valueOf(v); + break; + case IValueMeta.TYPE_INTEGER: + value = Long.valueOf(resultSet.getInt(k)); + break; + default: + value = resultSet.getObject(k); + } + return value; + } + @Override public void dispose() { // clean up the database diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java index d9c3ecd0678..423d7a5293e 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java @@ -24,7 +24,6 @@ import java.lang.reflect.Method; import java.sql.Connection; -import java.util.List; import java.util.Map; public class JdbcMetadataData extends BaseTransformData implements ITransformData { @@ -37,7 +36,7 @@ public class JdbcMetadataData extends BaseTransformData implements ITransformDat // used to store the DatabaseMetaData method that generates the data public Method method; // use to store the arguments to the method - public List arguments; + public Object[] arguments; // named kettle connection cache. Used when connection is a named connection specified by input // fields public Map databases = null; diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java index d61fbcc8b53..5598d696c86 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java @@ -50,7 +50,7 @@ public class JdbcMetadataDialog extends BaseTransformDialog implements ITransformDialog { private static final Class PKG = JdbcMetadataMeta.class; // for i18n purposes - private JdbcMetadataMeta meta; + private JdbcMetadataMeta input; private int middle = props.getMiddlePct(); private int margin = PropsUi.getMargin(); @@ -95,7 +95,7 @@ public class JdbcMetadataDialog extends BaseTransformDialog implements ITransfor public JdbcMetadataDialog( Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { super(parent, variables, (BaseTransformMeta) in, pipelineMeta, sname); - meta = (JdbcMetadataMeta) in; + input = (JdbcMetadataMeta) in; } private final String[] emptyFieldList = new String[0]; @@ -145,9 +145,6 @@ private void connectionSourceUpdated() { /** Remove the UI to enter method arguments The current values are stored and returned. */ private List removeArgumentsUI() { Control[] controls = metadataComposite.getChildren(); - logBasic( - "[removeArgumentsUI] metadataComposite start - Children # -> " - + controls.length); List currentValues = new ArrayList(); for (Control control : controls) { if (control == alwaysPassInputRowLabel @@ -158,25 +155,20 @@ private List removeArgumentsUI() { || control == argumentSourceFields || control == removeArgumentFieldsLabel || control == removeArgumentFieldsButton) continue; - if (control instanceof Combo ) { - if (control.isDisposed()) { - logBasic("removeArgumentsUI - trying to access a disposed control!"); - } - if (!control.isDisposed() && (((Combo) control).getText() != null || ((Combo) control).getText().length()>0)) { - currentValues.add(((Combo) control).getText()); - logBasic("removeArgumentsUI - control.getText(): " + ((Combo) control).getText()); + if (control instanceof ComboVar) { + if (!control.isDisposed() + && (((ComboVar) control).getText() != null + || ((ComboVar) control).getText().length() > 0)) { + currentValues.add(((ComboVar) control).getText()); } } if (!control.isDisposed()) { - logBasic("removeArgumentsUI - Disposing control!"); + logDebug("removeArgumentsUI - Disposing control!"); control.dispose(); - logBasic("removeArgumentsUI - number of children in parent composite: " + controls.length); + logDebug("removeArgumentsUI - number of children in parent composite: " + controls.length); } } - - logBasic( - "[removeArgumentsUI] metadataComposite end - Children # -> " - + metadataComposite.getChildren().length); + ; return currentValues; } @@ -189,7 +181,7 @@ private List removeArgumentsUI() { * @return The combobox where the user enters the argument. */ // private ComboVar createArgumentUI( - private Combo createArgumentUI( + private ComboVar createArgumentUI( Object[] argumentDescriptor, Control lastControl, String[] items) { String argumentName = (String) argumentDescriptor[0]; Label label = new Label(metadataComposite, SWT.RIGHT); @@ -203,10 +195,8 @@ private Combo createArgumentUI( labelFormData.top = new FormAttachment(lastControl, margin); label.setLayoutData(labelFormData); -// ComboVar comboVar = -// new ComboVar(variables, metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - Combo comboVar = - new Combo (metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + ComboVar comboVar = + new ComboVar(variables, metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); props.setLook(comboVar); FormData comboVarFormData = new FormData(); comboVarFormData.left = new FormAttachment(middle, 0); @@ -221,7 +211,7 @@ private Combo createArgumentUI( } /** Create UI to enter arguments. Return a new set of arguments to store in the meta object */ private List createArgumentsUI(Object[] argumentDescriptors, List currentValues) { - logBasic( + logDebug( "createArgumentsUI, currentValues = " + (currentValues == null ? "null" : currentValues.size())); Object[] argumentDescriptor; @@ -231,8 +221,7 @@ private List createArgumentsUI(Object[] argumentDescriptors, List createArgumentsUI(Object[] argumentDescriptors, List outputFields = getOutputFields(); outputFieldsTableView.clearAll(); IValueMeta[] fields = (IValueMeta[]) methodDescriptor[2]; @@ -278,12 +267,12 @@ private void populateFieldsTable(Object[] methodDescriptor) { } private void populateFieldsTable() { - logBasic("populateFieldsTable 2"); + logDebug("populateFieldsTable 2"); populateFieldsTable(JdbcMetadataMeta.getMethodDescriptor(methodCombo.getSelectionIndex())); } private void updateOutputFields(List outputFields) { - logBasic("updateOutputFields " + outputFields); + logDebug("updateOutputFields " + outputFields); if (outputFields == null) return; outputFieldsTableView.clearAll(); OutputField outputField; @@ -311,12 +300,12 @@ private void updateOutputFields(List outputFields) { * takes care of that. */ private void methodUpdated(List argumentValues) { - logBasic( + logDebug( "methodUpdated, argumentValues = " + (argumentValues == null ? "null" : argumentValues.size())); // first, remove the controls for the previous set of arguments List currentValues = removeArgumentsUI(); - logBasic("currentValue = " + currentValues.size()); + logDebug("currentValue = " + currentValues.size()); if (argumentValues == null) { argumentValues = new ArrayList(currentValues.size()); currentValues.addAll(argumentValues); @@ -329,41 +318,33 @@ private void methodUpdated(List argumentValues) { List newArguments = createArgumentsUI(argumentDescriptors, argumentValues); // update the arguments in the meta object - meta.setArguments(newArguments); - // show / hide the argument source ui depending on whether we have arguments - boolean visible = newArguments.size() > 0; - /* (SR) Temporarily removed - argumentSourceFields.setVisible(visible); - argumentSourceLabel.setVisible(visible); - removeArgumentFieldsLabel.setVisible(visible); - removeArgumentFieldsButton.setVisible(visible); - */ + input.setArguments(newArguments); + metadataComposite.layout(); } private void methodUpdated() { - logBasic("Parameterless methodUpdated called."); + logDebug("Parameterless methodUpdated called."); methodUpdated(null); } public String open() { dialogChanged = false; - // store some convenient SWT variables Shell parent = getParent(); // SWT code for preparing the dialog shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MIN | SWT.MAX); props.setLook(shell); - setShellImage(shell, meta); + setShellImage(shell, input); - changed = meta.hasChanged(); + changed = input.hasChanged(); - lsMod = e -> meta.setChanged(); + lsMod = e -> input.setChanged(); SelectionListener lsSelection = new SelectionAdapter() { @Override public void widgetSelected(SelectionEvent e) { - meta.setChanged(); + input.setChanged(); } }; // ------------------------------------------------------- // @@ -439,7 +420,7 @@ public void widgetSelected(SelectionEvent e) { connectionComposite.setLayout(connectionTabLayout); // Connection line - DatabaseMeta databaseMeta = pipelineMeta.findDatabase(meta.getConnectionName(), variables); + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(input.getConnectionName(), variables); wConnectionSource = addConnectionLine( connectionComposite, @@ -602,7 +583,6 @@ public void widgetSelected(SelectionEvent e) { methodCombo = new Combo(metadataComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); props.setLook(methodCombo); - // methodCombo.setEditable(false); methodCombo.addModifyListener(lsMod); FormData methodComboFormData = new FormData(); methodComboFormData.left = new FormAttachment(middle, 0); @@ -625,10 +605,10 @@ public void widgetDefaultSelected(SelectionEvent selectionEvent) {} @Override public void widgetSelected(SelectionEvent selectionEvent) { - logBasic("methodCombo changed, calling parameterless methodUpdated"); + logDebug("methodCombo changed, calling parameterless methodUpdated"); methodUpdated(); populateFieldsTable(); - meta.setChanged(); + input.setChanged(); } }; methodCombo.addSelectionListener(methodComboSelectionListener); @@ -662,14 +642,22 @@ public void widgetDefaultSelected(SelectionEvent selectionEvent) {} public void widgetSelected(SelectionEvent selectionEvent) { Control[] controls = metadataComposite.getChildren(); boolean selection = argumentSourceFields.getSelection(); + removeArgumentFieldsButton.setSelection(selection); removeArgumentFieldsButton.setEnabled(selection); String[] items = selection ? getFieldListForCombo() : emptyFieldList; - for (Control control : controls) { - if (!(control instanceof ComboVar)) continue; - ComboVar comboVar = (ComboVar) control; - comboVar.setItems(items); + + if (!selection) { + removeArgumentsUI(); + } else { + methodUpdated(null); + for (Control control : controls) { + if (!(control instanceof ComboVar)) continue; + ComboVar comboVar = (ComboVar) control; + comboVar.setItems(items); + } } - meta.setChanged(); + + input.setChanged(); } }; argumentSourceFields.addSelectionListener(argumentSourceFieldsSelectionListener); @@ -767,7 +755,7 @@ public void widgetDefaultSelected(SelectionEvent arg0) {} @Override public void widgetSelected(SelectionEvent arg0) { populateFieldsTable(); - meta.setChanged(); + input.setChanged(); } }); @@ -827,7 +815,7 @@ public void shellClosed(ShellEvent e) { setSize(); populateDialog(); - meta.setChanged(changed); + input.setChanged(changed); BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); @@ -844,7 +832,7 @@ private void setMethod(String method) { int index = JdbcMetadataMeta.getMethodDescriptorIndex(method); if (index == -1) throw new IllegalArgumentException("Index for method " + method + " is -1."); methodCombo.select(index); - logBasic("setMethod called, calling parameterless method updated"); + logDebug("setMethod called, calling parameterless method updated"); } /** @@ -855,50 +843,55 @@ private void populateDialog() { wTransformName.selectAll(); String value; - value = meta.getConnectionSource(); + value = input.getConnectionSource(); selectConnectionSource(value); - value = meta.getConnectionName(); + value = input.getConnectionName(); if (value != null) wConnectionSource.setText(value); - value = meta.getConnectionField(); + value = input.getConnectionField(); if (value != null) connectionField.setText(value); - value = meta.getJdbcDriverField(); + value = input.getJdbcDriverField(); if (value != null) jdbcDriverField.setText(value); - value = meta.getJdbcUrlField(); + value = input.getJdbcUrlField(); if (value != null) jdbcUrlField.setText(value); - value = meta.getJdbcUserField(); + value = input.getJdbcUserField(); if (value != null) jdbcUserField.setText(value); - value = meta.getJdbcPasswordField(); + value = input.getJdbcPasswordField(); if (value != null) jdbcPasswordField.setText(value); - alwaysPassInputRowButton.setSelection(meta.isAlwaysPassInputRow()); + alwaysPassInputRowButton.setSelection(input.isAlwaysPassInputRow()); - value = meta.getMethodName(); + value = input.getMethodName(); if (value != null) setMethod(value); - argumentSourceFields.setSelection(meta.isArgumentSourceFields()); - if (meta.getArguments() != null && meta.getArguments().size() > 0) { - methodUpdated(meta.getArguments()); + if (input.isArgumentSourceFields()) { + argumentSourceFields.setSelection(input.isArgumentSourceFields()); + if (input.getArguments() != null && input.getArguments().size() > 0) { + methodUpdated(input.getArguments()); + } else { + // If no arguments' values are saved build the arguments' list without assigning any values + methodUpdated(new ArrayList<>()); + } } - logBasic("Calling methodUpdated from populate dialog."); - if (meta.getOutputFields() != null && meta.getOutputFields().size() > 0) { - updateOutputFields(meta.getOutputFields()); + logDebug("Calling methodUpdated from populate dialog."); + if (input.getOutputFields() != null && input.getOutputFields().size() > 0) { + updateOutputFields(input.getOutputFields()); } - removeArgumentFieldsButton.setSelection(meta.isRemoveArgumentFields()); - removeArgumentFieldsButton.setEnabled(meta.isArgumentSourceFields()); + removeArgumentFieldsButton.setSelection(input.isRemoveArgumentFields()); + removeArgumentFieldsButton.setEnabled(input.isArgumentSourceFields()); } /** Called when the user cancels the dialog. */ private void cancel() { transformName = null; - meta.setChanged(changed); + input.setChanged(changed); dispose(); } @@ -907,28 +900,25 @@ private void ok() { transformName = wTransformName.getText(); // Save settings to the meta object - meta.setConnectionSource( + input.setConnectionSource( JdbcMetadataMeta.connectionSourceOptions[wConnectionSource.getSelectionIndex()]); - meta.setConnectionName(wConnectionSource.getText()); - meta.setConnectionField(connectionField.getText()); - meta.setJdbcDriverField(jdbcDriverField.getText()); - meta.setJdbcUrlField(jdbcUrlField.getText()); - meta.setJdbcUserField(jdbcUserField.getText()); - meta.setJdbcPasswordField(jdbcPasswordField.getText()); - meta.setAlwaysPassInputRow(alwaysPassInputRowButton.getSelection()); - meta.setMethodName(JdbcMetadataMeta.getMethodName(methodCombo.getSelectionIndex())); - meta.setArgumentSourceFields(argumentSourceFields.getSelection()); - meta.setArguments(getArguments()); - meta.setRemoveArgumentFields(removeArgumentFieldsButton.getSelection()); - meta.setOutputFields(getOutputFields()); - - meta.setChanged(dialogChanged || changed); - // close the SWT dialog window + input.setConnectionName(wConnectionSource.getText()); + input.setConnectionField(connectionField.getText()); + input.setJdbcDriverField(jdbcDriverField.getText()); + input.setJdbcUrlField(jdbcUrlField.getText()); + input.setJdbcUserField(jdbcUserField.getText()); + input.setJdbcPasswordField(jdbcPasswordField.getText()); + input.setAlwaysPassInputRow(alwaysPassInputRowButton.getSelection()); + input.setMethodName(JdbcMetadataMeta.getMethodName(methodCombo.getSelectionIndex())); + input.setArgumentSourceFields(argumentSourceFields.getSelection()); + input.setArguments(getArguments()); + input.setRemoveArgumentFields(removeArgumentFieldsButton.getSelection()); + input.setOutputFields(getOutputFields()); + dispose(); } private List getArguments() { - logBasic("getArguments"); List arguments = new ArrayList(); Control[] controls = metadataComposite.getChildren(); String text; From 02790b2d76a725fc15e1740f4fe59d67c980027c Mon Sep 17 00:00:00 2001 From: sramazzina Date: Thu, 23 Mar 2023 15:28:13 +0100 Subject: [PATCH 08/12] fix #2399 Definitely finished migrating Get JDBC Metadata to Hop --- .../transform/jdbcmetadata/JdbcMetadata.java | 335 +----------------- .../jdbcmetadata/JdbcMetadataData.java | 17 +- .../jdbcmetadata/JdbcMetadataDialog.java | 297 ++-------------- .../jdbcmetadata/JdbcMetadataMeta.java | 191 +--------- .../messages/messages_en_US.properties | 29 +- .../messages/messages_it_IT.properties | 78 ++-- 6 files changed, 82 insertions(+), 865 deletions(-) diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java index 2df3e35b0fc..89ce023ef13 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java @@ -31,10 +31,7 @@ import java.lang.reflect.Array; import java.lang.reflect.Method; import java.sql.*; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.Map; public class JdbcMetadata extends BaseTransform { public JdbcMetadata( @@ -93,9 +90,9 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc logDebug("Method has " + argc + " arguments."); String[] arguments = new String[meta.getArguments().size()]; meta.getArguments().toArray(arguments); - logDebug("We expected " + arguments.length + " arguments."); + logDebug("We expected " + arguments.length + " arguments' values."); if (argc != arguments.length) { - throw new Exception("Method has a " + argc + " arguments, we expected " + arguments.length); + throw new Exception("Method has a " + argc + " arguments, we expected " + arguments.length + " values."); } logDebug("Allocating arguments array."); data.arguments = new Object[argc]; @@ -140,80 +137,6 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc } } - /** - * Utility to create a jdbc connection. (Used when the connection source is JDBC or JDBCFields) - * - * @param driver The fully qualified classname identifying the jdbc driver - * @param url The driver-specific url to create the connection - * @param user The database user that wants to establish a connection - * @param password The password required to establish the connection - * @return A jdbc connection object. - * @throws Exception - */ - private Connection createJdbcConnection(String driver, String url, String user, String password) - throws Exception { - Class.forName(driver); - return DriverManager.getConnection(url, user, password); - } - - /** - * Initialize the connection - * - * @param meta - * @param data - * @throws Exception - */ - private void initConnection(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exception { - // Try to establish a connection in advance - logDebug("Try to establish a connection in advance."); - String connectionSource = meta.getConnectionSource(); - logDebug("Connection source: " + connectionSource); - Connection connection; - - if (JdbcMetadataMeta.connectionSourceOptionConnection.equals(connectionSource)) { - // connection is a named kettle connection - DatabaseMeta dbMeta = getPipelineMeta().findDatabase(meta.getConnectionName(), variables); - Database database = new Database(this, this, dbMeta); - database.connect(); - connection = database.getConnection(); - if (connection == null) { - throw new Exception("Connection returned by database object is null!"); - } - data.database = database; - } else if (JdbcMetadataMeta.connectionSourceOptionJDBC.equals(connectionSource)) { - // connection is a user-entered jdbc connection - String jdbcDriver = variables.resolve(meta.getJdbcDriverField()); - String jdbcUrl = variables.resolve(meta.getJdbcUrlField()); - String jdbcUser = variables.resolve(meta.getJdbcUserField()); - String jdbcPassword = variables.resolve(meta.getJdbcPasswordField()); - logDebug("Attempt to create JDBC connection."); - logDebug("Driver: " + jdbcDriver); - logDebug("Url: " + jdbcUrl); - logDebug("User: " + jdbcUser); - connection = createJdbcConnection(jdbcDriver, jdbcUrl, jdbcUser, jdbcPassword); - } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { - // Connection is a jdbc connection specified by field values - // we don't know the field values yet, but we can initialize a few vars to access their value - // later on - connection = null; - data.jdbcDriverField = -1; - data.jdbcUrlField = -1; - data.jdbcUserField = -1; - data.jdbcPasswordField = -1; - } else if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { - // Connection is a named kettle connection specified by a field value - // we don't know about the field value yet, but we can initialize a var to access its value - // later on - connection = null; - data.connectionField = -1; - } else { - // should not arrive here, just initialize the connection to make the compiler shut up. - connection = null; - } - // if the connection is null at this point, then we need to establish it on a row by row basis - data.connection = connection; - } - private void initOutputFields(JdbcMetadataMeta meta, JdbcMetadataData data) { List outputFields = meta.getOutputFields(); int n = outputFields.size(); @@ -242,11 +165,14 @@ public boolean init() { if (!super.init()) return false; try { - data.databases = new HashMap(); - data.connections = new HashMap(); - data.connectionKey = new String[4]; + DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(meta.getConnection(), variables); + if (databaseMeta == null) { + logError("Database connection is missing for transform " + getTransformName()); + return false; + } + data.db = new Database(this, this, databaseMeta); + data.db.connect(); initMethod(meta, data); - initConnection(meta, data); initOutputFields(meta, data); } catch (Exception exception) { logError( @@ -260,84 +186,7 @@ public boolean init() { return result; } - private String getConnectionNameFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.connectionField]); - } - - private String getJdbcDriverFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.jdbcDriverField]); - } - - private String getJdbcUrlFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.jdbcUrlField]); - } - - private String getJdbcUserFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.jdbcUserField]); - } - - private String getJdbcPasswordFromRow(JdbcMetadataData data, Object[] row) { - return (String) (row[data.jdbcPasswordField]); - } - /** - * This is called in the processRow function to obtain the contain to apply the metadata method - * to. - * - * @param meta - * @param data - * @param row - * @return - * @throws Exception - */ - private Connection getConnection(JdbcMetadataMeta meta, JdbcMetadataData data, Object[] row) - throws Exception { - Connection connection; - if (data.connection == null) { - // connection could not be initialized at init, so we have to obtain it now on a row by row - // basis. - String connectionSource = meta.getConnectionSource(); - if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { - // connection is a named kettle connection specified by a field value - String connectionName = getConnectionNameFromRow(data, row); - // try to get this named connection from the cache - Database database = data.databases.get(connectionName); - if (database == null) { - // we haven't seen this named connection before, try to find it. - DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(connectionName, variables); - database = new Database(this, this, databaseMeta); - connection = database.getConnection(); - if (connection == null) { - throw new IllegalArgumentException("Connection returned by database is null!"); - } - // cache the database for later use - data.databases.put(connectionName, database); - } else { - // database found in cache, get its connection - connection = database.getConnection(); - } - } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { - // database connectin is a jdbc connection defined by field values. - // try to find this connection in the cache - String[] key = data.connectionKey; - key[0] = getJdbcDriverFromRow(data, row); - key[1] = getJdbcUrlFromRow(data, row); - key[2] = getJdbcUserFromRow(data, row); - key[3] = getJdbcPasswordFromRow(data, row); - connection = data.connections.get(key); - if (connection == null) { - // connection not yet in the cache. Let's create it and cahce it. - connection = createJdbcConnection(key[0], key[1], key[2], key[3]); - data.connections.put(key, connection); - } - } else { - throw new Exception("Unexpected error acquiring connection"); - } - } else { - connection = data.connection; - } - return connection; - } /** * This is called in the processRow function to get the actual arguments for the jdbc metadata @@ -414,134 +263,8 @@ public boolean processRow() throws HopException { first = false; IRowMeta inputRowMeta = getInputRowMeta(); data.outputRowOffset = inputRowMeta.size(); - String connectionSource = meta.getConnectionSource(); boolean argumentSourceFields = meta.isArgumentSourceFields(); // check if we need to use fields. - // If so, store the indices so we can easily extract their values during transformation - if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource) - || JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource) - || argumentSourceFields) { - logDebug("Looking up indices of input fields."); - String fieldName; - String[] fieldNames = inputRowMeta.getFieldNames(); - logDebug("We have " + fieldNames.length + " input fields."); - List arguments = meta.getArguments(); - int argc = arguments.size(); - String stringArgument; - for (int i = 0; i < fieldNames.length; i++) { - fieldName = fieldNames[i]; - logDebug("Looking at field: " + fieldName); - // store indices for connection source fields - if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource)) { - if (fieldName.equals(meta.getConnectionField())) { - logDebug("Found the connection field at index: " + i); - data.connectionField = i; - } - } else if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource)) { - if (fieldName.equals(meta.getJdbcDriverField())) { - logDebug("Found the jdbcDriverField field at index: " + i); - data.jdbcDriverField = i; - } - if (fieldName.equals(meta.getJdbcUrlField())) { - logDebug("Found the jdbcUrlField field at index: " + i); - data.jdbcUrlField = i; - } - if (fieldName.equals(meta.getJdbcUserField())) { - logDebug("Found the jdbcUserField field at index: " + i); - data.jdbcUserField = i; - } - if (fieldName.equals(meta.getJdbcPasswordField())) { - logDebug("Found the jdbcPasswordField field at index: " + i); - data.jdbcPasswordField = i; - } - } - // store indices for argument fields - if (argumentSourceFields) { - logDebug("Trying to match argument fields against: " + fieldName); - for (int j = 0; j < argc; j++) { - stringArgument = arguments.get(j); - logDebug("Found argument " + j + ": " + stringArgument); - if (fieldName.equals(stringArgument)) { - logDebug("Match, storing index " + i); - data.argumentFieldIndices[j] = i; - } - } - } - } // end fields loop - - // ensure that we have all required fields. - if ((JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(connectionSource) - && (data.jdbcDriverField == -1 - || data.jdbcUrlField == -1 - || data.jdbcUserField == -1 - || data.jdbcPasswordField == -1)) - || (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(connectionSource) - && data.connectionField == -1)) { - throw new HopException("Not all fields for the connection source were found."); - } - - if (argumentSourceFields) { - // keep track of how many fields we used as args. - // We need this in case remove argument fields is enabled - // as this is the number of fields we need to discard from the input row. - int fieldsUsedAsArgs = 0; - // ensure all argument fields are bound to a valid field - argumentFields: - for (int j = 0; j < argc; j++) { - logDebug("Argument indices at " + j + ": " + data.argumentFieldIndices[j]); - if (data.argumentFieldIndices[j] == -1) { - // this argument does not point to any existing field. - if (arguments.get(j) == null || arguments.get(j).length() == 0) { - // the argument is blank, this is ok: we will pass null instead - data.argumentFieldIndices[j] = -2; - } else { - // this argument is not blank - this means it points to a non-existing field. - // this is an error. - Object[] descriptor = meta.getMethodDescriptor(); - Object[] args = (Object[]) descriptor[1]; - Object[] arg = (Object[]) args[j]; - throw new HopException( - "No field \"" - + arguments.get(j) - + "\" found for argument " - + j - + ": " - + arg[0]); - } - } else { - // this argument points to a valid field. - // let's check if this same field was already used as arg: - for (int i = 0; i < j; i++) { - if (data.argumentFieldIndices[i] == data.argumentFieldIndices[j]) { - // yes, it was used already. Let's check the next argument. - continue argumentFields; - } - } - // this field was not used already, so mark it as used. - fieldsUsedAsArgs++; - } - } - - if (meta.isRemoveArgumentFields()) { - int n = data.outputRowOffset; - data.outputRowOffset -= fieldsUsedAsArgs; - data.inputFieldsToCopy = new int[data.outputRowOffset]; - - inputFieldsToCopy: - for (int i = 0, j = 0; i < n; i++) { // for each field in the input row - for (int k = 0; k < argc; k++) { // for each method argument - if (data.argumentFieldIndices[k] == i) { - // this input field is used as argument. Continue to the next field. - continue inputFieldsToCopy; - } - } - // this field was not used as argument. make sure we copy it to the output. - data.inputFieldsToCopy[j++] = i; - } - } - } - logDebug("Done looking up indices of input fields."); - } // clone the input row structure and place it in our data object data.outputRowMeta = inputRowMeta.clone(); @@ -551,7 +274,6 @@ public boolean processRow() throws HopException { try { logRowlevel("Processing 1 input row"); - Connection connection = getConnection(meta, data, r); prepareMethodArguments(meta, data, r); if (getLogLevel() == LogLevel.ROWLEVEL) { logRowlevel("About to invoke method"); @@ -568,7 +290,7 @@ public boolean processRow() throws HopException { } } - DatabaseMetaData databaseMetaData = connection.getMetaData(); + DatabaseMetaData databaseMetaData = data.db.getConnection().getMetaData(); ResultSet resultSet = (ResultSet) data.method.invoke(databaseMetaData, data.arguments); ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); int columnCount = resultSetMetaData.getColumnCount(); @@ -666,48 +388,11 @@ public void dispose() { logError("Error cleaning up connection: " + ex.getMessage()); } data.connection = null; - - // clean up the database map - Iterator> dbIterator = data.databases.entrySet().iterator(); - while (dbIterator.hasNext()) { - Map.Entry current = dbIterator.next(); - Database database = current.getValue(); - try { - database.disconnect(); - } catch (Exception ex) { - logError("Error cleaning up database " + current.getKey() + ": " + ex.getMessage()); - } - } - data.databases.clear(); - data.databases = null; - - // clean up the connection map - Iterator> connectionIterator = - data.connections.entrySet().iterator(); - while (connectionIterator.hasNext()) { - Connection connection = connectionIterator.next().getValue(); - try { - if (!connection.isClosed()) { - connection.close(); - } - } catch (Exception ex) { - logError("Error cleaning up connection: " + ex.getMessage()); - } - } - data.connections.clear(); - data.connections = null; - data.arguments = null; data.method = null; data.argumentFieldIndices = null; data.inputFieldsToCopy = null; data.resultSetIndices = null; - data.connectionKey = null; - - data.jdbcDriverField = -1; - data.jdbcUrlField = -1; - data.jdbcUserField = -1; - data.jdbcPasswordField = -1; super.dispose(); } diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java index 423d7a5293e..5fd0e09fc16 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java @@ -24,7 +24,6 @@ import java.lang.reflect.Method; import java.sql.Connection; -import java.util.Map; public class JdbcMetadataData extends BaseTransformData implements ITransformData { @@ -37,21 +36,9 @@ public class JdbcMetadataData extends BaseTransformData implements ITransformDat public Method method; // use to store the arguments to the method public Object[] arguments; - // named kettle connection cache. Used when connection is a named connection specified by input - // fields - public Map databases = null; - // connection cache. Used when connection is a jdbc connection specified by input fields - public Map connections = null; + + public Database db; // key to the connection cache. - public String[] connectionKey = null; - // field index for named kettle connection - public int connectionField = -1; - // field indices for jdbc connections - public int jdbcDriverField = -1; - public int jdbcUrlField = -1; - public int jdbcUserField = -1; - public int jdbcPasswordField = -1; - // indices for fields used to specify method arguments public int[] argumentFieldIndices; // the offset in the output row from where we can add our metadata fields. // (we need this in case we're required to remove arguments fields from the input) diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java index 5598d696c86..f88eb2438f1 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java @@ -36,10 +36,12 @@ import org.apache.hop.ui.core.widget.TableView; import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; import org.eclipse.swt.SWT; -import org.eclipse.swt.custom.CCombo; import org.eclipse.swt.custom.CTabFolder; import org.eclipse.swt.custom.CTabItem; -import org.eclipse.swt.events.*; +import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.events.SelectionListener; import org.eclipse.swt.layout.FormAttachment; import org.eclipse.swt.layout.FormData; import org.eclipse.swt.layout.FormLayout; @@ -59,19 +61,8 @@ public class JdbcMetadataDialog extends BaseTransformDialog implements ITransfor private ModifyListener lsMod; private Composite metadataComposite; - // - private MetaSelectionLine - wConnectionSource; // text field holding the name of the field containing the connection name - private CCombo connectionField; - // text field holding the name of the field containing the driver name - private ComboVar jdbcDriverField; - // text field holding the name of the field containing the url - private ComboVar jdbcUrlField; - // text field holding the name of the field containing the user - private ComboVar jdbcUserField; - // text field holding the name of the field containing the password - private ComboVar jdbcPasswordField; - // + + private MetaSelectionLine wConnection; private Label alwaysPassInputRowLabel; // @@ -111,37 +102,6 @@ private String[] getFieldListForCombo() { return items; } - private void connectionSourceUpdated() { - int selectedIndex = wConnectionSource.getSelectionIndex(); - String option = JdbcMetadataMeta.connectionSourceOptions[selectedIndex]; - boolean connectionComboEnabled, connectionFieldEnabled, otherFieldsEnabled; - otherFieldsEnabled = connectionComboEnabled = connectionFieldEnabled = false; - String[] fields = emptyFieldList; - if (JdbcMetadataMeta.connectionSourceOptionConnection.equals(option)) { - connectionComboEnabled = true; - } else { - if (JdbcMetadataMeta.connectionSourceOptionConnectionField.equals(option)) { - connectionFieldEnabled = true; - connectionField.setItems(getFieldListForCombo()); - } else { - otherFieldsEnabled = true; - if (JdbcMetadataMeta.connectionSourceOptionJDBCFields.equals(option)) { - fields = getFieldListForCombo(); - } - } - } - wConnectionSource.setEnabled(connectionComboEnabled); - connectionField.setEnabled(connectionFieldEnabled); - jdbcDriverField.setEnabled(otherFieldsEnabled); - jdbcDriverField.setItems(fields); - jdbcUrlField.setEnabled(otherFieldsEnabled); - jdbcUrlField.setItems(fields); - jdbcUserField.setEnabled(otherFieldsEnabled); - jdbcUserField.setItems(fields); - jdbcPasswordField.setEnabled(otherFieldsEnabled); - jdbcPasswordField.setItems(fields); - } - /** Remove the UI to enter method arguments The current values are stored and returned. */ private List removeArgumentsUI() { Control[] controls = metadataComposite.getChildren(); @@ -303,6 +263,7 @@ private void methodUpdated(List argumentValues) { logDebug( "methodUpdated, argumentValues = " + (argumentValues == null ? "null" : argumentValues.size())); + // first, remove the controls for the previous set of arguments List currentValues = removeArgumentsUI(); logDebug("currentValue = " + currentValues.size()); @@ -319,7 +280,6 @@ private void methodUpdated(List argumentValues) { List newArguments = createArgumentsUI(argumentDescriptors, argumentValues); // update the arguments in the meta object input.setArguments(newArguments); - metadataComposite.layout(); } @@ -380,6 +340,13 @@ public void widgetSelected(SelectionEvent e) { lastControl = wTransformName; + // Connection line + DatabaseMeta databaseMeta = pipelineMeta.findDatabase(input.getConnection(), variables); + wConnection = addConnectionLine(shell, lastControl, databaseMeta, lsMod); + wConnection.addSelectionListener(lsSelection); + + lastControl = wConnection; + // pass the row checkbox alwaysPassInputRowLabel = new Label(shell, SWT.RIGHT); alwaysPassInputRowLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.passRow.Label")); @@ -406,165 +373,14 @@ public void widgetSelected(SelectionEvent e) { CTabFolder cTabFolder = new CTabFolder(shell, SWT.BORDER); props.setLook(cTabFolder, Props.WIDGET_STYLE_TAB); - // Connection tab - CTabItem connectionTab = new CTabItem(cTabFolder, SWT.NONE); - connectionTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.ConnectionTab.Label")); - connectionTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.ConnectionTab.Tooltip")); - - FormLayout connectionTabLayout = new FormLayout(); - connectionTabLayout.marginWidth = PropsUi.getFormMargin(); - connectionTabLayout.marginHeight = PropsUi.getFormMargin(); - - Composite connectionComposite = new Composite(cTabFolder, SWT.NONE); - props.setLook(connectionComposite); - connectionComposite.setLayout(connectionTabLayout); - - // Connection line - DatabaseMeta databaseMeta = pipelineMeta.findDatabase(input.getConnectionName(), variables); - wConnectionSource = - addConnectionLine( - connectionComposite, - wTransformName, - databaseMeta, - lsMod, - BaseMessages.getString(PKG, "JdbcMetadata.connectionSource.Label"), - BaseMessages.getString(PKG, "JdbcMetadata.connectionSource.Tooltip")); - wConnectionSource.addSelectionListener(lsSelection); - lastControl = wConnectionSource; - - // connection name field - Label connectionFieldLabel = new Label(connectionComposite, SWT.RIGHT); - connectionFieldLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.connectionField.Label")); - connectionFieldLabel.setToolTipText( - BaseMessages.getString(PKG, "JdbcMetadata.connectionField.Tooltip")); - props.setLook(connectionFieldLabel); - FormData connectionFieldLabelFormData = new FormData(); - connectionFieldLabelFormData.left = new FormAttachment(0, 0); - connectionFieldLabelFormData.right = new FormAttachment(middle, -margin); - connectionFieldLabelFormData.top = new FormAttachment(lastControl, margin); - connectionFieldLabel.setLayoutData(connectionFieldLabelFormData); - - connectionField = new CCombo(connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(connectionField); - connectionField.addModifyListener(lsMod); - FormData connectionFieldFormData = new FormData(); - connectionFieldFormData.left = new FormAttachment(middle, 0); - connectionFieldFormData.right = new FormAttachment(100, 0); - connectionFieldFormData.top = new FormAttachment(lastControl, margin); - connectionField.setLayoutData(connectionFieldFormData); - - lastControl = connectionField; - - // jdbc driver field - Label jdbcDriverLabel = new Label(connectionComposite, SWT.RIGHT); - jdbcDriverLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.driverField.Label")); - jdbcDriverLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.driverField.Tooltip")); - props.setLook(jdbcDriverLabel); - FormData jdbcDriverLabelFormData = new FormData(); - jdbcDriverLabelFormData.left = new FormAttachment(0, 0); - jdbcDriverLabelFormData.right = new FormAttachment(middle, -margin); - jdbcDriverLabelFormData.top = new FormAttachment(lastControl, margin); - jdbcDriverLabel.setLayoutData(jdbcDriverLabelFormData); - - jdbcDriverField = - new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(jdbcDriverField); - jdbcDriverField.addModifyListener(lsMod); - FormData jdbcDriverFieldFormData = new FormData(); - jdbcDriverFieldFormData.left = new FormAttachment(middle, 0); - jdbcDriverFieldFormData.right = new FormAttachment(100, 0); - jdbcDriverFieldFormData.top = new FormAttachment(lastControl, margin); - jdbcDriverField.setLayoutData(jdbcDriverFieldFormData); - - lastControl = jdbcDriverField; - - // jdbc url field - Label jdbcUrlLabel = new Label(connectionComposite, SWT.RIGHT); - jdbcUrlLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.urlField.Label")); - jdbcUrlLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.urlField.Tooltip")); - props.setLook(jdbcUrlLabel); - FormData jdbcUrlLabelFormData = new FormData(); - jdbcUrlLabelFormData.left = new FormAttachment(0, 0); - jdbcUrlLabelFormData.right = new FormAttachment(middle, -margin); - jdbcUrlLabelFormData.top = new FormAttachment(lastControl, margin); - jdbcUrlLabel.setLayoutData(jdbcUrlLabelFormData); - - jdbcUrlField = new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(jdbcUrlField); - jdbcUrlField.addModifyListener(lsMod); - FormData jdbcUrlFieldFormData = new FormData(); - jdbcUrlFieldFormData.left = new FormAttachment(middle, 0); - jdbcUrlFieldFormData.right = new FormAttachment(100, 0); - jdbcUrlFieldFormData.top = new FormAttachment(lastControl, margin); - jdbcUrlField.setLayoutData(jdbcUrlFieldFormData); - - lastControl = jdbcUrlField; - - // jdbc user field - Label jdbcUserLabel = new Label(connectionComposite, SWT.RIGHT); - jdbcUserLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.userField.Label")); - jdbcUserLabel.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.userField.Tooltip")); - props.setLook(jdbcUserLabel); - FormData jdbcUserLabelFormData = new FormData(); - jdbcUserLabelFormData.left = new FormAttachment(0, 0); - jdbcUserLabelFormData.right = new FormAttachment(middle, -margin); - jdbcUserLabelFormData.top = new FormAttachment(lastControl, margin); - jdbcUserLabel.setLayoutData(jdbcUserLabelFormData); - - jdbcUserField = - new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(jdbcUserField); - jdbcUserField.addModifyListener(lsMod); - FormData jdbcUserFieldFormData = new FormData(); - jdbcUserFieldFormData.left = new FormAttachment(middle, 0); - jdbcUserFieldFormData.right = new FormAttachment(100, 0); - jdbcUserFieldFormData.top = new FormAttachment(lastControl, margin); - jdbcUserField.setLayoutData(jdbcUserFieldFormData); - - lastControl = jdbcUserField; - - // jdbc password field - Label jdbcPasswordLabel = new Label(connectionComposite, SWT.RIGHT); - jdbcPasswordLabel.setText(BaseMessages.getString(PKG, "JdbcMetadata.passwordField.Label")); - jdbcPasswordLabel.setToolTipText( - BaseMessages.getString(PKG, "JdbcMetadata.passwordField.Tooltip")); - props.setLook(jdbcPasswordLabel); - FormData jdbcPasswordLabelFormData = new FormData(); - jdbcPasswordLabelFormData.left = new FormAttachment(0, 0); - jdbcPasswordLabelFormData.right = new FormAttachment(middle, -margin); - jdbcPasswordLabelFormData.top = new FormAttachment(lastControl, margin); - jdbcPasswordLabel.setLayoutData(jdbcPasswordLabelFormData); - - jdbcPasswordField = - new ComboVar(variables, connectionComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); - props.setLook(jdbcPasswordField); - jdbcPasswordField.addModifyListener(lsMod); - FormData jdbcPasswordFieldFormData = new FormData(); - jdbcPasswordFieldFormData.left = new FormAttachment(middle, 0); - jdbcPasswordFieldFormData.right = new FormAttachment(100, 0); - jdbcPasswordFieldFormData.top = new FormAttachment(lastControl, margin); - jdbcPasswordField.setLayoutData(jdbcPasswordFieldFormData); - - lastControl = jdbcPasswordField; - - // layout the connection tab - FormData connectionTabFormData = new FormData(); - connectionTabFormData.left = new FormAttachment(0, 0); - connectionTabFormData.top = new FormAttachment(0, 0); - connectionTabFormData.right = new FormAttachment(100, 0); - connectionTabFormData.bottom = new FormAttachment(100, 0); - connectionComposite.setLayoutData(connectionTabFormData); - connectionComposite.layout(); - connectionTab.setControl(connectionComposite); - // Metadata tab CTabItem metadataTab = new CTabItem(cTabFolder, SWT.NONE); metadataTab.setText(BaseMessages.getString(PKG, "JdbcMetadata.MetaDataTab.Label")); metadataTab.setToolTipText(BaseMessages.getString(PKG, "JdbcMetadata.MetaDataTab.Tooltip")); FormLayout metadataTabLayout = new FormLayout(); - connectionTabLayout.marginWidth = Const.FORM_MARGIN; - connectionTabLayout.marginHeight = Const.FORM_MARGIN; + metadataTabLayout.marginWidth = Const.FORM_MARGIN; + metadataTabLayout.marginHeight = Const.FORM_MARGIN; metadataComposite = new Composite(cTabFolder, SWT.NONE); props.setLook(metadataComposite); @@ -646,15 +462,10 @@ public void widgetSelected(SelectionEvent selectionEvent) { removeArgumentFieldsButton.setEnabled(selection); String[] items = selection ? getFieldListForCombo() : emptyFieldList; - if (!selection) { - removeArgumentsUI(); - } else { - methodUpdated(null); - for (Control control : controls) { - if (!(control instanceof ComboVar)) continue; - ComboVar comboVar = (ComboVar) control; - comboVar.setItems(items); - } + for (Control control : controls) { + if (!(control instanceof ComboVar)) continue; + ComboVar comboVar = (ComboVar) control; + comboVar.setItems(items); } input.setChanged(); @@ -795,23 +606,6 @@ public void widgetSelected(SelectionEvent arg0) { wOk.addListener(SWT.Selection, e -> ok()); wCancel.addListener(SWT.Selection, e -> cancel()); - // default listener (for hitting "enter") - // lsDef = - // new SelectionAdapter() { - // public void widgetDefaultSelected(SelectionEvent e) { - // ok(); - // } - // }; - // wTransformName.addSelectionListener(lsDef); - // jdbcDriverField.addSelectionListener(lsDef); - - // Detect X or ALT-F4 or something that kills this window and cancel the dialog properly - shell.addShellListener( - new ShellAdapter() { - public void shellClosed(ShellEvent e) { - cancel(); - } - }); setSize(); populateDialog(); @@ -822,12 +616,6 @@ public void shellClosed(ShellEvent e) { return transformName; } - private void selectConnectionSource(String connectionSourceOption) { - int index = JdbcMetadataMeta.getConnectionSourceOptionIndex(connectionSourceOption); - wConnectionSource.select(index); - connectionSourceUpdated(); - } - private void setMethod(String method) { int index = JdbcMetadataMeta.getMethodDescriptorIndex(method); if (index == -1) throw new IllegalArgumentException("Index for method " + method + " is -1."); @@ -843,40 +631,20 @@ private void populateDialog() { wTransformName.selectAll(); String value; - value = input.getConnectionSource(); - selectConnectionSource(value); - - value = input.getConnectionName(); - if (value != null) wConnectionSource.setText(value); - - value = input.getConnectionField(); - if (value != null) connectionField.setText(value); - - value = input.getJdbcDriverField(); - if (value != null) jdbcDriverField.setText(value); - - value = input.getJdbcUrlField(); - if (value != null) jdbcUrlField.setText(value); - - value = input.getJdbcUserField(); - if (value != null) jdbcUserField.setText(value); - - value = input.getJdbcPasswordField(); - if (value != null) jdbcPasswordField.setText(value); + value = input.getConnection(); + if (value != null) wConnection.setText(value); alwaysPassInputRowButton.setSelection(input.isAlwaysPassInputRow()); value = input.getMethodName(); if (value != null) setMethod(value); - if (input.isArgumentSourceFields()) { - argumentSourceFields.setSelection(input.isArgumentSourceFields()); - if (input.getArguments() != null && input.getArguments().size() > 0) { - methodUpdated(input.getArguments()); - } else { - // If no arguments' values are saved build the arguments' list without assigning any values - methodUpdated(new ArrayList<>()); - } + argumentSourceFields.setSelection(input.isArgumentSourceFields()); + if (input.getArguments() != null && input.getArguments().size() > 0) { + methodUpdated(input.getArguments()); + } else { + // If no arguments' values are saved build the arguments' list without assigning any values + methodUpdated(); } logDebug("Calling methodUpdated from populate dialog."); @@ -900,14 +668,7 @@ private void ok() { transformName = wTransformName.getText(); // Save settings to the meta object - input.setConnectionSource( - JdbcMetadataMeta.connectionSourceOptions[wConnectionSource.getSelectionIndex()]); - input.setConnectionName(wConnectionSource.getText()); - input.setConnectionField(connectionField.getText()); - input.setJdbcDriverField(jdbcDriverField.getText()); - input.setJdbcUrlField(jdbcUrlField.getText()); - input.setJdbcUserField(jdbcUserField.getText()); - input.setJdbcPasswordField(jdbcPasswordField.getText()); + input.setConnection(wConnection.getText()); input.setAlwaysPassInputRow(alwaysPassInputRowButton.getSelection()); input.setMethodName(JdbcMetadataMeta.getMethodName(methodCombo.getSelectionIndex())); input.setArgumentSourceFields(argumentSourceFields.getSelection()); diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java index b9a92ec7cff..efabbfd6db3 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataMeta.java @@ -365,24 +365,6 @@ public class JdbcMetadataMeta extends BaseTransformMeta PKG = JdbcMetadataMeta.class; // for i18n purposes - public static final String connectionSourceOptionConnection = "Connection"; - public static final String connectionSourceOptionConnectionField = "ConnectionField"; - public static final String connectionSourceOptionJDBC = "JDBC"; - public static final String connectionSourceOptionJDBCFields = "JDBCFields"; - - public static final String[] connectionSourceOptions = - new String[] { - connectionSourceOptionConnection, - connectionSourceOptionConnectionField, - connectionSourceOptionJDBC, - connectionSourceOptionJDBCFields - }; - /** * Constructor should call super() to make sure the base class has a chance to initialize * properly. @@ -421,171 +390,21 @@ public JdbcMetadataMeta() { } public void setDefault() { - connectionSource = "Connection"; - connectionName = ""; - jdbcDriverField = ""; - jdbcUrlField = ""; - jdbcUserField = ""; - jdbcPasswordField = ""; methodName = "getCatalogs"; argumentSourceFields = false; outputFields = new ArrayList<>(); outputFields.add(new OutputField("TABLE_CAT","TABLE_CAT")); } - /** Stores the connectionsource. */ - @HopMetadataProperty - private String connectionSource; - /** - * Get the index of the connection source option - * - * @param connectionSourceOption - * @return - */ - public static int getConnectionSourceOptionIndex(String connectionSourceOption) { - for (int i = 0; i < connectionSourceOptions.length; i++) { - if (connectionSourceOptions[i].equals(connectionSourceOption)) return i; - } - return -1; - } - /** - * Getter for the name of the field containing the connection source - * - * @return the source of the connection data - */ - public String getConnectionSource() { - return connectionSource; - } - /** - * Setter for the name of the field containing the connection source - * - * @param connectionSource the source for the connection data - */ - public void setConnectionSource(String connectionSource) { - if (connectionSource == null) connectionSource = connectionSourceOptions[0]; - else if (getConnectionSourceOptionIndex(connectionSource) == -1) - throw new IllegalArgumentException( - connectionSource + " is not a valid value for connectionSource."); - this.connectionSource = connectionSource; - } - - /** Stores the name of connection. */ - @HopMetadataProperty - private String connectionName; - /** - * Getter for the name of the connection - * - * @return the name of the connection - */ - public String getConnectionName() { - return connectionName; - } - /** - * Setter for the name of the connection - * - * @param connectionName the name of the connection - */ - public void setConnectionName(String connectionName) { - this.connectionName = connectionName; - } - - /** Stores the name of field holding the connection name. */ - @HopMetadataProperty - private String connectionField; - /** - * Getter for the name of the field holding the connection name - * - * @return the name of the field holding the connection name - */ - public String getConnectionField() { - return connectionField; - } - /** - * Setter for the name of the field holding the name of the connection - * - * @param connectionField the name of the field holding the connection name - */ - public void setConnectionField(String connectionField) { - this.connectionField = connectionField; - } - /** Stores the name of the field containing the name of the jdbc driver. */ - @HopMetadataProperty - private String jdbcDriverField; - /** - * Getter for the name of the field containing the jdbc driver - * - * @return the name of the field containing the jdbc driver - */ - public String getJdbcDriverField() { - return jdbcDriverField; - } - /** - * Setter for the name of the field containing the jdbc driver - * - * @param jdbcDriverField the name of the field containing the jdbc driver - */ - public void setJdbcDriverField(String jdbcDriverField) { - this.jdbcDriverField = jdbcDriverField; - } - - /** Stores the name of the field containing the url for the jdbc connection. */ @HopMetadataProperty - private String jdbcUrlField; - /** - * Getter for the name of the field containing the jdbc url - * - * @return the name of the field containing the jdbc url - */ - public String getJdbcUrlField() { - return jdbcUrlField; - } - /** - * Setter for the name of the field containing the jdbc url - * - * @param jdbcUrlField the name of the field containing the jdbc url - */ - public void setJdbcUrlField(String jdbcUrlField) { - this.jdbcUrlField = jdbcUrlField; - } + private String connection; - /** Stores the name of the field containing the username for the jdbc connection. */ - @HopMetadataProperty - private String jdbcUserField; - /** - * Getter for the name of the field containing the jdbc user - * - * @return the name of the field containing the jdbc user - */ - public String getJdbcUserField() { - return jdbcUserField; - } - /** - * Setter for the name of the field containing the jdbc user - * - * @param jdbcUserField the name of the field containing the jdbc user - */ - public void setJdbcUserField(String jdbcUserField) { - this.jdbcUserField = jdbcUserField; + public String getConnection() { + return connection; } - /** Stores the name of the field containing the password for the jdbc connection. */ - @HopMetadataProperty - private String jdbcPasswordField; - /** - * Getter for the name of the field containing the jdbc password - * - * @return the name of the field containing the jdbc password - */ - public String getJdbcPasswordField() { - return jdbcPasswordField; - } - /** - * Setter for the name of the field containing the jdbc password - * - * @param jdbcPasswordField the name of the field containing the jdbc password - */ - public void setJdbcPasswordField(String jdbcPasswordField) { - this.jdbcPasswordField = jdbcPasswordField; + public void setConnection(String connection) { + this.connection = connection; } /** Stores the whether the input row should be returned even if no metadata was found */ diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties index b4bd0b07bf7..bfdfef7c57a 100644 --- a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties +++ b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_en_US.properties @@ -18,39 +18,22 @@ JdbcMetadata.Name=Get JDBC Metadata JdbcMetadata.Name.Description=Get JDBC Metadata -JdbcMetadata.connectionSource.Label=Connection Source -JdbcMetadata.connectionSource.Tooltip=Specifies how to establish a database connection -JdbcMetadata.connectionSource.options.Connection=Hop Connection -JdbcMetadata.connectionSource.options.ConnectionField=Hop connection defined by field -JdbcMetadata.connectionSource.options.JDBC=JDBC connection -JdbcMetadata.connectionSource.options.JDBCFields=JDBC connection defined by fields -JdbcMetadata.ConnectionTab.Label=Connection -JdbcMetadata.ConnectionTab.Tooltip=Use this tab to specify how a database connection will be acquired -JdbcMetadata.MetaDataTab.Label=Method and Arguments +JdbcMetadata.MetaDataTab.Label=Method and arguments JdbcMetadata.MetaDataTab.Tooltip=Use this tab to specify the kind of metadata and any required parameters -JdbcMetadata.FieldsTab.Label=Output Fields +JdbcMetadata.FieldsTab.Label=Output fields JdbcMetadata.FieldsTab.Tooltip=Use this tab to define which metadata fields you want to add to the outputstream -JdbcMetadata.connectionField.Label=Connection field -JdbcMetadata.connectionField.Tooltip=The name of the field from the input stream that specifies a named Hop connection -JdbcMetadata.driverField.Label=Driver class -JdbcMetadata.driverField.Tooltip=The fully qualified classname of the jdbc driver -JdbcMetadata.urlField.Label=URL -JdbcMetadata.urlField.Tooltip=The driver-specific URL to establish a database connection -JdbcMetadata.userField.Label=Username -JdbcMetadata.userField.Tooltip=The database user that establishes the connection -JdbcMetadata.passwordField.Label=Password -JdbcMetadata.passwordField.Tooltip=The password of the database user that establishes the connection + JdbcMetadata.passRow.Label=Always pass the input row? JdbcMetadata.passRow.Tooltip=Check this if you want to pass the input row even if no metadata was found. (Metadata output fields will be null in this case) -JdbcMetadata.metadataMethod.Label=MetaData Method +JdbcMetadata.metadataMethod.Label=Metadata method JdbcMetadata.metadataMethod.Tooltip=This specifies the kind of metadata to retrieve. -JdbcMetadata.argumentSource.Label=Get Arguments from fields? +JdbcMetadata.argumentSource.Label=Get arguments from fields? JdbcMetadata.argumentSource.Tooltip=Check this if you want to acquire the values for the method arguments from the fields of the input stream JdbcMetadata.removeArgumentFields.Label=Remove argument fields? JdbcMetadata.removeArgumentFields.Tooltip=Check this if you do not want to pass on the method argument fields to the output stream -JdbcMetadata.getFieldsButton.Label=Get Fields +JdbcMetadata.getFieldsButton.Label=Get fields JdbcMetadata.getFieldsButton.Tooltip=Click to retrieve all possible output fields for the current metadata method JdbcMetadata.FieldName.Label=Field name JdbcMetadata.FieldName.Tooltip=The name of the output field as it is returned by the metadata method diff --git a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties index b4bd0b07bf7..72e2b4b0d54 100644 --- a/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties +++ b/plugins/transforms/jdbc-metadata/src/main/resources/org/apache/pipeline/transform/jdbcmetadata/messages/messages_it_IT.properties @@ -18,44 +18,26 @@ JdbcMetadata.Name=Get JDBC Metadata JdbcMetadata.Name.Description=Get JDBC Metadata -JdbcMetadata.connectionSource.Label=Connection Source -JdbcMetadata.connectionSource.Tooltip=Specifies how to establish a database connection -JdbcMetadata.connectionSource.options.Connection=Hop Connection -JdbcMetadata.connectionSource.options.ConnectionField=Hop connection defined by field -JdbcMetadata.connectionSource.options.JDBC=JDBC connection -JdbcMetadata.connectionSource.options.JDBCFields=JDBC connection defined by fields +JdbcMetadata.MetaDataTab.Label=Metodi ed argomenti +JdbcMetadata.MetaDataTab.Tooltip=Usa questo tab per specificare il tipo dei metadati ed inserire i parametri richiesti +JdbcMetadata.FieldsTab.Label=Campi di output +JdbcMetadata.FieldsTab.Tooltip=Usa questo tab per definire quali campi dei metadati di output vuoi aggiungere allo stream di uscita -JdbcMetadata.ConnectionTab.Label=Connection -JdbcMetadata.ConnectionTab.Tooltip=Use this tab to specify how a database connection will be acquired -JdbcMetadata.MetaDataTab.Label=Method and Arguments -JdbcMetadata.MetaDataTab.Tooltip=Use this tab to specify the kind of metadata and any required parameters -JdbcMetadata.FieldsTab.Label=Output Fields -JdbcMetadata.FieldsTab.Tooltip=Use this tab to define which metadata fields you want to add to the outputstream -JdbcMetadata.connectionField.Label=Connection field -JdbcMetadata.connectionField.Tooltip=The name of the field from the input stream that specifies a named Hop connection -JdbcMetadata.driverField.Label=Driver class -JdbcMetadata.driverField.Tooltip=The fully qualified classname of the jdbc driver -JdbcMetadata.urlField.Label=URL -JdbcMetadata.urlField.Tooltip=The driver-specific URL to establish a database connection -JdbcMetadata.userField.Label=Username -JdbcMetadata.userField.Tooltip=The database user that establishes the connection -JdbcMetadata.passwordField.Label=Password -JdbcMetadata.passwordField.Tooltip=The password of the database user that establishes the connection -JdbcMetadata.passRow.Label=Always pass the input row? -JdbcMetadata.passRow.Tooltip=Check this if you want to pass the input row even if no metadata was found. (Metadata output fields will be null in this case) -JdbcMetadata.metadataMethod.Label=MetaData Method -JdbcMetadata.metadataMethod.Tooltip=This specifies the kind of metadata to retrieve. -JdbcMetadata.argumentSource.Label=Get Arguments from fields? -JdbcMetadata.argumentSource.Tooltip=Check this if you want to acquire the values for the method arguments from the fields of the input stream -JdbcMetadata.removeArgumentFields.Label=Remove argument fields? -JdbcMetadata.removeArgumentFields.Tooltip=Check this if you do not want to pass on the method argument fields to the output stream -JdbcMetadata.getFieldsButton.Label=Get Fields -JdbcMetadata.getFieldsButton.Tooltip=Click to retrieve all possible output fields for the current metadata method -JdbcMetadata.FieldName.Label=Field name -JdbcMetadata.FieldName.Tooltip=The name of the output field as it is returned by the metadata method -JdbcMetadata.OutputFieldName.Label=Rename to -JdbcMetadata.OutputFieldName.Tooltip=The custom name for this output field +JdbcMetadata.passRow.Label=Fai sempre passare una riga? +JdbcMetadata.passRow.Tooltip=Seleziona l''opzione se vuoi fare passare sempre le righe di input anche se non ci sono metadati in output (i campi in output dovuti ai metadati saranno tutti null in questo caso) +JdbcMetadata.metadataMethod.Label=Metodo da richiamare +JdbcMetadata.metadataMethod.Tooltip=Specifica il tipo di metadati da recuperare. +JdbcMetadata.argumentSource.Label=Valore degli argomenti dai campi? +JdbcMetadata.argumentSource.Tooltip=Seleziona l''opzione se vuoi acquisire i valori per gli argomenti dei metodi da campi dello stream di input +JdbcMetadata.removeArgumentFields.Label=Rimuove i campi degli argomenti? +JdbcMetadata.removeArgumentFields.Tooltip=Seleziona l''opzione se non vuoi passare gli argomenti della chiamata al metodo nello stream di output +JdbcMetadata.getFieldsButton.Label=Estrai campi +JdbcMetadata.getFieldsButton.Tooltip=Premi per estrarre tutti i possibili campi di output per il metodo di DatabaseMetadata selezionato +JdbcMetadata.FieldName.Label=Nome campo +JdbcMetadata.FieldName.Tooltip=Nome del campo di output come viene restituito dal metodo di DatabaseMetadata +JdbcMetadata.OutputFieldName.Label=Rinomina a +JdbcMetadata.OutputFieldName.Tooltip=Nome custom il campo di output JdbcMetadata.methods.getBestRowIdentifier=Best row identifier JdbcMetadata.methods.getCatalogs=Catalogs @@ -72,35 +54,35 @@ JdbcMetadata.methods.getTables=Tables JdbcMetadata.methods.getTypeInfo=Data types JdbcMetadata.methods.getVersionColumns=Version columns -JdbcMetadata.arguments.catalog.Label=Catalog +JdbcMetadata.arguments.catalog.Label=Catalogo JdbcMetadata.arguments.catalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database -JdbcMetadata.arguments.parentCatalog.Label=Parent catalog +JdbcMetadata.arguments.parentCatalog.Label=Catalogo parent JdbcMetadata.arguments.parentCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database -JdbcMetadata.arguments.foreignCatalog.Label=Foreign catalog +JdbcMetadata.arguments.foreignCatalog.Label=Catalogo foreign JdbcMetadata.arguments.foreignCatalog.Tooltip=A catalog name; must match the catalog name as it is stored in the database -JdbcMetadata.arguments.columnNamePattern.Label=Column name pattern +JdbcMetadata.arguments.columnNamePattern.Label=Colonna (pattern) JdbcMetadata.arguments.columnNamePattern.Tooltip=A column name pattern; must match the column name as it is stored in the database JdbcMetadata.arguments.nullable.Label=Nullable? JdbcMetadata.arguments.nullable.Tooltip=Include columns that are nullable JdbcMetadata.arguments.schema.Label=Schema JdbcMetadata.arguments.schema.Tooltip=A schema name; must match the schema name as it is stored in the database -JdbcMetadata.arguments.parentSchema.Label=Parent schema +JdbcMetadata.arguments.parentSchema.Label=Schema parent JdbcMetadata.arguments.parentSchema.Tooltip=A schema name; must match the schema name as it is stored in the database -JdbcMetadata.arguments.foreignSchema.Label=Foreign schema +JdbcMetadata.arguments.foreignSchema.Label=Schema foreign JdbcMetadata.arguments.foreignSchema.Tooltip=A schema name; must match the schema name as it is stored in the database -JdbcMetadata.arguments.schemaPattern.Label=Schema pattern +JdbcMetadata.arguments.schemaPattern.Label=Schema (pattern) JdbcMetadata.arguments.schemaPattern.Tooltip=A schema pattern name; must match the schema name as it is stored in the database; JdbcMetadata.arguments.scope.Label=Identity scope JdbcMetadata.arguments.scope.Tooltip=The scope of interest -JdbcMetadata.arguments.table.Label=Table +JdbcMetadata.arguments.table.Label=Tabella JdbcMetadata.arguments.table.Tooltip=A table name; must match the table name as it is stored in the database -JdbcMetadata.arguments.parentTable.Label=Parent table +JdbcMetadata.arguments.parentTable.Label=Tabella padre JdbcMetadata.arguments.parentTable.Tooltip=A table name; must match the table name as it is stored in the database -JdbcMetadata.arguments.foreignTable.Label=Foreign table +JdbcMetadata.arguments.foreignTable.Label=Tabella foreign JdbcMetadata.arguments.foreignTable.Tooltip=A table name; must match the table name as it is stored in the database -JdbcMetadata.arguments.tableNamePattern.Label=Table name pattern +JdbcMetadata.arguments.tableNamePattern.Label=Tabella (pattern) JdbcMetadata.arguments.tableNamePattern.Tooltip=A table name pattern; may be a fully-qualified name -JdbcMetadata.arguments.tableTypes.Label=Table types +JdbcMetadata.arguments.tableTypes.Label=Tipi di tabella JdbcMetadata.arguments.tableTypes.Tooltip=A list of table types JdbcMetaData.CheckResult.ReceivingRows.OK=Transform is receiving input from other transforms. From 6d4d3b28a640252a9c822e58ddf95cb10dbef8cb Mon Sep 17 00:00:00 2001 From: Hans Van Akelyen Date: Fri, 31 Mar 2023 09:13:07 +0200 Subject: [PATCH 09/12] #2399 update version and rebase --- assemblies/plugins/transforms/jdbc-metadata/pom.xml | 4 ++-- plugins/transforms/jdbc-metadata/pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assemblies/plugins/transforms/jdbc-metadata/pom.xml b/assemblies/plugins/transforms/jdbc-metadata/pom.xml index 564eac05d76..489fd1d7ad1 100644 --- a/assemblies/plugins/transforms/jdbc-metadata/pom.xml +++ b/assemblies/plugins/transforms/jdbc-metadata/pom.xml @@ -23,12 +23,12 @@ org.apache.hop hop-assemblies-plugins-transforms - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT hop-assemblies-plugins-transforms-jdbc-metadata - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT pom Hop Assemblies Plugins Transforms JDBC Metadata diff --git a/plugins/transforms/jdbc-metadata/pom.xml b/plugins/transforms/jdbc-metadata/pom.xml index e68fd34c6ae..4ebdd3e1d02 100644 --- a/plugins/transforms/jdbc-metadata/pom.xml +++ b/plugins/transforms/jdbc-metadata/pom.xml @@ -23,7 +23,7 @@ org.apache.hop hop-plugins-transforms - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT hop-transform-jdbc-metadata From 9659615ad860a91190d65cd9779587bf182e9e34 Mon Sep 17 00:00:00 2001 From: Hans Van Akelyen Date: Fri, 31 Mar 2023 14:50:30 +0200 Subject: [PATCH 10/12] #2399 add plugin to dist --- assemblies/plugins/dist/pom.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/assemblies/plugins/dist/pom.xml b/assemblies/plugins/dist/pom.xml index 5596ab767b2..d3e873017dd 100644 --- a/assemblies/plugins/dist/pom.xml +++ b/assemblies/plugins/dist/pom.xml @@ -1570,6 +1570,19 @@ + + org.apache.hop + hop-assemblies-plugins-transforms-jdbc-metadata + ${project.version} + zip + + + * + * + + + + org.apache.hop hop-assemblies-plugins-transforms-joinrows From 365b4c380f80d53e81548e14d940fe60d90a5182 Mon Sep 17 00:00:00 2001 From: sramazzina Date: Mon, 17 Apr 2023 15:08:03 +0200 Subject: [PATCH 11/12] fix #2399 Added support to run transform without any input flow wherever needed. [ITA-TRANS] Fixed a minor typo in Italian translation --- .../transform/jdbcmetadata/JdbcMetadata.java | 164 ++++++++++++++---- .../jdbcmetadata/JdbcMetadataData.java | 2 +- .../jdbcmetadata/JdbcMetadataDialog.java | 1 + .../hopgui/messages/messages_it_IT.properties | 2 +- 4 files changed, 130 insertions(+), 39 deletions(-) diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java index 89ce023ef13..1d9cef64933 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadata.java @@ -17,23 +17,26 @@ package org.apache.pipeline.transform.jdbcmetadata; +import java.lang.reflect.Array; +import java.lang.reflect.Method; +import java.sql.*; +import java.util.List; import org.apache.hop.core.database.Database; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.logging.LogLevel; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; import org.apache.hop.pipeline.Pipeline; import org.apache.hop.pipeline.PipelineMeta; import org.apache.hop.pipeline.transform.BaseTransform; import org.apache.hop.pipeline.transform.TransformMeta; -import java.lang.reflect.Array; -import java.lang.reflect.Method; -import java.sql.*; -import java.util.List; - public class JdbcMetadata extends BaseTransform { + + boolean treatAsInputTransform = false; + public JdbcMetadata( TransformMeta transformMeta, JdbcMetadataMeta meta, @@ -78,38 +81,41 @@ private Object stringListToObjectArray(String stringValue, Class type) { * @throws Exception */ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exception { - //set up the method to call + // Set up the method to call logDebug("Setting up method to call."); Method method = meta.getMethod(); data.method = method; - //Try to set up the arguments for the method + // Try to set up the arguments for the method logDebug("Setting up method arguments."); Class[] argumentTypes = method.getParameterTypes(); int argc = argumentTypes.length; logDebug("Method has " + argc + " arguments."); + // If no arguments, treat it as an input transform and run it without any providing any input + // stream + treatAsInputTransform = argc == 0; String[] arguments = new String[meta.getArguments().size()]; meta.getArguments().toArray(arguments); logDebug("We expected " + arguments.length + " arguments' values."); if (argc != arguments.length) { - throw new Exception("Method has a " + argc + " arguments, we expected " + arguments.length + " values."); + throw new Exception( + "Method has a " + argc + " arguments, we expected " + arguments.length + " values."); } logDebug("Allocating arguments array."); data.arguments = new Object[argc]; String stringArgument; if (meta.isArgumentSourceFields()) { - //arguments are specified by values coming from fields. - //we don't know about the fields yet, so we - //initialize with bogus value so we can check if all fields were found - logDebug("Allocating fieldindices array for arguments."); + // arguments are specified by values coming from fields. + // we don't know about the fields yet, so we + // initialize with bogus value so we can check if all fields were found + logDebug("Allocating field indices array for arguments."); data.argumentFieldIndices = new int[argc]; for (int i = 0; i < argc; i++) { data.argumentFieldIndices[i] = -1; } - } - else { - //arguments are specified directly by the user in the step. - //here we convert the string values into proper argument values + } else { + // arguments are specified directly by the user in the step. + // here we convert the string values into proper argument values Class argumentType; Object argument; for (int i = 0; i < argc; i++) { @@ -117,18 +123,15 @@ private void initMethod(JdbcMetadataMeta meta, JdbcMetadataData data) throws Exc stringArgument = arguments[i]; if (stringArgument == null) { argument = null; - } - else { + } else { stringArgument = variables.resolve(stringArgument); if (argumentType.isArray()) { if (stringArgument.length() == 0) { argument = null; - } - else { + } else { argument = stringListToObjectArray(stringArgument, argumentType.getComponentType()); } - } - else { + } else { argument = stringToArgumentValue(stringArgument, argumentType); } } @@ -186,8 +189,6 @@ public boolean init() { return result; } - - /** * This is called in the processRow function to get the actual arguments for the jdbc metadata * method @@ -233,11 +234,13 @@ private void prepareMethodArguments(JdbcMetadataMeta meta, JdbcMetadataData data private Object[] createOutputRow( JdbcMetadataMeta meta, JdbcMetadataData data, Object[] inputRow) { Object[] outputRow = new Object[data.outputRowMeta.size()]; - if (data.inputFieldsToCopy == null) { - System.arraycopy(inputRow, 0, outputRow, 0, getInputRowMeta().size()); - } else { - for (int i = 0; i < data.inputFieldsToCopy.length; i++) { - outputRow[i] = inputRow[data.inputFieldsToCopy[i]]; + if (!treatAsInputTransform) { + if (data.inputFieldsToCopy == null) { + System.arraycopy(inputRow, 0, outputRow, 0, getInputRowMeta().size()); + } else { + for (int i = 0; i < data.inputFieldsToCopy.length; i++) { + outputRow[i] = inputRow[data.inputFieldsToCopy[i]]; + } } } return outputRow; @@ -251,7 +254,7 @@ public boolean processRow() throws HopException { // if no more rows are expected, indicate step is finished and processRow() should not be called // again - if (r == null) { + if (!treatAsInputTransform && r == null) { setOutputDone(); return false; } @@ -260,14 +263,96 @@ public boolean processRow() throws HopException { // it is used to guard some processing tasks, like figuring out field indexes // in the row structure that only need to be done once if (first) { - first = false; - IRowMeta inputRowMeta = getInputRowMeta(); - data.outputRowOffset = inputRowMeta.size(); - boolean argumentSourceFields = meta.isArgumentSourceFields(); - // check if we need to use fields. + if (!treatAsInputTransform) { + first = false; + IRowMeta inputRowMeta = getInputRowMeta(); + data.outputRowOffset = inputRowMeta.size(); + boolean argumentSourceFields = meta.isArgumentSourceFields(); + logDebug("Looking up indices of input fields."); + String[] fieldNames = inputRowMeta.getFieldNames(); + logDebug("We have " + fieldNames.length + " input fields."); + List arguments = meta.getArguments(); + int argc = arguments.size(); + // store indices for argument fields + if (argumentSourceFields) { + String stringArgument; + for (int i = 0; i < fieldNames.length; i++) { + String fieldName = fieldNames[i]; + logDebug("Looking at field: " + fieldName); + for (int j = 0; j < argc; j++) { + stringArgument = arguments.get(j); + logDebug("Found argument " + j + ": " + stringArgument); + if (fieldName.equals(stringArgument)) { + logDebug("Match, storing index " + i); + data.argumentFieldIndices[j] = i; + } + } + } + // keep track of how many fields we used as args. + // We need this in case remove argument fields is enabled + // as this is the number of fields we need to discard from the input row. + int fieldsUsedAsArgs = 0; + // ensure all argument fields are bound to a valid field + argumentFields: + for (int j = 0; j < argc; j++) { + logDebug("Argument indices at " + j + ": " + data.argumentFieldIndices[j]); + if (data.argumentFieldIndices[j] == -1) { + // this argument does not point to any existing field. + if (arguments.get(j) == null || arguments.get(j).length() == 0) { + // the argument is blank, this is ok: we will pass null instead + data.argumentFieldIndices[j] = -2; + } else { + // this argument is not blank - this means it points to a non-existing field. + // this is an error. + Object[] descriptor = meta.getMethodDescriptor(); + Object[] args = (Object[]) descriptor[1]; + Object[] arg = (Object[]) args[j]; + throw new HopException( + "No field \"" + + arguments.get(j) + + "\" found for argument " + + j + + ": " + + ((String) arg[0])); + } + } else { + // this argument points to a valid field. + // let's check if this same field was already used as arg: + for (int i = 0; i < j; i++) { + if (data.argumentFieldIndices[i] == data.argumentFieldIndices[j]) { + // yes, it was used already. Let's check the next argument. + continue argumentFields; + } + } + // this field was not used already, so mark it as used. + fieldsUsedAsArgs++; + } + } + + if (meta.isRemoveArgumentFields()) { + int n = data.outputRowOffset; + data.outputRowOffset -= fieldsUsedAsArgs; + data.inputFieldsToCopy = new int[data.outputRowOffset]; - // clone the input row structure and place it in our data object - data.outputRowMeta = inputRowMeta.clone(); + inputFieldsToCopy: + for (int i = 0, j = 0; i < n; i++) { // for each field in the input row + for (int k = 0; k < argc; k++) { // for each method argument + if (data.argumentFieldIndices[k] == i) { + // this input field is used as argument. Continue to the next field. + continue inputFieldsToCopy; + } + } + // this field was not used as argument. make sure we copy it to the output. + data.inputFieldsToCopy[j++] = i; + } + } + } + logDebug("Done looking up indices of input fields."); + // clone the input row structure and place it in our data object + data.outputRowMeta = getInputRowMeta().clone(); + } else { + data.outputRowMeta = new RowMeta(); + } // use meta.getFields() to change it, so it reflects the output row structure meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); } // end of first @@ -333,6 +418,11 @@ public boolean processRow() throws HopException { logBasic("Linenr " + getLinesRead()); // Some basic logging } + if (treatAsInputTransform) { + setOutputDone(); + return false; + } + // indicate that processRow() should be called again return true; } diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java index 5fd0e09fc16..9a34290e6e0 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataData.java @@ -42,7 +42,7 @@ public class JdbcMetadataData extends BaseTransformData implements ITransformDat public int[] argumentFieldIndices; // the offset in the output row from where we can add our metadata fields. // (we need this in case we're required to remove arguments fields from the input) - public int outputRowOffset = -1; + public int outputRowOffset = 0; // public int[] inputFieldsToCopy; // the indices of the columns in the resultset diff --git a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java index f88eb2438f1..604f23fd777 100644 --- a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java +++ b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/pipeline/transform/jdbcmetadata/JdbcMetadataDialog.java @@ -495,6 +495,7 @@ public void widgetSelected(SelectionEvent selectionEvent) { removeArgumentFieldsButtonFormData.right = new FormAttachment(100, 0); removeArgumentFieldsButtonFormData.top = new FormAttachment(lastControl, margin); removeArgumentFieldsButton.setLayoutData(removeArgumentFieldsButtonFormData); + removeArgumentFieldsButton.addSelectionListener(lsSelection); // layout the metdata tab FormData metadataTabFormData = new FormData(); diff --git a/ui/src/main/resources/org/apache/hop/ui/hopgui/messages/messages_it_IT.properties b/ui/src/main/resources/org/apache/hop/ui/hopgui/messages/messages_it_IT.properties index 189face5e10..2548c34205e 100644 --- a/ui/src/main/resources/org/apache/hop/ui/hopgui/messages/messages_it_IT.properties +++ b/ui/src/main/resources/org/apache/hop/ui/hopgui/messages/messages_it_IT.properties @@ -232,7 +232,7 @@ PipelineLog.Dialog.SavePipelineBeforeRunning.Title=Il file \u00E8 cambiato\! PipelineLog.Dialog.UnexpectedErrorDuringPreview.Message=Si \u00E8 verificato un errore inatteso durante l''anteprima PipelineLog.Dialog.UnexpectedErrorDuringPreview.Title=Errore inatteso durante l''anteprima PipelineLog.Log.DoPreview=Anteprima\!\!\! -PipelineLog.Log.LaunchingPipeline=So avviando la pipeline [ +PipelineLog.Log.LaunchingPipeline=Sto avviando la pipeline [ PipelineLog.Log.PipelineHasFinished=La pipeline \u00E8 terminata\! PipelineLog.Log.PipelineOpened=Pipeline aperta. PipelineLog.Log.ProcessingOfPipelineStopped=Esecuzione pipeline interrotta. From 6e342deda5db663f2ebe901d2c103869fc327fcc Mon Sep 17 00:00:00 2001 From: Hans Van Akelyen Date: Tue, 2 May 2023 10:54:29 +0200 Subject: [PATCH 12/12] Add documentation and test, #2399 --- .../images/transforms/icons/jdbcmetadata.svg | 71 +++++++ docs/hop-user-manual/modules/ROOT/nav.adoc | 1 + .../pipeline/transforms/jdbcmetadata.adoc | 88 +++++++++ .../database/0026-jdbc-metadata.hpl | 181 ++++++++++++++++++ .../database/main-0026-jdbc-metadata.hwf | 92 +++++++++ 5 files changed, 433 insertions(+) create mode 100644 docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/jdbcmetadata.svg create mode 100644 docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jdbcmetadata.adoc create mode 100644 integration-tests/database/0026-jdbc-metadata.hpl create mode 100644 integration-tests/database/main-0026-jdbc-metadata.hwf diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/jdbcmetadata.svg b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/jdbcmetadata.svg new file mode 100644 index 00000000000..600dbc8485f --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/jdbcmetadata.svg @@ -0,0 +1,71 @@ + + + +image/svg+xml \ No newline at end of file diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index f01e96c2855..a6963991abf 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -157,6 +157,7 @@ under the License. *** xref:pipeline/transforms/insertupdate.adoc[Insert / Update] *** xref:pipeline/transforms/javafilter.adoc[Java Filter] *** xref:pipeline/transforms/javascript.adoc[JavaScript] +*** xref:pipeline/transforms/jdbcmetadata.adoc[JDBC Metadata] *** xref:pipeline/transforms/joinrows.adoc[Join Rows] *** xref:pipeline/transforms/jsoninput.adoc[JSON Input] *** xref:pipeline/transforms/jsonoutput.adoc[JSON Output] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jdbcmetadata.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jdbcmetadata.adoc new file mode 100644 index 00000000000..aaaecc4f81b --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/jdbcmetadata.adoc @@ -0,0 +1,88 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US +:description: The JDBC Metadata transform allows you to fetch metadata from a database connection, this includes schemas, tables and other objects. +:imagesdir: ../../../assets/images + += image:transforms/icons/jdbcmetadata.svg[JDBC Metadata transform Icon, role="image-doc-icon"] JDBC Metadata + +[%noheader,cols="3a,1a", role="table-no-borders" ] +|=== +| +== Description + +The JDBC Metadata transform allows you to fetch metadata from a database connection, this includes schemas, tables and other objects. + +| +== Supported Engines +[%noheader,cols="2,1a",frame=none, role="table-supported-engines"] +!=== +!Hop Engine! image:check_mark.svg[Supported, 24] +!Spark! image:question_mark.svg[Maybe Supported, 24] +!Flink! image:question_mark.svg[Maybe Supported, 24] +!Dataflow! image:question_mark.svg[Maybe Supported, 24] +!=== +|=== + +== Method and Arguments tab +In this tab, you can specify the java method of the DatabaseMetaData object which is to be called to obtain metadata. It also controls some other aspects of the steps behavior. + +*Always pass the input row:* Controls how this step behaves in case there is no metadata. If this option is not checked, the step will not produce any output rows. If this option is checked, the incoming row will still be passed. Any output fields that originate from the metadata resultset will be NULL in this case. + +*Metadata Method:* This dropdown list can be used to specify which metadata method to call upon the DatabaseMetaData object to obtain metadata: + +- Catalogs: see DatabaseMetaData.getCatalogs() + +- Best row identifier: see link:++http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getBestRowIdentifier(java.lang.String, java.lang.String, java.lang.String, int, boolean)++[DatabaseMetaData.getBestRowIdentifier()] + +- Column privileges: see link:++http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getColumnPrivileges(java.lang.String, java.lang.String, java.lang.String, java.lang.String)++[DatabaseMetaData.getColumnPrivileges()] + +- Columns: see link:++http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getColumns(java.lang.String, java.lang.String, java.lang.String, java.lang.String)++[DatabaseMetaData.getColumns()] + +- Cross references: see link:++http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getCrossReference(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String)++[DatabaseMetaData.getCrossReferences()] + +- Exported key columns: see link:++http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getExportedKeys(java.lang.String, java.lang.String, java.lang.String)++[DatabaseMetaData.getExportedKeys()] + +- Foreign key columns: see link:++ttp://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getImportedKeys(java.lang.String, java.lang.String, java.lang.String)++[DatabaseMetaData.getImportedKeys()] + +- Primary key columns: see link:++http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getPrimaryKeys(java.lang.String, java.lang.String, java.lang.String)++[DatabaseMetaData.getPrimaryKeys()] + +- Schemas: see DatabaseMetaData.getSchemas() + +- Table privileges: see link:++http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getTablePrivileges(java.lang.String, java.lang.String, java.lang.String)++[DatabaseMetaData.getTablePrivileges()] + +- Table types: see DatabaseMetaData.getTableTypes() + +- Tables: see link:++http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getTables(java.lang.String, java.lang.String, java.lang.String, java.lang.String[])++[DatabaseMetaData.getTables()] + +- Data types: see DatabaseMetaData.getTypeInfo() + +- Version columns: see link:++http://docs.oracle.com/javase/7/docs/api/java/sql/DatabaseMetaData.html#getVersionColumns(java.lang.String, java.lang.String, java.lang.String)++[DatabaseMetaData.getVersionColumns()] + +- After selecting a method, fields will be added to the tab so you can specify any arguments required by the method. For a description of the method-specific arguments and their meaning, please refer to the javadoc for each method (linked above). + +*Get arguments from fields:* If this option is checked, arguments to the method can be specified by selecting a field from the incoming stream, and the value of that field is used as actual argument value. If this option is not checked, argument values can be entered directly as a literal into the argument fields. + +*Remove argument fields:* This option is applicable when the Get arguments from fields is checked. When checked, the fields that are selected as argument fields are removed from the output stream. This is typically a convenient option when you chain several JdbcMetaData steps in sequence, using the fields coming out of upstream steps as argument fields for downstream steps. In such a case using this option will remove a lot of duplicate fields. + +== Output Fields tab +This tab allows you to control how the output metadata is added to the outputstream. + +*Output Fields:* This grid is automatically filled with the appropriate fields as you select a particular metadata method on the Method and Aruments tab. Use this gridview to remove or rename output fields. + +*Get fields:* use this button to refill the grid. This will re-add any removed fields, but preserve any renamed fields. \ No newline at end of file diff --git a/integration-tests/database/0026-jdbc-metadata.hpl b/integration-tests/database/0026-jdbc-metadata.hpl new file mode 100644 index 00000000000..526b63ffa87 --- /dev/null +++ b/integration-tests/database/0026-jdbc-metadata.hpl @@ -0,0 +1,181 @@ + + + + + 0026-jdbc-metadata + Y + + + + Normal + + + N + 1000 + 100 + - + 2023/03/07 11:23:18.711 + - + 2023/03/07 11:23:18.711 + + + + + + Get JDBC Metadata + count rows + Y + + + count rows + >= 1 + Y + + + >= 1 + Abort + Y + + + + Get JDBC Metadata + JdbcMetadata + + Y + + 1 + + none + + + N + N + + + unit-test-db + getCatalogs + + + TABLE_CAT + TABLE_CAT + + + N + + + 112 + 96 + + + + count rows + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + counter + + COUNT_ANY + + + + Y + + + N + + grp + + + 304 + 96 + + + + >= 1 + FilterRows + + Y + + 1 + + none + + + + + + + >= + counter + Y + - + + N + -1 + ####0;-####0 + constant + 0 + 1 + Integer + + + + + + 480 + 96 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + + 0 + + + 624 + 96 + + + + + + diff --git a/integration-tests/database/main-0026-jdbc-metadata.hwf b/integration-tests/database/main-0026-jdbc-metadata.hwf new file mode 100644 index 00000000000..5255c89559b --- /dev/null +++ b/integration-tests/database/main-0026-jdbc-metadata.hwf @@ -0,0 +1,92 @@ + + + + main-0026-jdbc-metadata + Y + + + + - + 2022/12/02 13:51:33.435 + - + 2022/12/02 13:51:33.435 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 80 + 80 + + + + 0026-jdbc-metadata.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/0026-jdbc-metadata.hpl + + + Basic + + Y + + N + local + N + N + Y + N + 272 + 80 + + + + + + Start + 0026-jdbc-metadata.hpl + Y + Y + Y + + + + + +