From 535603a9fc4ddc9ee791863386d3b9e71489ca4c Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Sat, 19 Nov 2022 16:11:14 +0800 Subject: [PATCH 01/11] add jira source --- plugin-mapping.properties | 1 + .../connector-http-jira/pom.xml | 40 +++++++ .../seatunnel/jira/source/JiraSource.java | 66 +++++++++++ .../jira/source/JiraSourceFactory.java | 55 +++++++++ .../jira/source/config/JiraSourceConfig.java | 31 +++++ .../source/config/JiraSourceParameter.java | 34 ++++++ .../connector-http/pom.xml | 3 +- seatunnel-dist/pom.xml | 6 + .../connector-http-e2e/pom.xml | 8 +- .../e2e/connector/http/HttpJiraIT.java | 76 ++++++++++++ .../test/resources/jira_json_to_assert.conf | 93 +++++++++++++++ .../resources/mockserver-jira-config.json | 109 ++++++++++++++++++ 12 files changed, 520 insertions(+), 2 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-jira/pom.xml create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceParameter.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json diff --git a/plugin-mapping.properties b/plugin-mapping.properties index d212f538b91..8967fb245be 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -148,3 +148,4 @@ seatunnel.source.GoogleSheets = connector-google-sheets seatunnel.sink.Tablestore = connector-tablestore seatunnel.source.Lemlist = connector-http-lemlist seatunnel.source.Klaviyo = connector-http-klaviyo +seatunnel.source.Jira = connector-http-jira diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/pom.xml b/seatunnel-connectors-v2/connector-http/connector-http-jira/pom.xml new file mode 100644 index 00000000000..34b7830ead6 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/pom.xml @@ -0,0 +1,40 @@ + + + + + connector-http + org.apache.seatunnel + ${revision} + + 4.0.0 + + connector-http-jira + + + + org.apache.seatunnel + connector-http-base + ${project.version} + + + + diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java new file mode 100644 index 00000000000..44f57e354ee --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jira.source; + +import static org.apache.seatunnel.connectors.seatunnel.http.util.AuthorizationUtil.getTokenByBasicAuth; + +import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.config.CheckConfigUtil; +import org.apache.seatunnel.common.config.CheckResult; +import org.apache.seatunnel.common.constants.PluginType; +import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader; +import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext; +import org.apache.seatunnel.connectors.seatunnel.http.source.HttpSource; +import org.apache.seatunnel.connectors.seatunnel.http.source.HttpSourceReader; +import org.apache.seatunnel.connectors.seatunnel.jira.source.config.JiraSourceConfig; +import org.apache.seatunnel.connectors.seatunnel.jira.source.config.JiraSourceParameter; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@AutoService(SeaTunnelSource.class) +public class JiraSource extends HttpSource { + private final JiraSourceParameter jiraSourceParameter = new JiraSourceParameter(); + + @Override + public String getPluginName() { + return "Jira"; + } + + @Override + public void prepare(Config pluginConfig) throws PrepareFailException { + CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, JiraSourceConfig.URL.key(), JiraSourceConfig.PASSWORD.key()); + if (!result.isSuccess()) { + throw new PrepareFailException(getPluginName(), PluginType.SOURCE, result.getMsg()); + } + //get accessToken by basic auth + String accessToken = getTokenByBasicAuth("", pluginConfig.getString(JiraSourceConfig.PASSWORD.key())); + jiraSourceParameter.buildWithConfig(pluginConfig, accessToken); + buildSchemaWithConfig(pluginConfig); + } + + @Override + public AbstractSingleSplitReader createReader(SingleSplitReaderContext readerContext) throws Exception { + return new HttpSourceReader(this.jiraSourceParameter, readerContext, this.deserializationSchema); + } +} diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java new file mode 100644 index 00000000000..54819e96090 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jira.source; + +import org.apache.seatunnel.api.configuration.util.Condition; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.api.table.factory.TableSourceFactory; +import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig; +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpRequestMethod; +import org.apache.seatunnel.connectors.seatunnel.jira.source.config.JiraSourceConfig; + +import com.google.auto.service.AutoService; + +@AutoService(Factory.class) +public class JiraSourceFactory implements TableSourceFactory { + @Override + public String factoryIdentifier() { + return "Jira"; + } + + @Override + public OptionRule optionRule() { + return OptionRule.builder() + .required(JiraSourceConfig.URL) + .required(JiraSourceConfig.PASSWORD) + .optional(JiraSourceConfig.METHOD) + .optional(JiraSourceConfig.HEADERS) + .optional(JiraSourceConfig.PARAMS) + .conditional(Condition.of(HttpConfig.METHOD, HttpRequestMethod.POST), JiraSourceConfig.BODY) + .conditional(Condition.of(HttpConfig.FORMAT, "json"), SeaTunnelSchema.SCHEMA) + .optional(JiraSourceConfig.FORMAT) + .optional(JiraSourceConfig.POLL_INTERVAL_MILLS) + .optional(JiraSourceConfig.RETRY) + .optional(JiraSourceConfig.RETRY_BACKOFF_MAX_MS) + .optional(JiraSourceConfig.RETRY_BACKOFF_MULTIPLIER_MS) + .build(); + } +} diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java new file mode 100644 index 00000000000..3daaf9fbba9 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jira.source.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig; + +public class JiraSourceConfig extends HttpConfig { + public static final String AUTHORIZATION = "Authorization"; + public static final Option PASSWORD = Options.key("password") + .stringType() + .noDefaultValue() + .withDescription("Lemlist login api key"); + +} diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceParameter.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceParameter.java new file mode 100644 index 00000000000..3e1ae6dbd57 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceParameter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jira.source.config; + +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpParameter; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import java.util.HashMap; + +public class JiraSourceParameter extends HttpParameter { + public void buildWithConfig(Config pluginConfig, String accessToken) { + super.buildWithConfig(pluginConfig); + // put authorization in headers + this.headers = this.getHeaders() == null ? new HashMap<>() : this.getHeaders(); + this.headers.put(JiraSourceConfig.AUTHORIZATION, accessToken); + this.setHeaders(this.headers); + } +} diff --git a/seatunnel-connectors-v2/connector-http/pom.xml b/seatunnel-connectors-v2/connector-http/pom.xml index bcc6df222ef..07f81454cce 100644 --- a/seatunnel-connectors-v2/connector-http/pom.xml +++ b/seatunnel-connectors-v2/connector-http/pom.xml @@ -36,6 +36,7 @@ connector-http-myhours connector-http-lemlist connector-http-klaviyo + connector-http-jira - \ No newline at end of file + diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index ed370c24091..243c685a8ad 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -339,6 +339,12 @@ ${project.version} provided + + org.apache.seatunnel + connector-http-jira + ${project.version} + provided + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml index ebdceefc97e..9a08be636b2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml @@ -56,6 +56,12 @@ ${project.version} test + + org.apache.seatunnel + connector-http-jira + ${project.version} + test + - \ No newline at end of file + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java new file mode 100644 index 00000000000..1b5851837a3 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.connector.http; + +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.TestContainer; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.util.stream.Stream; + +public class HttpJiraIT extends TestSuiteBase implements TestResource { + + private static final String IMAGE = "mockserver/mockserver:5.14.0"; + + private GenericContainer mockserverContainer; + + @BeforeAll + @Override + public void startUp() { + this.mockserverContainer = new GenericContainer<>(DockerImageName.parse(IMAGE)) + .withNetwork(NETWORK) + .withNetworkAliases("mockserver") + .withExposedPorts(1080) + .withCopyFileToContainer(MountableFile.forHostPath(new File(HttpIT.class.getResource( + "/mockserver-lemlist-config.json").getPath()).getAbsolutePath()), + "/tmp/mockserver-lemlist-config.json") + .withEnv("MOCKSERVER_INITIALIZATION_JSON_PATH", "/tmp/mockserver-lemlist-config.json") + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) + .waitingFor(new HttpWaitStrategy().forPath("/").forStatusCode(404)); + Startables.deepStart(Stream.of(mockserverContainer)).join(); + } + + @AfterAll + @Override + public void tearDown() { + if (mockserverContainer != null) { + mockserverContainer.stop(); + } + } + + @TestTemplate + public void testHttpLemlistSourceToAssertSink(TestContainer container) throws IOException, InterruptedException { + Container.ExecResult execResult = container.executeJob("/lemlist_json_to_assert.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf new file mode 100644 index 00000000000..d5931d78626 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf @@ -0,0 +1,93 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + Lemlist { + url = "http://mockserver:1080/api/team" + password = "Seatunnel-test" + method = "GET" + format = "json" + schema = { + fields { + _id = string + name = string + userIds = "array" + createdBy = string + createdAt = string + apiKey = string + billing = { + quantity = int + ok = boolean + plan = string + } + } + } + } +} + +sink { + Console {} + Assert { + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 5 + }, + { + rule_type = MIN_ROW + rule_value = 5 + } + ], + + field_rules = [ + { + field_name = _id + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = apiKey + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = createdAt + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json new file mode 100644 index 00000000000..f03848e36bd --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json @@ -0,0 +1,109 @@ +// https://www.mock-server.com/mock_server/getting_started.html#request_matchers + +[ + { + "httpRequest": { + "method" : "GET", + "path": "/api/team" + }, + "httpResponse": { + "body": [ + { + "_id": "tea_aaqam5a3BkY8aje24", + "name": "PiedPiper", + "userIds": ["usr_45y54yug42yh4h66j7j"], + "createdBy": "usr_45y54yug42yh4h66j7j", + "createdAt": "2018-04-30T12:19:42.829Z", + "apiKey": "aa13722b45b9c475cc686231b1af6583", + "billing": { + "quantity": 1, + "ok": true, + "plan": "freetrial" + } + }, + { + "_id": "tea_abt325f32332aje4364", + "name": "TaoZex", + "userIds": ["usr_gh954gbiu5bg4t5l54t43t"], + "createdBy": "gh954gbiu5bg4t5l54t43t", + "createdAt": "2018-07-30T12:19:42.829Z", + "apiKey": "y4vu3yf74g3b4o3878438f4837fg4g48", + "billing": { + "quantity": 2, + "ok": true, + "plan": "sport" + } + }, + { + "_id": "tea_fta8f7tas68fgsf6as", + "name": "Frivlc", + "userIds": ["usr_r3w4t5y6h65u79f8ehfe6"], + "createdBy": "r3w4t5y6h65u79f8ehfe6", + "createdAt": "2021-04-30T12:19:42.829Z", + "apiKey": "8fh9473gfo847hf874bhwf76h4uifh44", + "billing": { + "quantity": 1, + "ok": false, + "plan": "test" + } + }, + { + "_id": "tea_6sdtgsrgdsghirq32r", + "name": "Lisa", + "userIds": ["usr_84974h3fyg453u5tkg"], + "createdBy": "usr_84974h3fyg453u5tkg", + "createdAt": "2018-04-30T12:26:42.829Z", + "apiKey": "7gf987f756agff7uagfo87agf8oaf3", + "billing": { + "quantity": 0, + "ok": true, + "plan": "study" + } + }, + { + "_id": "tea_78yhwg7e5rsyges8o7g", + "name": "Jack", + "userIds": ["usr_768h3fyv34i7w6g4y4fw"], + "createdBy": "usr_768h3fyv34i7w6g4y4fw", + "createdAt": "2019-04-30T11:19:42.829Z", + "apiKey": "3kruy4v3ir764gklrug4kw7rw4li784", + "billing": { + "quantity": 2, + "ok": false, + "plan": "eat food" + } + } + ] + } + }, + { + "httpRequest": { + "method" : "GET", + "path": "/api/campaigns" + }, + "httpResponse": { + "body": [ + { + "_id":"bydeV", + "name":"TaoZex's campaign 1" + }, + { + "_id":"wIIdk", + "name":"TaoZex's campaign 2" + }, + { + "_id":"qvHRy", + "name":"TaoZex's campaign 3" + }, + { + "_id":"WfBPo", + "name":"TaoZex's campaign 4" + }, + { + "_id":"yPhVF", + "name":"TaoZex's campaign 5" + } + ] + } + } +] From bcb777e7fbaf9e6a99b9c991aeeb4ba6109b33b8 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Mon, 21 Nov 2022 22:43:45 +0800 Subject: [PATCH 02/11] update jira connector --- docs/en/connector-v2/source/Jira.md | 158 ++++++++++++++++++ .../seatunnel/jira/source/JiraSource.java | 4 +- .../jira/source/JiraSourceFactory.java | 3 +- .../jira/source/config/JiraSourceConfig.java | 8 +- .../test/resources/jira_json_to_assert.conf | 36 +--- .../resources/mockserver-jira-config.json | 99 +---------- 6 files changed, 182 insertions(+), 126 deletions(-) create mode 100644 docs/en/connector-v2/source/Jira.md diff --git a/docs/en/connector-v2/source/Jira.md b/docs/en/connector-v2/source/Jira.md new file mode 100644 index 00000000000..672645b498c --- /dev/null +++ b/docs/en/connector-v2/source/Jira.md @@ -0,0 +1,158 @@ +# Jira + +> Jira source connector + +## Description + +Used to read data from Jira. + +## Key features + +- [x] [batch](../../concept/connector-v2-features.md) +- [ ] [stream](../../concept/connector-v2-features.md) +- [ ] [exactly-once](../../concept/connector-v2-features.md) +- [x] [schema projection](../../concept/connector-v2-features.md) +- [ ] [parallelism](../../concept/connector-v2-features.md) +- [ ] [support user-defined split](../../concept/connector-v2-features.md) + +## Options + +| name | type | required | default value | +| --------------------------- | ------ | -------- | ------------- | +| url | String | Yes | - | +| email | String | Yes | - | +| api_token | String | Yes | - | +| method | String | No | get | +| schema.fields | Config | No | - | +| format | String | No | json | +| params | Map | No | - | +| body | String | No | - | +| poll_interval_ms | int | No | - | +| retry | int | No | - | +| retry_backoff_multiplier_ms | int | No | 100 | +| retry_backoff_max_ms | int | No | 10000 | +| common-options | config | No | - | + +### url [String] + +http request url + +### email [String] + +Jira Email + +### api_token [String] + +Jira API Token + +https://id.atlassian.com/manage-profile/security/api-tokens + +### method [String] + +http request method, only supports GET, POST method + +### params [Map] + +http params + +### body [String] + +http body + +### poll_interval_ms [int] + +request http api interval(millis) in stream mode + +### retry [int] + +The max retry times if request http return to `IOException` + +### retry_backoff_multiplier_ms [int] + +The retry-backoff times(millis) multiplier if request http failed + +### retry_backoff_max_ms [int] + +The maximum retry-backoff times(millis) if request http failed + +### format [String] + +the format of upstream data, now only support `json` `text`, default `json`. + +when you assign format is `json`, you should also assign schema option, for example: + +upstream data is the following: + +```json + +{"code": 200, "data": "get success", "success": true} + +``` + +you should assign schema as the following: + +```hocon + +schema { + fields { + code = int + data = string + success = boolean + } +} + +``` + +connector will generate data as the following: + +| code | data | success | +|------|-------------|---------| +| 200 | get success | true | + +when you assign format is `text`, connector will do nothing for upstream data, for example: + +upstream data is the following: + +```json + +{"code": 200, "data": "get success", "success": true} + +``` + +connector will generate data as the following: + +| content | +|---------| +| {"code": 200, "data": "get success", "success": true} | + +### schema [Config] + +#### fields [Config] + +the schema fields of upstream data + +### common options + +Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details + +## Example + +```hocon +Jira { + url = "https://liugddx.atlassian.net/rest/api/3/search" + email = "test@test.com" + api_token = "xxx" + schema { + fields { + id = string + name = string + } + } +} +``` + +## Changelog + +### next version + +- Add Jira Source Connector diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java index 44f57e354ee..6b6597e8218 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java @@ -49,12 +49,12 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) throws PrepareFailException { - CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, JiraSourceConfig.URL.key(), JiraSourceConfig.PASSWORD.key()); + CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, JiraSourceConfig.URL.key(), JiraSourceConfig.EMAIL.key(), JiraSourceConfig.API_TOKEN.key()); if (!result.isSuccess()) { throw new PrepareFailException(getPluginName(), PluginType.SOURCE, result.getMsg()); } //get accessToken by basic auth - String accessToken = getTokenByBasicAuth("", pluginConfig.getString(JiraSourceConfig.PASSWORD.key())); + String accessToken = getTokenByBasicAuth(pluginConfig.getString(JiraSourceConfig.EMAIL.key()), pluginConfig.getString(JiraSourceConfig.API_TOKEN.key())); jiraSourceParameter.buildWithConfig(pluginConfig, accessToken); buildSchemaWithConfig(pluginConfig); } diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java index 54819e96090..c37f949b5d3 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSourceFactory.java @@ -39,7 +39,8 @@ public String factoryIdentifier() { public OptionRule optionRule() { return OptionRule.builder() .required(JiraSourceConfig.URL) - .required(JiraSourceConfig.PASSWORD) + .required(JiraSourceConfig.EMAIL) + .required(JiraSourceConfig.API_TOKEN) .optional(JiraSourceConfig.METHOD) .optional(JiraSourceConfig.HEADERS) .optional(JiraSourceConfig.PARAMS) diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java index 3daaf9fbba9..9eda2e6e458 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/config/JiraSourceConfig.java @@ -23,9 +23,13 @@ public class JiraSourceConfig extends HttpConfig { public static final String AUTHORIZATION = "Authorization"; - public static final Option PASSWORD = Options.key("password") + public static final Option EMAIL = Options.key("email") .stringType() .noDefaultValue() - .withDescription("Lemlist login api key"); + .withDescription("Jira email"); + public static final Option API_TOKEN = Options.key("api_token") + .stringType() + .noDefaultValue() + .withDescription("Jira API Token"); } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf index d5931d78626..8fa18d85470 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf @@ -28,17 +28,10 @@ source { format = "json" schema = { fields { - _id = string - name = string - userIds = "array" - createdBy = string - createdAt = string - apiKey = string - billing = { - quantity = int - ok = boolean - plan = string - } + expand = string + startAt = int + maxResults = int + total = string } } } @@ -48,20 +41,9 @@ sink { Console {} Assert { rules { - row_rules = [ - { - rule_type = MAX_ROW - rule_value = 5 - }, - { - rule_type = MIN_ROW - rule_value = 5 - } - ], - field_rules = [ { - field_name = _id + field_name = expand field_type = string field_value = [ { @@ -70,8 +52,8 @@ sink { ] }, { - field_name = apiKey - field_type = string + field_name = startAt + field_type = int field_value = [ { rule_type = NOT_NULL @@ -79,7 +61,7 @@ sink { ] }, { - field_name = createdAt + field_name = maxResults field_type = string field_value = [ { @@ -90,4 +72,4 @@ sink { ] } } -} \ No newline at end of file +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json index f03848e36bd..0072f7f9320 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-jira-config.json @@ -4,104 +4,15 @@ { "httpRequest": { "method" : "GET", - "path": "/api/team" + "path": "/rest/api/3/search" }, "httpResponse": { "body": [ { - "_id": "tea_aaqam5a3BkY8aje24", - "name": "PiedPiper", - "userIds": ["usr_45y54yug42yh4h66j7j"], - "createdBy": "usr_45y54yug42yh4h66j7j", - "createdAt": "2018-04-30T12:19:42.829Z", - "apiKey": "aa13722b45b9c475cc686231b1af6583", - "billing": { - "quantity": 1, - "ok": true, - "plan": "freetrial" - } - }, - { - "_id": "tea_abt325f32332aje4364", - "name": "TaoZex", - "userIds": ["usr_gh954gbiu5bg4t5l54t43t"], - "createdBy": "gh954gbiu5bg4t5l54t43t", - "createdAt": "2018-07-30T12:19:42.829Z", - "apiKey": "y4vu3yf74g3b4o3878438f4837fg4g48", - "billing": { - "quantity": 2, - "ok": true, - "plan": "sport" - } - }, - { - "_id": "tea_fta8f7tas68fgsf6as", - "name": "Frivlc", - "userIds": ["usr_r3w4t5y6h65u79f8ehfe6"], - "createdBy": "r3w4t5y6h65u79f8ehfe6", - "createdAt": "2021-04-30T12:19:42.829Z", - "apiKey": "8fh9473gfo847hf874bhwf76h4uifh44", - "billing": { - "quantity": 1, - "ok": false, - "plan": "test" - } - }, - { - "_id": "tea_6sdtgsrgdsghirq32r", - "name": "Lisa", - "userIds": ["usr_84974h3fyg453u5tkg"], - "createdBy": "usr_84974h3fyg453u5tkg", - "createdAt": "2018-04-30T12:26:42.829Z", - "apiKey": "7gf987f756agff7uagfo87agf8oaf3", - "billing": { - "quantity": 0, - "ok": true, - "plan": "study" - } - }, - { - "_id": "tea_78yhwg7e5rsyges8o7g", - "name": "Jack", - "userIds": ["usr_768h3fyv34i7w6g4y4fw"], - "createdBy": "usr_768h3fyv34i7w6g4y4fw", - "createdAt": "2019-04-30T11:19:42.829Z", - "apiKey": "3kruy4v3ir764gklrug4kw7rw4li784", - "billing": { - "quantity": 2, - "ok": false, - "plan": "eat food" - } - } - ] - } - }, - { - "httpRequest": { - "method" : "GET", - "path": "/api/campaigns" - }, - "httpResponse": { - "body": [ - { - "_id":"bydeV", - "name":"TaoZex's campaign 1" - }, - { - "_id":"wIIdk", - "name":"TaoZex's campaign 2" - }, - { - "_id":"qvHRy", - "name":"TaoZex's campaign 3" - }, - { - "_id":"WfBPo", - "name":"TaoZex's campaign 4" - }, - { - "_id":"yPhVF", - "name":"TaoZex's campaign 5" + "expand": "schema,names", + "startAt": 0, + "maxResults": 50, + "total": 3 } ] } From 383ce58128ba7770f37981385b2ce575108dff29 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Tue, 22 Nov 2022 11:06:42 +0800 Subject: [PATCH 03/11] Update seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java Co-authored-by: hailin0 --- .../apache/seatunnel/e2e/connector/http/HttpJiraIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java index 1b5851837a3..16b933cefdf 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java @@ -51,10 +51,10 @@ public void startUp() { .withNetwork(NETWORK) .withNetworkAliases("mockserver") .withExposedPorts(1080) - .withCopyFileToContainer(MountableFile.forHostPath(new File(HttpIT.class.getResource( - "/mockserver-lemlist-config.json").getPath()).getAbsolutePath()), - "/tmp/mockserver-lemlist-config.json") - .withEnv("MOCKSERVER_INITIALIZATION_JSON_PATH", "/tmp/mockserver-lemlist-config.json") + .withCopyFileToContainer(MountableFile.forHostPath(new File(HttpJiraIT.class.getResource( + "/mockserver-jira-config.json").getPath()).getAbsolutePath()), + "/tmp/mockserver-jira-config.json") + .withEnv("MOCKSERVER_INITIALIZATION_JSON_PATH", "/tmp/mockserver-jira-config.json") .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) .waitingFor(new HttpWaitStrategy().forPath("/").forStatusCode(404)); Startables.deepStart(Stream.of(mockserverContainer)).join(); From b90a60c164d272b641603dbb51706aec6ddbfea9 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Tue, 22 Nov 2022 11:06:56 +0800 Subject: [PATCH 04/11] Update seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java Co-authored-by: hailin0 --- .../org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java index 16b933cefdf..85cc1e9dc7d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java @@ -70,7 +70,7 @@ public void tearDown() { @TestTemplate public void testHttpLemlistSourceToAssertSink(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob("/lemlist_json_to_assert.conf"); + Container.ExecResult execResult = container.executeJob("/jira_json_to_assert.conf"); Assertions.assertEquals(0, execResult.getExitCode()); } } From 264717ef8611b49426698f4cbf891e1d89a0f387 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Tue, 22 Nov 2022 11:07:17 +0800 Subject: [PATCH 05/11] Update seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java Co-authored-by: hailin0 --- .../org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java index 85cc1e9dc7d..94a513e171c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpJiraIT.java @@ -69,7 +69,7 @@ public void tearDown() { } @TestTemplate - public void testHttpLemlistSourceToAssertSink(TestContainer container) throws IOException, InterruptedException { + public void testHttpJiraSourceToAssertSink(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = container.executeJob("/jira_json_to_assert.conf"); Assertions.assertEquals(0, execResult.getExitCode()); } From 2d256e713e966e7e0683eb58c19793ba5f9740d0 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Tue, 22 Nov 2022 11:12:41 +0800 Subject: [PATCH 06/11] fix rv error. --- .../connectors/seatunnel/jira/source/JiraSource.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java index 6b6597e8218..6d112de4c75 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-jira/src/main/java/org/apache/seatunnel/connectors/seatunnel/jira/source/JiraSource.java @@ -20,10 +20,12 @@ import static org.apache.seatunnel.connectors.seatunnel.http.util.AuthorizationUtil.getTokenByBasicAuth; import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.source.Boundedness; import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.common.config.CheckConfigUtil; import org.apache.seatunnel.common.config.CheckResult; +import org.apache.seatunnel.common.constants.JobMode; import org.apache.seatunnel.common.constants.PluginType; import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader; import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext; @@ -47,6 +49,14 @@ public String getPluginName() { return "Jira"; } + @Override + public Boundedness getBoundedness() { + if (JobMode.BATCH.equals(jobContext.getJobMode())) { + return Boundedness.BOUNDED; + } + throw new UnsupportedOperationException("Jira source connector not support unbounded operation"); + } + @Override public void prepare(Config pluginConfig) throws PrepareFailException { CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, JiraSourceConfig.URL.key(), JiraSourceConfig.EMAIL.key(), JiraSourceConfig.API_TOKEN.key()); From fec2aae1e6b9e062f58500b98ce4e173532c6f00 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Tue, 22 Nov 2022 11:45:52 +0800 Subject: [PATCH 07/11] Update docs/en/connector-v2/source/Jira.md Co-authored-by: hailin0 --- docs/en/connector-v2/source/Jira.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/en/connector-v2/source/Jira.md b/docs/en/connector-v2/source/Jira.md index 672645b498c..0f170b695d8 100644 --- a/docs/en/connector-v2/source/Jira.md +++ b/docs/en/connector-v2/source/Jira.md @@ -144,8 +144,10 @@ Jira { api_token = "xxx" schema { fields { - id = string - name = string + expand = string + startAt = bigint + maxResults = int + total = int } } } From 1e6cf13b8370a6ae13d529ea1423cc87438a21a1 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Tue, 22 Nov 2022 19:36:14 +0800 Subject: [PATCH 08/11] fix ci error. --- .../src/test/resources/jira_json_to_assert.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf index 8fa18d85470..03c2b77c59a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf @@ -21,7 +21,7 @@ env { } source { - Lemlist { + Jira { url = "http://mockserver:1080/api/team" password = "Seatunnel-test" method = "GET" From e7c11c783030256ce4d39a5f71f1d641a4ad7e7e Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Tue, 22 Nov 2022 20:33:29 +0800 Subject: [PATCH 09/11] fix ci error. --- .../src/test/resources/jira_json_to_assert.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf index 03c2b77c59a..389848db7b2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf @@ -62,7 +62,7 @@ sink { }, { field_name = maxResults - field_type = string + field_type = int field_value = [ { rule_type = NOT_NULL From f71881322a03dffe72bcfffc2c8aedee4dd26847 Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Tue, 22 Nov 2022 20:51:41 +0800 Subject: [PATCH 10/11] fix ci error. --- .../src/test/resources/jira_json_to_assert.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf index 389848db7b2..5969429d0f5 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf @@ -22,7 +22,7 @@ env { source { Jira { - url = "http://mockserver:1080/api/team" + url = "http://mockserver:1080/rest/api/3/search" password = "Seatunnel-test" method = "GET" format = "json" From ac65a6859b39d3fc37215580561be49a9b7f15dc Mon Sep 17 00:00:00 2001 From: liugddx <804167098@qq.com> Date: Tue, 22 Nov 2022 21:53:26 +0800 Subject: [PATCH 11/11] fix ci error --- .../src/test/resources/jira_json_to_assert.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf index 5969429d0f5..3be4e859870 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/jira_json_to_assert.conf @@ -23,7 +23,8 @@ env { source { Jira { url = "http://mockserver:1080/rest/api/3/search" - password = "Seatunnel-test" + email = "admin@test.com" + api_token = "token" method = "GET" format = "json" schema = {