Skip to content

Commit

Permalink
[Improvement] Improve some maven profile configurations (apache#2944)
Browse files Browse the repository at this point in the history 
* Improve maven profile configurations

* Add github workflows configurations change
  • Loading branch information
@zhoujinsong
zhoujinsong authored Jun 20, 2024
1 parent 00963ac commit 4f07ad0
Show file tree
Hide file tree
Showing 49 changed files with 351 additions and 548 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/docker-images.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ jobs:
OPTIMIZER_FLINK=${{ matrix.flink }} && \
echo "OPTIMIZER_FLINK=-Dflink-optimizer.flink-version${OPTIMIZER_FLINK}" >> $GITHUB_ENV
if [[ "$OPTIMIZER_FLINK" < "1.15" ]]; then
echo "Adding -Pflink-pre-1.15 for Flink version less than 1.15"
echo "OPTIMIZER_FLINK=-Pflink-pre-1.15 -Dflink-optimizer.flink-version=${OPTIMIZER_FLINK}" >> $GITHUB_ENV
echo "Adding -Pflink-optimizer-pre-1.15 for Flink version less than 1.15"
echo "OPTIMIZER_FLINK=-Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=${OPTIMIZER_FLINK}" >> $GITHUB_ENV
fi
- name: Set ENV Amoro version
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ Amoro is built using Maven with JDK 8 and JDK 17(only for `amoro-mixed-format/am
* Build and skip dashboard: `mvn clean package -Pskip-dashboard-build`
* Build with hadoop 2.x(the default is 3.x) dependencies: `mvn clean package -DskipTests -Dhadoop=v2`
* Specify Flink version for Flink optimizer(the default is 1.18.1): `mvn clean package -DskipTests -Dflink-optimizer.flink-version=1.15.4`
* If the version of Flink is below 1.15.0, you also need to add the `-Pflink-pre-1.15` parameter: `mvn clean package -DskipTests -Pflink-pre-1.15 -Dflink-optimizer.flink-version=1.14.6`
* If the version of Flink is below 1.15.0, you also need to add the `-Pflink-optimizer-pre-1.15` parameter: `mvn clean package -DskipTests -Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=1.14.6`
* Specify Spark version for Spark optimizer(the default is 3.3.3): `mvn clean package -DskipTests -Dspark-optimizer.spark-version=3.2.2`
* Build `amoro-mixed-format-trino` module under JDK 17: `mvn clean package -DskipTests -Ptrino-spotless,build-mixed-format-trino -pl 'amoro-mixed-format/amoro-mixed-format-trino' -am`.
* Build all modules: `mvn clean package -DskipTests -Ptoolchain,build-mixed-format-trino`, besides you need config `toolchains.xml` in `${user.home}/.m2/` dir with content below.
Expand Down
2 changes: 1 addition & 1 deletion amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@

<profiles>
<profile>
<id>flink-pre-1.15</id>
<id>flink-optimizer-pre-1.15</id>
<activation>
<property>
<name>flink-optimizer.flink-version</name>
Expand Down
28 changes: 0 additions & 28 deletions ...mixed-format/amoro-mixed-format-flink/v1.15/amoro-mixed-format-flink-runtime-1.15/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,32 +249,4 @@
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>deploy-maven-central</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin-version}</version>
<executions>
<execution>
<id>empty-javadoc-jar</id>
<goals>
<goal>jar</goal>
</goals>
<phase>package</phase>
<configuration>
<classifier>javadoc</classifier>
<classesDirectory>${basedir}/javadoc</classesDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
28 changes: 0 additions & 28 deletions ...mixed-format/amoro-mixed-format-flink/v1.16/amoro-mixed-format-flink-runtime-1.16/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,32 +249,4 @@
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>deploy-maven-central</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin-version}</version>
<executions>
<execution>
<id>empty-javadoc-jar</id>
<goals>
<goal>jar</goal>
</goals>
<phase>package</phase>
<configuration>
<classifier>javadoc</classifier>
<classesDirectory>${basedir}/javadoc</classesDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
28 changes: 0 additions & 28 deletions ...mixed-format/amoro-mixed-format-flink/v1.17/amoro-mixed-format-flink-runtime-1.17/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,32 +250,4 @@
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>deploy-maven-central</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin-version}</version>
<executions>
<execution>
<id>empty-javadoc-jar</id>
<goals>
<goal>jar</goal>
</goals>
<phase>package</phase>
<configuration>
<classifier>javadoc</classifier>
<classesDirectory>${basedir}/javadoc</classesDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
28 changes: 0 additions & 28 deletions ...o-mixed-format/amoro-mixed-format-spark/v3.2/amoro-mixed-format-spark-runtime-3.2/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -447,32 +447,4 @@
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>deploy-maven-central</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin-version}</version>
<executions>
<execution>
<id>empty-javadoc-jar</id>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
<configuration>
<classifier>javadoc</classifier>
<classesDirectory>${basedir}/javadoc</classesDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
28 changes: 0 additions & 28 deletions ...o-mixed-format/amoro-mixed-format-spark/v3.3/amoro-mixed-format-spark-runtime-3.3/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,32 +470,4 @@
</plugin>
</plugins>
</build>

<profiles>
<profile>
<id>deploy-maven-central</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>${maven-jar-plugin-version}</version>
<executions>
<execution>
<id>empty-javadoc-jar</id>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
<configuration>
<classifier>javadoc</classifier>
<classesDirectory>${basedir}/javadoc</classesDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
49 changes: 10 additions & 39 deletions amoro-mixed-format/amoro-mixed-format-trino/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
<commons-io.version>2.4</commons-io.version>
<jol.version>0.16</jol.version>
<surefire-testng.version>3.2.5</surefire-testng.version>
<skip-build-mixed-format-trino>true</skip-build-mixed-format-trino>
</properties>

<dependencies>
Expand Down Expand Up @@ -553,7 +554,7 @@
<artifactId>maven-assembly-plugin</artifactId>
<version>2.6</version>
<configuration>
<skipAssembly>true</skipAssembly>
<skipAssembly>${skip-build-mixed-format-trino}</skipAssembly>
</configuration>
<executions>
<execution>
Expand All @@ -575,8 +576,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<skip>true</skip>
<skipMain>true</skipMain>
<skip>${skip-build-mixed-format-trino}</skip>
<skipMain>${skip-build-mixed-format-trino}</skipMain>
<verbose>true</verbose>
<fork>true</fork>
<compilerVersion>17</compilerVersion>
Expand All @@ -599,7 +600,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
<skip>${skip-build-mixed-format-trino}</skip>
</configuration>
</plugin>

Expand All @@ -622,40 +623,9 @@
<profiles>
<profile>
<id>build-mixed-format-trino</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<skip>false</skip>
<skipMain>false</skipMain>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.6</version>
<configuration>
<skipAssembly>false</skipAssembly>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<finalName>amoro-mixed-trino-${project.version}</finalName>
<descriptor>assembly.xml</descriptor>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<properties>
<skip-build-mixed-format-trino>false</skip-build-mixed-format-trino>
</properties>
</profile>
<profile>
<id>toolchain</id>
Expand Down Expand Up @@ -695,7 +665,8 @@
<skip>false</skip>
<java>
<excludes>
<exclude>**/org/apache/**/*.java</exclude>
<exclude>**/org/apache/hadoop/**/*.java</exclude>
<exclude>**/org/apache/parquet/**/*.java</exclude>
<exclude>**/io/trino/**/*.java</exclude>
</excludes>
</java>
Expand Down
10 changes: 7 additions & 3 deletions ...amoro-mixed-format-trino/src/main/java/io/trino/parquet/reader/TimestampColumnReader.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
Expand Down
4 changes: 2 additions & 2 deletions ...d-format-trino/src/main/java/org/apache/amoro/trino/DefaultMixedFormatCatalogFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

package org.apache.amoro.trino;

import io.trino.spi.classloader.ThreadContextClassLoader;
import org.apache.amoro.api.CatalogMeta;
import org.apache.amoro.mixed.MixedFormatCatalog;
import org.apache.amoro.mixed.CatalogLoader;
import org.apache.amoro.mixed.MixedFormatCatalog;
import org.apache.amoro.table.TableMetaStore;
import org.apache.amoro.utils.MixedCatalogUtil;
import io.trino.spi.classloader.ThreadContextClassLoader;

import javax.inject.Inject;

Expand Down
4 changes: 2 additions & 2 deletions ...rmat-trino/src/main/java/org/apache/amoro/trino/MixedFormatCatalogSupportTableSuffix.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,15 @@
package org.apache.amoro.trino;

import org.apache.amoro.TableFormat;
import org.apache.amoro.mixed.MixedFormatCatalog;
import org.apache.amoro.io.AuthenticatedFileIO;
import org.apache.amoro.mixed.MixedFormatCatalog;
import org.apache.amoro.op.UpdatePartitionProperties;
import org.apache.amoro.scan.ChangeTableIncrementalScan;
import org.apache.amoro.table.MixedTable;
import org.apache.amoro.table.BasicUnkeyedTable;
import org.apache.amoro.table.ChangeTable;
import org.apache.amoro.table.KeyedTable;
import org.apache.amoro.table.MetadataColumns;
import org.apache.amoro.table.MixedTable;
import org.apache.amoro.table.TableBuilder;
import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.table.TableMetaStore;
Expand Down
17 changes: 9 additions & 8 deletions ...mixed-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,6 @@
import static io.trino.plugin.hive.util.HiveUtil.isHiveSystemSchema;
import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;

import org.apache.amoro.mixed.MixedFormatCatalog;
import org.apache.amoro.table.MixedTable;
import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.trino.keyed.KeyedConnectorMetadata;
import org.apache.amoro.trino.keyed.KeyedTableHandle;
import org.apache.amoro.trino.unkeyed.IcebergMetadata;
import io.airlift.slice.Slice;
import io.trino.spi.TrinoException;
import io.trino.spi.connector.BeginTableExecuteResult;
Expand Down Expand Up @@ -55,9 +49,15 @@
import io.trino.spi.expression.ConnectorExpression;
import io.trino.spi.statistics.ComputedStatistics;
import io.trino.spi.statistics.TableStatistics;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.amoro.mixed.MixedFormatCatalog;
import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList;
import org.apache.amoro.shade.guava32.com.google.common.collect.Iterators;
import org.apache.amoro.table.MixedTable;
import org.apache.amoro.table.TableIdentifier;
import org.apache.amoro.trino.keyed.KeyedConnectorMetadata;
import org.apache.amoro.trino.keyed.KeyedTableHandle;
import org.apache.amoro.trino.unkeyed.IcebergMetadata;
import org.apache.iceberg.exceptions.NoSuchTableException;

import java.util.Collection;
import java.util.Iterator;
Expand Down Expand Up @@ -455,7 +455,8 @@ public void rollback() {

public MixedTable getMixedTable(SchemaTableName schemaTableName) {
return tableCache.computeIfAbsent(
schemaTableName, ignore -> mixedFormatCatalog.loadTable(getTableIdentifier(schemaTableName)));
schemaTableName,
ignore -> mixedFormatCatalog.loadTable(getTableIdentifier(schemaTableName)));
}

private TableIdentifier getTableIdentifier(SchemaTableName schemaTableName) {
Expand Down
6 changes: 3 additions & 3 deletions ...d-format-trino/src/main/java/org/apache/amoro/trino/MixedFormatConnectorSplitManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,16 @@

package org.apache.amoro.trino;

import org.apache.amoro.trino.keyed.KeyedConnectorSplitManager;
import org.apache.amoro.trino.keyed.KeyedTableHandle;
import org.apache.amoro.trino.unkeyed.IcebergSplitManager;
import io.trino.spi.connector.ConnectorSession;
import io.trino.spi.connector.ConnectorSplitManager;
import io.trino.spi.connector.ConnectorSplitSource;
import io.trino.spi.connector.ConnectorTableHandle;
import io.trino.spi.connector.ConnectorTransactionHandle;
import io.trino.spi.connector.Constraint;
import io.trino.spi.connector.DynamicFilter;
import org.apache.amoro.trino.keyed.KeyedConnectorSplitManager;
import org.apache.amoro.trino.keyed.KeyedTableHandle;
import org.apache.amoro.trino.unkeyed.IcebergSplitManager;

import javax.inject.Inject;

Expand Down
Loading

0 comments on commit 4f07ad0

Please sign in to comment.