-
Notifications
You must be signed in to change notification settings - Fork 3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add Alluxio file system #21603
Add Alluxio file system #21603
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,125 @@ | ||
<?xml version="1.0" encoding="UTF-8"?> | ||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | ||
<modelVersion>4.0.0</modelVersion> | ||
|
||
<parent> | ||
<groupId>io.trino</groupId> | ||
<artifactId>trino-root</artifactId> | ||
<version>460-SNAPSHOT</version> | ||
<relativePath>../../pom.xml</relativePath> | ||
</parent> | ||
|
||
<artifactId>trino-filesystem-alluxio</artifactId> | ||
<description>Trino Filesystem - Alluxio</description> | ||
|
||
<properties> | ||
<air.main.basedir>${project.parent.basedir}</air.main.basedir> | ||
<air.compiler.fail-warnings>true</air.compiler.fail-warnings> | ||
</properties> | ||
|
||
<dependencies> | ||
<dependency> | ||
<groupId>com.google.inject</groupId> | ||
<artifactId>guice</artifactId> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>io.trino</groupId> | ||
<artifactId>trino-filesystem</artifactId> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>io.trino</groupId> | ||
<artifactId>trino-memory-context</artifactId> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>io.trino</groupId> | ||
<artifactId>trino-spi</artifactId> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>org.alluxio</groupId> | ||
<artifactId>alluxio-core-client-fs</artifactId> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>org.alluxio</groupId> | ||
<artifactId>alluxio-core-common</artifactId> | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This uses Why are the following needed?
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @JiamingMai please mark this as resolved if it's already addressed or reply with follow-up comments There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ping There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. aws excluded, all other unneeded dependencies excluded as well. |
||
</dependency> | ||
|
||
<dependency> | ||
<groupId>org.alluxio</groupId> | ||
<artifactId>alluxio-core-transport</artifactId> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>com.github.docker-java</groupId> | ||
<artifactId>docker-java-api</artifactId> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>io.airlift</groupId> | ||
<artifactId>junit-extensions</artifactId> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>io.trino</groupId> | ||
<artifactId>trino-filesystem</artifactId> | ||
<version>${project.version}</version> | ||
<classifier>tests</classifier> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>io.trino</groupId> | ||
<artifactId>trino-spi</artifactId> | ||
<type>test-jar</type> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>io.trino</groupId> | ||
<artifactId>trino-testing</artifactId> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>io.trino</groupId> | ||
<artifactId>trino-testing-containers</artifactId> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>org.assertj</groupId> | ||
<artifactId>assertj-core</artifactId> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>org.junit.jupiter</groupId> | ||
<artifactId>junit-jupiter-api</artifactId> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>org.junit.jupiter</groupId> | ||
<artifactId>junit-jupiter-engine</artifactId> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>org.testcontainers</groupId> | ||
<artifactId>junit-jupiter</artifactId> | ||
<scope>test</scope> | ||
</dependency> | ||
|
||
<dependency> | ||
<groupId>org.testcontainers</groupId> | ||
<artifactId>testcontainers</artifactId> | ||
<scope>test</scope> | ||
</dependency> | ||
</dependencies> | ||
</project> |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
/* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
package io.trino.filesystem.alluxio; | ||
|
||
import alluxio.client.file.URIStatus; | ||
import io.trino.filesystem.FileEntry; | ||
import io.trino.filesystem.FileIterator; | ||
import io.trino.filesystem.Location; | ||
|
||
import java.io.IOException; | ||
import java.time.Instant; | ||
import java.util.Iterator; | ||
import java.util.List; | ||
import java.util.Optional; | ||
|
||
import static io.trino.filesystem.alluxio.AlluxioUtils.convertToLocation; | ||
import static java.util.Objects.requireNonNull; | ||
|
||
public class AlluxioFileIterator | ||
implements FileIterator | ||
{ | ||
private final Iterator<URIStatus> files; | ||
private final String mountRoot; | ||
|
||
public AlluxioFileIterator(List<URIStatus> files, String mountRoot) | ||
{ | ||
this.files = requireNonNull(files.iterator(), "files is null"); | ||
this.mountRoot = requireNonNull(mountRoot, "mountRoot is null"); | ||
} | ||
|
||
@Override | ||
public boolean hasNext() | ||
throws IOException | ||
{ | ||
return files.hasNext(); | ||
} | ||
|
||
@Override | ||
public FileEntry next() | ||
throws IOException | ||
{ | ||
if (!hasNext()) { | ||
return null; | ||
} | ||
URIStatus fileStatus = files.next(); | ||
Location location = convertToLocation(fileStatus, mountRoot); | ||
return new FileEntry( | ||
location, | ||
fileStatus.getLength(), | ||
Instant.ofEpochMilli(fileStatus.getLastModificationTimeMs()), | ||
Optional.empty()); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is
io.netty:netty-all
required? Can we exclude it?Why do we need
org.rocksdb:rocksdbjni
? Can we exclude it?Please exclude
log4j-slf4j-impl
as that makes SLF4J use Log4j as the implementation.Do we need both
slf4j-api
andlog4j-api
? Isalluxio-core-client-fs
directly using both libraries?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@JiamingMai please mark this as resolved if it's already addressed or reply with follow-up comments
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
ping
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
netty is required. other unneeded dependencies excluded