Skip to content

Commit

Permalink
#505
Browse files Browse the repository at this point in the history
  • Loading branch information
@luigi-asprino
luigi-asprino committed Oct 29, 2024
1 parent c7df672 commit 6f74890
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 78 deletions.
12 changes: 6 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,12 +568,12 @@ usage: java -jar sparql.anything-<version> -q query [-f <output format>] [-v
and not using blank nodes)
-e,--explain OPTIONAL - Explain query
execution
-l,--load <load> OPTIONAL - The path to one RDF
file or a folder including a set
of files to be loaded. When
present, the data is loaded in
memory and the query executed
against it.
-l,--load <URL or filepath> OPTIONAL - The path or the URL to
one RDF file or a filepath to a
folder including a set of files
to be loaded. When present, the
data is loaded in memory and the
query executed against it.
-f,--format <string> OPTIONAL - Format of the output
file. Supported values: JSON,
XML, CSV, TEXT, TTL, NT, NQ.
Expand Down
4 changes: 2 additions & 2 deletions sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/CLI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,8 @@ void init(){
options.addOption(Option.builder(EXPLAIN).argName("explain").hasArg(false)
.desc("OPTIONAL - Explain query execution").longOpt(EXPLAIN_LONG).build());

options.addOption(Option.builder(LOAD).argName("load").hasArg().desc(
"OPTIONAL - The path to one RDF file or a folder including a set of files to be loaded. When present, the data is loaded in memory and the query executed against it.")
options.addOption(Option.builder(LOAD).argName("URL or filepath").hasArg().desc(
"OPTIONAL - The path or the URL to one RDF file or a filepath to a folder including a set of files to be loaded. When present, the data is loaded in memory and the query executed against it.")
.longOpt(LOAD_LONG).build());

options.addOption(Option.builder(FORMAT).argName("string").hasArg().desc(
Expand Down
142 changes: 74 additions & 68 deletions sparql-anything-cli/src/main/java/io/github/sparqlanything/cli/SPARQLAnything.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@

import java.io.*;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
Expand Down Expand Up @@ -463,75 +464,8 @@ public static void main(String[] args) throws Exception {
if(logger.isTraceEnabled()) {
logger.trace("[time] After init: {}", System.currentTimeMillis() - duration);
}
Dataset kb = null;
String load = cli.getLoad();
if (load != null) {

logger.info("Loading data from: {}", load);
if(logger.isTraceEnabled()) {
logger.trace("[time] Before load: {}", System.currentTimeMillis() - duration);
}
// XXX Check if load is a URI first
File loadSource;
try{
loadSource = new File(new URL(load).toURI());
}catch(MalformedURLException e){
loadSource = new File(load);
}
if (loadSource.isDirectory()) {

logger.info("Loading files from directory: {}", loadSource);
// If directory, load all files
List<File> list = new ArrayList<>();
Collection<File> files = FileUtils.listFiles(loadSource, null, true);
for (File f : files) {
logger.info("Adding file to be loaded: {}", f);
list.add(f);
}
kb = DatasetFactory.createGeneral();
for (File f : list) {
try {
Model m = ModelFactory.createDefaultModel();
// read into the model.
m.read(f.getAbsolutePath());
kb.addNamedModel(f.toURI().toString(), m);
} catch (Exception e) {
logger.error("An error occurred while loading {}", f);
logger.error(" - Problem was: {}", e.getMessage());
if(logger.isDebugEnabled()){
logger.error("",e);
}
}
}
logger.info("Loaded {} triples", kb.asDatasetGraph().getUnionGraph().size());
} else if (loadSource.isFile()) {
// If it is a file, load it
logger.info("Load file: {}", loadSource);
Path base = Paths.get(".");
try{
Path p = loadSource.toPath();
if(!p.isAbsolute()){
p = base.relativize(loadSource.toPath());
}
kb = DatasetFactory.create(p.toFile().toURI().toString());
} catch (Exception e) {
logger.error("An error occurred while loading {}", loadSource);
logger.error(" - Problem was: ", e);
}
} else {
if(!loadSource.exists()){
logger.error("Option 'load' failed (resource does not exist): {}", loadSource);
}else {
logger.error("Option 'load' failed (not a file or directory): {}", loadSource);
}
return;
}
if(logger.isTraceEnabled()) {
logger.trace("[time] After load: {}", System.currentTimeMillis() - duration);
}
} else {
kb = DatasetFactory.createGeneral();
}
Dataset kb = createDataset(cli.getLoad());

String outputFileName = cli.getOutputFile();
String outputPattern = cli.getOutputPattern();
Expand Down Expand Up @@ -598,6 +532,78 @@ public static void main(String[] args) throws Exception {
}
}

private static Dataset createDataset(String load) {
Dataset kb = DatasetFactory.createGeneral();
if (load != null) {

logger.info("Loading data from: {}", load);
if (logger.isTraceEnabled()) {
logger.trace("[time] Before load: {}", System.currentTimeMillis() - duration);
}
// XXX Check if load is a URI first
File loadSource;
try {
loadSource = new File(new URL(load).toURI());
} catch (MalformedURLException | URISyntaxException e) {
loadSource = new File(load);
} catch (IllegalArgumentException e) {
Model m = ModelFactory.createDefaultModel();
RDFDataMgr.read(m, load);
kb.addNamedModel(load, m);
return kb;
}
if (loadSource.isDirectory()) {

logger.info("Loading files from directory: {}", loadSource);
// If directory, load all files
Collection<File> files = FileUtils.listFiles(loadSource, null, true);
for (File f : files) {
logger.info("Adding file to be loaded: {}", f);
try {
Model m = ModelFactory.createDefaultModel();
// read into the model.
m.read(f.getAbsolutePath());
kb.addNamedModel(f.toURI().toString(), m);
} catch (Exception e) {
logger.error("An error occurred while loading {}", f);
logger.error(" - Problem was: {}", e.getMessage());
if (logger.isDebugEnabled()) {
logger.error("", e);
}
}
}

logger.info("Loaded {} triples", kb.asDatasetGraph().getUnionGraph().size());
} else if (loadSource.isFile()) {
// If it is a file, load it
logger.info("Load file: {}", loadSource);
Path base = Paths.get(".");
try {
Path p = loadSource.toPath();
if (!p.isAbsolute()) {
p = base.relativize(loadSource.toPath());
}
kb = DatasetFactory.create(p.toFile().toURI().toString());
} catch (Exception e) {
logger.error("An error occurred while loading {}", loadSource);
logger.error(" - Problem was: ", e);
}
} else {
if (!loadSource.exists()) {
logger.error("Option 'load' failed (resource does not exist): {}", loadSource);
} else {
logger.error("Option 'load' failed (not a file or directory): {}", loadSource);
}
return kb;
}
if (logger.isTraceEnabled()) {
logger.trace("[time] After load: {}", System.currentTimeMillis() - duration);
}
}

return kb;
}

public static String callMain(String[] args) throws Exception {
// Thanks to: https://stackoverflow.com/a/8708357/1035608
// Create a stream to hold the output
Expand Down
11 changes: 11 additions & 0 deletions sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/CLITest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,4 +92,15 @@ public void remoteQuery() throws Exception {
String queryFile = "https://sparql-anything.cc/examples/CLITestOnFileQuery.sparql";
query(f, queryFile);
}

@Test
public void testRemoteLoad() throws Exception {
Assume.assumeTrue(HTTPHelper.checkHostIsReachable("https://sparql-anything.cc"));
String q = Objects.requireNonNull(getClass().getClassLoader().getResource("count-triples.sparql")).toString();
String l = "https://sparql-anything.cc/examples/example.ttl";
String out = SPARQLAnything.callMain(new String[]{
"-q", q, "-l", l
});
Assert.assertTrue(out.contains("10"));
}
}
4 changes: 2 additions & 2 deletions sparql-anything-cli/src/test/java/io/github/sparqlanything/cli/LoadSubdirsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import org.apache.jena.sys.JenaSystem;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -38,8 +39,7 @@ public void test() throws Exception {

String q = Objects.requireNonNull(getClass().getClassLoader().getResource("count-triples.sparql")).toString();
String d = Objects.requireNonNull(getClass().getClassLoader().getResource("./load-subdirs")).toURI().toString();
SPARQLAnything sa = new SPARQLAnything();
String out = sa.callMain(new String[]{
String out = SPARQLAnything.callMain(new String[]{
"-q", q, "-l", d
});
Assert.assertTrue(out.contains("51"));
Expand Down

0 comments on commit 6f74890

Please sign in to comment.