Skip to content

Commit

Permalink
Full implemenation of RestTestsFromDocSnippetTask
Browse files Browse the repository at this point in the history
  • Loading branch information
@breskeby
breskeby committed Apr 9, 2024
1 parent 7c034a9 commit edecfac
Show file tree
Hide file tree
Showing 1,043 changed files with 177,407 additions and 2,162 deletions.
11 changes: 6 additions & 5 deletions ...al/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ abstract class RestTestsFromSnippetsTask extends SnippetsTask {
abstract FileOperations getFileOperations();

@Inject
RestTestsFromDocSnippetsTask(ObjectFactory objectFactory) {
RestTestsFromSnippetsTask(ObjectFactory objectFactory) {
testRoot = objectFactory.directoryProperty()
TestBuilder builder = new TestBuilder()
perSnippet = new Action<Snippet>() {
Expand All @@ -97,7 +97,7 @@ abstract class RestTestsFromSnippetsTask extends SnippetsTask {
DirectoryProperty getTestRoot() {
return testRoot
}
/**
/**
* Is this snippet a candidate for conversion to `// CONSOLE`?
*/
static isConsoleCandidate(Snippet snippet) {
Expand Down Expand Up @@ -202,6 +202,7 @@ abstract class RestTestsFromSnippetsTask extends SnippetsTask {
* Called each time a snippet is encountered. Tracks the snippets and
* calls buildTest to actually build the test.
*/

void handleSnippet(Snippet snippet) {
if (RestTestsFromSnippetsTask.isConsoleCandidate(snippet)) {
unconvertedCandidates.add(snippet.path.toString()
Expand Down Expand Up @@ -276,7 +277,7 @@ abstract class RestTestsFromSnippetsTask extends SnippetsTask {
* we're going to use these constructs, but we might so we
* output the skip just in case. */
current.println(" - skip:")
current.println(" features: ")
current.println(" features:")
current.println(" - default_shards")
current.println(" - stash_in_key")
current.println(" - stash_in_path")
Expand Down Expand Up @@ -332,8 +333,8 @@ abstract class RestTestsFromSnippetsTask extends SnippetsTask {

private void response(Snippet response) {
if (null == response.skip) {
current.println(" - match: ")
current.println(" \$body: ")
current.println(" - match:")
current.println(" \$body:")
replaceBlockQuote(response.contents).eachLine {
current.println(" $it")
}
Expand Down
1 change: 1 addition & 0 deletions ...-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,7 @@ class SnippetsTask extends DefaultTask {
throw new InvalidUserDataException("$location: Extra content "
+ "$message ('$cutOutNoNl') matching [$pattern]: $s")
}
println "s = $s"
while (m.find()) {
if (m.start() != offset) {
extraContent("between [$offset] and [${m.start()}]")
Expand Down
30 changes: 14 additions & 16 deletions ...d-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/DocSnippetTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,20 +128,12 @@ private static void escapeSubstitutions(
@TaskAction
void executeTask() {
for (File file : docs) {
String lastLanguage;
String name;
int lastLanguageLine;
Snippet snippet = null;
StringBuilder contents = null;
List<Map.Entry<String, String>> substitutions = new ArrayList<>();

parseDocFile(docs.getDir(), file, substitutions);
//
// emit(snippet, contents, defaultSubstitutions, substitutions);
System.out.println("file = " + file);
parseDocFiles(docs.getDir(), file, new ArrayList<>());
}
}

List<Snippet> parseDocFile(File rootDir, File docFile, List<Map.Entry<String, String>> substitutions) {
List<Snippet> parseDocFiles(File rootDir, File docFile, List<Map.Entry<String, String>> substitutions) {
String lastLanguage = null;
Snippet snippet = null;
String name = null;
Expand Down Expand Up @@ -213,10 +205,15 @@ List<Snippet> parseDocFile(File rootDir, File docFile, List<Map.Entry<String, St
continue;
}
emit(snippet, contents.toString(), defaultSubstitutions, substitutions);
substitutions = new ArrayList<>();;
snippet = null;
contents = null;
}
if (snippet != null) {
emit(snippet, contents.toString(), defaultSubstitutions, substitutions);
contents = null;
snippet = null;
substitutions = new ArrayList<>();
}
} catch (IOException e) {
e.printStackTrace();
Expand Down Expand Up @@ -270,7 +267,7 @@ private boolean testHandled(
int lineNumber,
String line,
Snippet snippet,
final List<Map.Entry<String, String>> substitutions
List<Map.Entry<String, String>> substitutions
) {
Matcher matcher = Pattern.compile("\\/\\/\s*TEST(\\[(.+)\\])?\s*").matcher(line);
if (matcher.matches()) {
Expand Down Expand Up @@ -321,14 +318,14 @@ private boolean testHandled(
return false;
}

public void extraContent(String message, String s, int offset, String location, String pattern) {
public void extraContent(String message, String content, int offset, String location, String pattern) {
StringBuilder cutOut = new StringBuilder();
cutOut.append(s.substring(offset - 6, offset));
cutOut.append(content.substring(offset - 6, offset));
cutOut.append('*');
cutOut.append(s.substring(offset, Math.min(offset + 5, s.length())));
cutOut.append(content.substring(offset, Math.min(offset + 5, content.length())));
String cutOutNoNl = cutOut.toString().replace("\n", "\\n");
throw new InvalidUserDataException(
location + ": Extra content " + message + " ('" + cutOutNoNl + "') matching [" + pattern + "]: " + s
location + ": Extra content " + message + " ('" + cutOutNoNl + "') matching [" + pattern + "]: " + content
);
}

Expand All @@ -352,6 +349,7 @@ protected void parse(String location, String content, String pattern, BiConsumer
testHandler.accept(m, offset == content.length());
}
if (offset == 0) {
System.out.println("content = " + content);
throw new InvalidUserDataException(location + ": Didn't match " + pattern + ": " + content);
}
if (offset != content.length()) {
Expand Down
17 changes: 16 additions & 1 deletion build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Docs2Plugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,14 @@
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.file.Directory;
import org.gradle.api.file.ProjectLayout;
import org.gradle.api.internal.file.FileOperations;
import org.gradle.api.plugins.JavaPluginExtension;
import org.gradle.api.provider.Provider;
import org.gradle.api.tasks.TaskProvider;

import java.util.Map;

import javax.inject.Inject;

public class Docs2Plugin implements Plugin<Project> {
Expand Down Expand Up @@ -85,6 +88,18 @@ public void apply(Project project) {
}
});
});

Provider<Directory> restRootDir = projectLayout.getBuildDirectory().dir("rest");
TaskProvider<RestTestsFromDocSnippetTask> buildRestTests = project.getTasks()
.register("buildRestTests", RestTestsFromDocSnippetTask.class, task -> {
task.setDefaultSubstitutions(commonDefaultSubstitutions);
task.getTestRoot().convention(restRootDir);
task.doFirst(task1 -> fileOperations.delete(restRootDir.get()));
});

// TODO: This effectively makes testRoot not customizable, which we don't do anyway atm
JavaPluginExtension byType = project.getExtensions().getByType(JavaPluginExtension.class);
byType.getSourceSets().getByName("yamlRestTest").getOutput().dir(Map.of("builtBy", buildRestTests), restRootDir);
}

}
95 changes: 65 additions & 30 deletions ...nal/src/main/java/org/elasticsearch/gradle/internal/docs/RestTestsFromDocSnippetTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,24 +31,39 @@
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import javax.inject.Inject;

public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {

private Map<String, String> setups = new HashMap<>();

private Map<String, String> teardowns = new HashMap();

/**
* Test setups defined in the build instead of the docs so they can be
* shared between many doc files.
*/
@Input
Map<String, String> setups = new HashMap<>();
public Map<String, String> getSetups() {
return setups;
}

public void setSetups(Map<String, String> setups) {
this.setups = setups;
}

/**
* Test teardowns defined in the build instead of the docs so they can be
* shared between many doc files.
*/
@Input
Map<String, String> teardowns = new HashMap();
public Map<String, String> getTeardowns() {
return teardowns;
}

public void setTeardowns(Map<String, String> teardowns) {
this.teardowns = teardowns;
}

/**
* A list of files that contain snippets that *probably* should be
Expand All @@ -57,17 +72,33 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
* If there are unconverted snippets not in this list then this task will
* fail. All files are paths relative to the docs dir.
*/
private List<String> expectedUnconvertedCandidates;

@Input
List<String> expectedUnconvertedCandidates = new ArrayList<>();
public List<String> getExpectedUnconvertedCandidates() {
return expectedUnconvertedCandidates;
}

public void setExpectedUnconvertedCandidates(List<String> expectedUnconvertedCandidates) {
this.expectedUnconvertedCandidates = expectedUnconvertedCandidates;
}

/**
* Root directory of the tests being generated. To make rest tests happy
* we generate them in a testRoot which is contained in this directory.
*/
private DirectoryProperty testRoot;

private Set<String> names = new HashSet<>();

@Internal
Set<String> names = new HashSet<>();
public Set<String> getNames() {
return names;
}

public void setNames(Set<String> names) {
this.names = names;
}

@Inject
public abstract FileOperations getFileOperations();
Expand All @@ -76,6 +107,7 @@ public abstract class RestTestsFromDocSnippetTask extends DocSnippetTask {
* Root directory containing all the files generated by this task. It is
* contained within testRoot.
*/
@OutputDirectory
File getOutputRoot() {
return new File(testRoot.get().getAsFile(), "/rest-api-spec/test");
}
Expand All @@ -92,8 +124,8 @@ public RestTestsFromDocSnippetTask(ObjectFactory objectFactory) {

perSnippet = snippet -> builder.handleSnippet(snippet);
doLast(task -> {
builder.checkUnconverted();
builder.finishLastTest();
builder.checkUnconverted();
});
}

Expand All @@ -104,16 +136,19 @@ private class TestBuilder {
*/
private static final List BAD_LANGUAGES = List.of("json", "javascript");

static final String method = "?<method>GET|PUT|POST|HEAD|OPTIONS|DELETE)";
static final String pathAndQuery = "(?<pathAndQuery>[^\n]+)";
static final String badBody = "GET|PUT|POST|HEAD|OPTIONS|DELETE|startyaml|#";
static final String body = "(?<body>(?:\n(?!$badBody)[^\n]+)+)";
static final String rawRequest = "(?:" + method + "\s" + pathAndQuery + body + "?)";
static final String yamlRequest = "(?:startyaml(?s)(?<yaml>.+?)(?-s)endyaml)";
static final String nonComment = "(?:" + rawRequest + "|" + yamlRequest + ")";
static final String comment = "(?<comment>#.+)";
String method = "(?<method>GET|PUT|POST|HEAD|OPTIONS|DELETE)";
String pathAndQuery = "(?<pathAndQuery>[^\\n]+)";

String badBody = "GET|PUT|POST|HEAD|OPTIONS|DELETE|startyaml|#";
String body = "(?<body>(?:\\n(?!" + badBody + ")[^\\n]+)+)";

String rawRequest = "(?:" + method + "\\s+" + pathAndQuery + body + "?)";

String yamlRequest = "(?:startyaml(?s)(?<yaml>.+?)(?-s)endyaml)";
String nonComment = "(?:" + rawRequest + "|" + yamlRequest + ")";
String comment = "(?<comment>#.+)";

private static final String SYNTAX = "(?:" + comment + "|" + nonComment + ")";
String SYNTAX = "(?:" + comment + "|" + nonComment + ")\\n+";

/**
* Files containing all snippets that *probably* should be converted
Expand Down Expand Up @@ -160,20 +195,20 @@ public void handleSnippet(Snippet snippet) {
previousTest = snippet;
return;
}
if (snippet.testResponse || snippet.language == "console-result") {
if (snippet.testResponse || snippet.language.equals("console-result")) {
if (previousTest == null) {
throw new InvalidUserDataException("$snippet: No paired previous test");
throw new InvalidUserDataException(snippet + ": No paired previous test");
}
if (previousTest.path != snippet.path) {
throw new InvalidUserDataException("$snippet: Result can't be first in file");
if (previousTest.path.equals(snippet.path) == false) {
throw new InvalidUserDataException(snippet + ": Result can't be first in file");
}
response(snippet);
return;
}
if ((snippet.language == "js") && (snippet.console)) {
throw new InvalidUserDataException("$snippet: Use `[source,console]` instead of `// CONSOLE`.");
if (("js".equals(snippet.language)) && snippet.console != null && snippet.console) {
throw new InvalidUserDataException(snippet + ": Use `[source,console]` instead of `// CONSOLE`.");
}
if (snippet.test || snippet.language == "console") {
if (snippet.test || snippet.language.equals("console")) {
test(snippet);
previousTest = snippet;
return;
Expand All @@ -187,7 +222,7 @@ private void test(Snippet test) {
if (test.continued) {
/* Catch some difficult to debug errors with // TEST[continued]
* and throw a helpful error message. */
if (previousTest == null || previousTest.path != test.path) {
if (previousTest == null || previousTest.path.equals(test.path) == false) {
throw new InvalidUserDataException("// TEST[continued] " + "cannot be on first snippet in a file: " + test);
}
if (previousTest != null && previousTest.testSetup) {
Expand All @@ -211,7 +246,7 @@ private void test(Snippet test) {
* we're going to use these constructs, but we might so we
* output the skip just in case. */
current.println(" - skip:");
current.println(" features: ");
current.println(" features:");
current.println(" - default_shards");
current.println(" - stash_in_key");
current.println(" - stash_in_path");
Expand All @@ -223,7 +258,7 @@ private void test(Snippet test) {
throw new InvalidUserDataException("Continued snippets " + "can't be skipped");
}
current.println(" - always_skip");
current.println(" reason: $test.skip");
current.println(" reason: " + test.skip);
}
if (test.setup != null) {
setup(test);
Expand All @@ -238,8 +273,8 @@ private void test(Snippet test) {

private void response(Snippet response) {
if (null == response.skip) {
current.println(" - match: ");
current.println(" \\$body: ");
current.println(" - match:");
current.println(" $body:");
replaceBlockQuote(response.contents).lines().forEach(line -> current.println(" " + line));
}
}
Expand Down Expand Up @@ -338,11 +373,11 @@ void emitDo(
for (String param : query.split("&")) {
String[] tokenizedQuery = param.split("=");
String paramName = tokenizedQuery[0];
String paramValue = tokenizedQuery.length > 1 ? tokenized[1] : null;
String paramValue = tokenizedQuery.length > 1 ? tokenizedQuery[1] : null;
if (paramValue == null) {
paramValue = "";
}
current.println(" $name: \"" + paramValue + "\"");
current.println(" " + paramName + ": \"" + paramValue + "\"");
}
}
if (body != null) {
Expand Down Expand Up @@ -399,7 +434,7 @@ private void body(Snippet snippet, boolean inSetup) {
}

private PrintWriter setupCurrent(Snippet test) {
if (lastDocsPath == test.path) {
if (test.path.equals(lastDocsPath)) {
return current;
}
names.clear();
Expand Down
2 changes: 1 addition & 1 deletion build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/docs/Snippet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ String getLocation() {

private void assertValidCurlInput() {
// Try to detect snippets that contain `curl`
if (language == "sh" || language == "shell") {
if ("sh".equals(language) || "shell".equals(language)) {
curl = contents.contains("curl");
if (console == Boolean.FALSE && curl == false) {
throw new InvalidUserDataException(name + ": " + "No need for NOTCONSOLE if snippet doesn't " + "contain `curl`.");
Expand Down
Loading

0 comments on commit edecfac

Please sign in to comment.