Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DOCS] Move x-pack docs to docs/reference dir #99209

Merged
merged 5 commits into from
Sep 12, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
252 changes: 250 additions & 2 deletions docs/build.gradle
Copy link
Contributor Author

@jrodewig jrodewig Sep 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ ext.docsFileTree = fileTree(projectDir) {
exclude 'README.asciidoc'
// Broken code snippet tests
exclude 'reference/graph/explore.asciidoc'
// These files simply don't pass yet. We should figure out how to fix them.
exclude 'reference/watcher/reference/actions.asciidoc'
exclude 'reference/rest-api/security/ssl.asciidoc'
Comment on lines +27 to +29
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if (BuildParams.inFipsJvm) {
// We don't support this component in FIPS 140
exclude 'reference/ingest/processors/attachment.asciidoc'
Expand All @@ -48,7 +51,13 @@ tasks.named("buildRestTests").configure {
'reference/ml/anomaly-detection/apis/post-data.asciidoc',
'reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc',
'reference/ml/anomaly-detection/apis/update-snapshot.asciidoc',
'reference/ml/anomaly-detection/apis/update-job.asciidoc'
'reference/ml/anomaly-detection/apis/update-job.asciidoc',
'reference/security/authentication/user-cache.asciidoc',
'reference/security/authorization/run-as-privilege.asciidoc',
'reference/security/ccs-clients-integrations/http.asciidoc',
'reference/rest-api/watcher/put-watch.asciidoc',
'reference/rest-api/watcher/stats.asciidoc',
'reference/watcher/example-watches/watching-time-series-data.asciidoc'
]
}

Expand All @@ -74,7 +83,6 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {

// enable regexes in painless so our tests don't complain about example snippets that use them
setting 'script.painless.regex.enabled', 'true'
setting 'xpack.security.enabled', 'false'
setting 'path.repo', "${buildDir}/cluster/shared/repo"
Closure configFile = {
extraConfigFile it, file("src/test/cluster/config/$it")
Expand All @@ -101,6 +109,42 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {

requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")

extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("oidc/op-jwks.json")
extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml")
extraConfigFile 'testClient.crt', project(':x-pack:plugin:security').file("src/test/resources/org/elasticsearch/xpack/security/action/pki_delegation/testClient.crt")
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.authc.api_key.enabled', 'true'
setting 'xpack.security.authc.token.enabled', 'true'
// disable the ILM history for doc tests to avoid potential lingering tasks that'd cause test flakiness
setting 'indices.lifecycle.history_index_enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.authc.realms.file.file.order', '0'
setting 'xpack.security.authc.realms.native.native.order', '1'
setting 'xpack.security.authc.realms.oidc.oidc1.order', '2'
setting 'xpack.security.authc.realms.oidc.oidc1.op.issuer', 'http://127.0.0.1:8080'
setting 'xpack.security.authc.realms.oidc.oidc1.op.authorization_endpoint', "http://127.0.0.1:8080/c2id-login"
setting 'xpack.security.authc.realms.oidc.oidc1.op.token_endpoint', "http://127.0.0.1:8080/c2id/token"
setting 'xpack.security.authc.realms.oidc.oidc1.op.jwkset_path', 'op-jwks.json'
setting 'xpack.security.authc.realms.oidc.oidc1.rp.redirect_uri', 'https://my.fantastic.rp/cb'
setting 'xpack.security.authc.realms.oidc.oidc1.rp.client_id', 'elasticsearch-rp'
keystore 'xpack.security.authc.realms.oidc.oidc1.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
setting 'xpack.security.authc.realms.oidc.oidc1.rp.response_type', 'id_token'
setting 'xpack.security.authc.realms.oidc.oidc1.claims.principal', 'sub'
setting 'xpack.security.authc.realms.pki.pki1.order', '3'
setting 'xpack.security.authc.realms.pki.pki1.certificate_authorities', '[ "testClient.crt" ]'
setting 'xpack.security.authc.realms.pki.pki1.delegation.enabled', 'true'
setting 'xpack.security.authc.realms.saml.saml1.order', '4'
setting 'xpack.security.authc.realms.saml.saml1.sp.logout', 'https://kibana.org/logout'
setting 'xpack.security.authc.realms.saml.saml1.idp.entity_id', 'https://my-idp.org'
setting 'xpack.security.authc.realms.saml.saml1.idp.metadata.path', 'idp-docs-metadata.xml'
setting 'xpack.security.authc.realms.saml.saml1.sp.entity_id', 'https://kibana.org'
setting 'xpack.security.authc.realms.saml.saml1.sp.acs', 'https://kibana.org/api/security/saml/callback'
setting 'xpack.security.authc.realms.saml.saml1.attributes.principal', 'uid'
setting 'xpack.security.authc.realms.saml.saml1.attributes.name', 'urn:oid:2.5.4.3'

user username: 'test_admin'
user username: 'test_user'

// build the cluster with all plugins
project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
/* Skip repositories. We just aren't going to be able to test them so it
Expand Down Expand Up @@ -1653,6 +1697,210 @@ setups['setup-snapshots'] = setups['setup-repository'] + '''
body: |
#atomic_red_data#
'''

setups['my_inactive_watch'] = '''
- do:
watcher.put_watch:
id: "my_watch"
active: false
body: >
{
"trigger": {
"schedule": {
"hourly": {
"minute": [ 0, 5 ]
}
}
},
"input": {
"simple": {
"payload": {
"send": "yes"
}
}
},
"condition": {
"always": {}
},
"actions": {
"test_index": {
"index": {
"index": "test"
}
}
}
}
- match: { _id: "my_watch" }
'''

setups['my_active_watch'] = setups['my_inactive_watch'].replace(
'active: false', 'active: true')

setups['role_mapping'] = '''
- do:
security.put_role_mapping:
name: "mapping1"
body: >
{
"enabled": true,
"roles": [ "user" ],
"rules": { "field": { "username": "*" } }
}
'''

setups['admin_role'] = '''
- do:
security.put_role:
name: "my_admin_role"
body: >
{
"cluster": ["all"],
"indices": [
{"names": ["index1", "index2" ], "privileges": ["all"], "field_security" : {"grant" : [ "title", "body" ]}}
],
"run_as": [ "other_user" ],
"metadata" : {"version": 1}
}
'''
setups['jacknich_user'] = '''
- do:
security.put_user:
username: "jacknich"
body: >
{
"password" : "l0ng-r4nd0m-p@ssw0rd",
"roles" : [ "admin", "other_role1" ],
"full_name" : "Jack Nicholson",
"email" : "[email protected]",
"metadata" : { "intelligence" : 7 }
}
- do:
security.activate_user_profile:
body: >
{
"grant_type": "password",
"username": "jacknich",
"password" : "l0ng-r4nd0m-p@ssw0rd"
}
'''
setups['app0102_privileges'] = '''
- do:
security.put_privileges:
body: >
{
"myapp": {
"read": {
"application": "myapp",
"name": "read",
"actions": [
"data:read/*",
"action:login" ],
"metadata": {
"description": "Read access to myapp"
}
}
}
}
'''
setups['service_token42'] = '''
- do:
security.create_service_token:
namespace: elastic
service: fleet-server
name: token42
'''
setups['user_profiles'] = '''
- do:
security.put_user:
username: "jacknich"
body: >
{
"password" : "l0ng-r4nd0m-p@ssw0rd",
"roles" : [ "admin", "other_role1" ],
"full_name" : "Jack Nicholson",
"email" : "[email protected]"
}
- do:
security.put_user:
username: "jackrea"
body: >
{
"password" : "l0ng-r4nd0m-p@ssw0rd",
"roles" : [ "admin" ],
"full_name" : "Jack Reacher",
"email" : "[email protected]"
}
- do:
security.put_user:
username: "jackspa"
body: >
{
"password" : "l0ng-r4nd0m-p@ssw0rd",
"roles" : [ "user" ],
"full_name" : "Jack Sparrow",
"email" : "[email protected]"
}
- do:
security.activate_user_profile:
body: >
{
"grant_type": "password",
"username": "jacknich",
"password" : "l0ng-r4nd0m-p@ssw0rd"
}
- do:
security.activate_user_profile:
body: >
{
"grant_type": "password",
"username": "jackrea",
"password" : "l0ng-r4nd0m-p@ssw0rd"
}
- do:
security.activate_user_profile:
body: >
{
"grant_type": "password",
"username": "jackspa",
"password" : "l0ng-r4nd0m-p@ssw0rd"
}
# jacknich
- do:
security.update_user_profile_data:
uid: "u_79HkWkwmnBH5gqFKwoxggWPjEBOur1zLPXQPEl1VBW0_0"
body: >
{
"labels": {
"direction": "north"
},
"data": {
"app1": {
"key1": "value1"
}
}
}
# jackrea
- do:
security.update_user_profile_data:
uid: "u_P_0BMHgaOK3p7k-PFWUCbw9dQ-UFjt01oWJ_Dp2PmPc_0"
body: >
{
"labels": {
"direction": "west"
}
}
# jackspa
- do:
security.update_user_profile_data:
uid: "u_8RKO7AKfEbSiIHZkZZ2LJy2MUSDPWDr3tMI_CkIGApU_0"
body: >
{
"labels": {
"direction": "south"
}
}
'''

// fake data used by the correlation bucket agg
buildRestTests.setups['correlate_latency'] = '''
- do:
Expand Down
2 changes: 1 addition & 1 deletion docs/reference/rest-api/usage.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
------------------------------------------------------------
GET /_xpack/usage
------------------------------------------------------------
// TEST[s/usage/usage?filter_path=-watcher.execution.actions.logging*/]
// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*/]
// This response filter removes watcher logging results if they are included
// to avoid errors in the CI builds.

Expand Down
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.core.Strings;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ClientYamlDocsTestClient;
Expand All @@ -46,10 +49,12 @@
import java.util.List;
import java.util.Map;

import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
import static org.hamcrest.Matchers.is;

//The default 20 minutes timeout isn't always enough, but Darwin CI hosts are incredibly slow...
@TimeoutSuite(millis = 40 * TimeUnits.MINUTE)
Expand Down Expand Up @@ -222,6 +227,74 @@ protected boolean isXpackInfoTest() {
return testName != null && (testName.contains("/info/") || testName.contains("\\info\\"));
}

private static final String USER_TOKEN = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()));

/**
* All tests run as a an administrative user but use <code>es-shield-runas-user</code> to become a less privileged user.
*/
@Override
protected Settings restClientSettings() {
return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", USER_TOKEN).build();
}

/**
* Deletes users after every test just in case any test adds any.
*/
@After
public void deleteUsers() throws Exception {
ClientYamlTestResponse response = getAdminExecutionContext().callApi("security.get_user", emptyMap(), emptyList(), emptyMap());
@SuppressWarnings("unchecked")
Map<String, Object> users = (Map<String, Object>) response.getBody();
for (String user : users.keySet()) {
Map<?, ?> metadataMap = (Map<?, ?>) ((Map<?, ?>) users.get(user)).get("metadata");
Boolean reserved = metadataMap == null ? null : (Boolean) metadataMap.get("_reserved");
if (reserved == null || reserved == false) {
logger.warn("Deleting leftover user {}", user);
getAdminExecutionContext().callApi("security.delete_user", singletonMap("username", user), emptyList(), emptyMap());
}
}
}

/**
* Re-enables watcher after every test just in case any test disables it.
*/
@After
public void reenableWatcher() throws Exception {
if (isWatcherTest()) {
assertBusy(() -> {
ClientYamlTestResponse response = getAdminExecutionContext().callApi("watcher.stats", emptyMap(), emptyList(), emptyMap());
String state = (String) response.evaluate("stats.0.watcher_state");

switch (state) {
case "stopped":
ClientYamlTestResponse startResponse = getAdminExecutionContext().callApi(
"watcher.start",
emptyMap(),
emptyList(),
emptyMap()
);
boolean isAcknowledged = (boolean) startResponse.evaluate("acknowledged");
assertThat(isAcknowledged, is(true));
throw new AssertionError("waiting until stopped state reached started state");
case "stopping":
throw new AssertionError("waiting until stopping state reached stopped state to start again");
case "starting":
throw new AssertionError("waiting until starting state reached started state");
case "started":
// all good here, we are done
break;
default:
throw new AssertionError("unknown state[" + state + "]");
}
});
}
}

protected boolean isWatcherTest() {
String testName = getTestName();
return testName != null && (testName.contains("watcher/") || testName.contains("watcher\\"));
}

/**
* Compares the results of running two analyzers against many random
* strings. The goal is to figure out if two anlayzers are "the same" by
Expand Down
Loading