From f3477c32da31341e3a763a608beddba282b31e07 Mon Sep 17 00:00:00 2001 From: "pavel.stastny" Date: Sun, 1 Nov 2020 23:13:32 +0100 Subject: [PATCH] Issue #774 --- .../kramerius/auth/basic/AuthFilter.java | 2 +- .../auth/thirdparty/AuthenticatedUsers.java | 10 +- .../shibb/rules/objects/AttributeValue.java | 4 +- .../shibb/rules/objects/HeaderValue.java | 4 +- .../rules/objects/ShibbolethContext.java | 12 +- .../thirdparty/shibb/rules/objects/Value.java | 4 +- .../social/OpenIDAuthenticatedUsers.java | 8 +- .../thirdparty/social/utils/OpenIDFlag.java | 12 +- .../thirdparty/utils/RemoteUsersUtils.java | 23 +- .../auth/shibb/RequestSupportForTests.java | 2 +- .../ShibbolethRulesFirstAndLastnameTest.java | 2 +- .../ShibbolethRulesRegularExpressionTest.java | 4 +- .../auth/shibb/utils/ShibbolethUtilsTest.java | 26 +- .../criteria/utils/CriteriaDNNTUtils.java | 86 +----- .../impl/DatabaseStatisticsAccessLogImpl.java | 262 ++++++++++++++---- .../java/cz/cas/lib/knav/ApplyMWUtils.java | 13 +- .../ParametrizedIterateRepoMovingWall.java | 2 +- .../kramerius/services/NKPIterations.java | 35 --- 18 files changed, 256 insertions(+), 255 deletions(-) delete mode 100644 solr-migration/src/test/java/cz/incad/kramerius/services/NKPIterations.java diff --git a/authfilters/src/main/java/cz/incad/kramerius/auth/basic/AuthFilter.java b/authfilters/src/main/java/cz/incad/kramerius/auth/basic/AuthFilter.java index a1ba399705..994b3edfb5 100644 --- a/authfilters/src/main/java/cz/incad/kramerius/auth/basic/AuthFilter.java +++ b/authfilters/src/main/java/cz/incad/kramerius/auth/basic/AuthFilter.java @@ -94,7 +94,7 @@ public void doFilter(ServletRequest arg0, ServletResponse arg1, FilterChain filt } } - public void basicAuth(FilterChain arg2, HttpServletRequest request, HttpServletResponse response) throws UnsupportedEncodingException, NoSuchAlgorithmException, IOException, ServletException { + public void basicAuth(FilterChain arg2, HttpServletRequest request, HttpServletResponse response) throws NoSuchAlgorithmException, IOException, ServletException { String header = request.getHeader("Authorization"); if (header!=null && header.trim().startsWith("Basic")) { String uname = header.trim().substring("Basic".length()).trim(); diff --git a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/AuthenticatedUsers.java b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/AuthenticatedUsers.java index 25448f0be0..887c0b1de0 100644 --- a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/AuthenticatedUsers.java +++ b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/AuthenticatedUsers.java @@ -4,14 +4,14 @@ public interface AuthenticatedUsers { - public String calculateUserName(HttpServletRequest request); + String calculateUserName(HttpServletRequest request); - public HttpServletRequest updateRequest(HttpServletRequest req); + HttpServletRequest updateRequest(HttpServletRequest req); - public String storeUserPropertiesToSession(HttpServletRequest req, String userName) throws Exception; + String storeUserPropertiesToSession(HttpServletRequest req, String userName) throws Exception; - public void disconnectUser(String userName); + void disconnectUser(String userName); - public String getUserPassword(String userName); + String getUserPassword(String userName); } diff --git a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/AttributeValue.java b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/AttributeValue.java index 84a9f9adb2..34c76f72d4 100644 --- a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/AttributeValue.java +++ b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/AttributeValue.java @@ -43,8 +43,6 @@ public boolean match(Value val, HttpServletRequest request) { String foreignVal = val.getValue(request); if (thisVal != null && foreignVal != null) { return thisVal.equals(foreignVal); - } else if (thisVal == null && foreignVal == null) { - return true; - } else return false; + } else return thisVal == null && foreignVal == null; } } diff --git a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/HeaderValue.java b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/HeaderValue.java index 8b48edbf01..58d24afa2f 100644 --- a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/HeaderValue.java +++ b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/HeaderValue.java @@ -47,9 +47,7 @@ public boolean match(Value val, HttpServletRequest request) { String foreignVal = val.getValue(request); if (thisVal != null && foreignVal != null) { return thisVal.equals(foreignVal); - } else if (thisVal == null && foreignVal == null) { - return true; - } else return false; + } else return thisVal == null && foreignVal == null; } } diff --git a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/ShibbolethContext.java b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/ShibbolethContext.java index 06167188ab..41f8d212ab 100644 --- a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/ShibbolethContext.java +++ b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/ShibbolethContext.java @@ -7,16 +7,16 @@ public interface ShibbolethContext { - public void associateFirstName(String firstName); + void associateFirstName(String firstName); - public void associateLastName(String lastName); + void associateLastName(String lastName); - public void associateSessionAttribute(String key, String value); + void associateSessionAttribute(String key, String value); - public void associateRole(String rname); + void associateRole(String rname); - public boolean isRoleAssociated(String rname); + boolean isRoleAssociated(String rname); - public HttpServletRequest getHttpServletRequest(); + HttpServletRequest getHttpServletRequest(); } diff --git a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/Value.java b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/Value.java index 8bc9310b30..bf920a09bf 100644 --- a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/Value.java +++ b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/shibb/rules/objects/Value.java @@ -29,7 +29,7 @@ public interface Value { * @param request HTTP Request * @return */ - public String getValue(HttpServletRequest request); + String getValue(HttpServletRequest request); /** * Returns true if given value is equals to this one @@ -37,5 +37,5 @@ public interface Value { * @param request HTTP request * @return */ - public boolean match(Value val, HttpServletRequest request); + boolean match(Value val, HttpServletRequest request); } diff --git a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/social/OpenIDAuthenticatedUsers.java b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/social/OpenIDAuthenticatedUsers.java index a67809d74c..31aba19d8e 100644 --- a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/social/OpenIDAuthenticatedUsers.java +++ b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/social/OpenIDAuthenticatedUsers.java @@ -42,7 +42,7 @@ public String calculateUserName(HttpServletRequest request) { } @Override - public String updateExistingUser(String userName,OpenIdUserWrapper wrapper) throws Exception { + public String updateExistingUser(String userName,OpenIdUserWrapper wrapper) { String password = GeneratePasswordUtils.generatePswd(); JSONArray users = RemoteUsersUtils.getUser(userName); JSONObject jsonObject = users.getJSONObject(0); @@ -51,14 +51,14 @@ public String updateExistingUser(String userName,OpenIdUserWrapper wrapper) thro return password; } - public boolean checkIfUserExists(String userName) throws Exception { + public boolean checkIfUserExists(String userName) { JSONArray users = RemoteUsersUtils.getUser(userName); return users.length() > 0 ; } @Override - protected String createNewUser(String user, OpenIdUserWrapper w) throws Exception { + protected String createNewUser(String user, OpenIdUserWrapper w) { String password = GeneratePasswordUtils.generatePswd(); JSONObject json = w.toJSON(password); RemoteUsersUtils.createUser( json); @@ -66,7 +66,7 @@ protected String createNewUser(String user, OpenIdUserWrapper w) throws Exceptio } @Override - protected OpenIdUserWrapper createUserWrapper(HttpServletRequest req, String userName) throws Exception { + protected OpenIdUserWrapper createUserWrapper(HttpServletRequest req, String userName) { Profile profile = OpenIDFlag.flagFromRequest(req).profile(req); return new OpenIdUserWrapper(userName,profile); } diff --git a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/social/utils/OpenIDFlag.java b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/social/utils/OpenIDFlag.java index 7e7444ddf4..42eba2f055 100644 --- a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/social/utils/OpenIDFlag.java +++ b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/social/utils/OpenIDFlag.java @@ -63,7 +63,7 @@ public OpenIDFlag next(HttpServletRequest req) throws Exception { }, LOGGED { @Override - public OpenIDFlag next(HttpServletRequest req) throws Exception { + public OpenIDFlag next(HttpServletRequest req) { req.getSession(true).setAttribute(STATE_KEY, STORED.name()); return OpenIDFlag.STORED; @@ -72,20 +72,20 @@ public OpenIDFlag next(HttpServletRequest req) throws Exception { STORED { @Override - public OpenIDFlag next(HttpServletRequest req) throws Exception { + public OpenIDFlag next(HttpServletRequest req) { req.getSession(true).setAttribute(STATE_KEY, STORED.name()); return OpenIDFlag.STORED; } }; - public SocialAuthManager authManager(HttpServletRequest req) throws Exception { + public SocialAuthManager authManager(HttpServletRequest req) { Object attribute = req.getSession(true).getAttribute(MANAGER_KEY); - return (SocialAuthManager) (attribute != null ? attribute : null); + return (SocialAuthManager) (attribute); } - public Profile profile(HttpServletRequest req) throws Exception { + public Profile profile(HttpServletRequest req) { Object attribute = req.getSession(true).getAttribute(PROFILE_KEY); - return (Profile) (attribute != null ? attribute : null); + return (Profile) (attribute); } diff --git a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/utils/RemoteUsersUtils.java b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/utils/RemoteUsersUtils.java index 7e02d0b32f..44b49386b0 100644 --- a/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/utils/RemoteUsersUtils.java +++ b/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/utils/RemoteUsersUtils.java @@ -4,7 +4,6 @@ import javax.ws.rs.core.MediaType; -import org.apache.commons.configuration.ConfigurationException; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -23,7 +22,7 @@ public class RemoteUsersUtils { public static JSONArray getUser(String userName) - throws ConfigurationException, JSONException { + throws JSONException { Client c = Client.create(); String url = KConfiguration.getInstance().getConfiguration() .getString("api.point") @@ -80,7 +79,7 @@ public static void newPasswordUser( String userId, } } - public static String createUser(JSONObject jsonObject) throws Exception { + public static String createUser(JSONObject jsonObject) { String url = KConfiguration.getInstance().getConfiguration() .getString("api.point") + "/admin/users"; Client c = Client.create(); @@ -96,7 +95,7 @@ public static String createUser(JSONObject jsonObject) throws Exception { return t; } - public static String deleteUser(String usr) throws Exception { + public static String deleteUser(String usr) { String url = KConfiguration.getInstance().getConfiguration() .getString("api.point") + "/admin/users/"+usr; Client c = Client.create(); @@ -111,19 +110,5 @@ public static String deleteUser(String usr) throws Exception { return t; } - public static void main(String[] args) throws JSONException, Exception { - String str = "_third_party_googleplus_114387057939312155006"; - JSONArray user = getUser(str); - System.out.println(user); - System.out.println(user.length()); - - - // String str = "{\"lname\":\"_tthird_party_googleplus_114387057939312155006\",\"firstname\":\"Pavel\",\"password\":\".dlPGvYJT4iW\",\"surname\":\"Stastny\"}"; -// -// String createUser = createUser(new JSONObject(str)); -// System.out.println(createUser); - - } - - + } diff --git a/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/RequestSupportForTests.java b/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/RequestSupportForTests.java index fb481ab6b6..915119b555 100644 --- a/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/RequestSupportForTests.java +++ b/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/RequestSupportForTests.java @@ -56,7 +56,7 @@ public static void callExpectation(HttpServletRequest req, Enumeration k EasyMock.expect(req.getHeader(k)).andAnswer(new IAnswer() { @Override - public String answer() throws Throwable { + public String answer() { return table.get(k); } }).anyTimes(); diff --git a/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/rules/ShibbolethRulesFirstAndLastnameTest.java b/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/rules/ShibbolethRulesFirstAndLastnameTest.java index 2a37b0d7f8..b34196da5d 100644 --- a/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/rules/ShibbolethRulesFirstAndLastnameTest.java +++ b/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/rules/ShibbolethRulesFirstAndLastnameTest.java @@ -32,7 +32,7 @@ public void testFirstAndLastName() throws TokenStreamException, RecognitionExcep HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class); EasyMock.expect(req.getHeaderNames()).andAnswer(new IAnswer() { @Override - public Enumeration answer() throws Throwable { + public Enumeration answer() { return RequestSupportForTests.getLoggedShibLowerCaseTable().keys(); } }); diff --git a/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/rules/ShibbolethRulesRegularExpressionTest.java b/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/rules/ShibbolethRulesRegularExpressionTest.java index 317de6596c..89dea24dc6 100644 --- a/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/rules/ShibbolethRulesRegularExpressionTest.java +++ b/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/rules/ShibbolethRulesRegularExpressionTest.java @@ -41,11 +41,11 @@ public void testShibRules2() throws TokenStreamException, RecognitionException { testShibRules(shibRules2); } - private void testShibRules(String rules) throws RecognitionException, TokenStreamException { + private void testShibRules(String rules) throws TokenStreamException, RecognitionException { HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class); EasyMock.expect(req.getHeaderNames()).andAnswer(new IAnswer() { @Override - public Enumeration answer() throws Throwable { + public Enumeration answer() { return RequestSupportForTests.getLoggedShibLowerCaseTable().keys(); } }); diff --git a/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/utils/ShibbolethUtilsTest.java b/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/utils/ShibbolethUtilsTest.java index 06bed429c9..9f53675069 100644 --- a/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/utils/ShibbolethUtilsTest.java +++ b/authfilters/src/test/java/cz/incad/kramerius/auth/shibb/utils/ShibbolethUtilsTest.java @@ -42,7 +42,7 @@ public void testIsUnderShibbolethSession_NotLogged() { EasyMock.expect(req.getHeaderNames()).andAnswer(new IAnswer() { @Override - public Enumeration answer() throws Throwable { + public Enumeration answer() { return RequestSupportForTests.getNotLoggedShibTable().keys(); } }); @@ -62,7 +62,7 @@ public void testIsUnderShibbolethSession_Logged() { EasyMock.expect(req.getHeaderNames()).andAnswer(new IAnswer() { @Override - public Enumeration answer() throws Throwable { + public Enumeration answer() { return RequestSupportForTests.getLoggedShibTable().keys(); } }); @@ -81,7 +81,7 @@ public void testIsUnderShibbolethSession2_Logged() { EasyMock.expect(req.getHeaderNames()).andAnswer(new IAnswer() { @Override - public Enumeration answer() throws Throwable { + public Enumeration answer() { return RequestSupportForTests.getLoggedShibLowerCaseTable().keys(); } }); @@ -99,7 +99,7 @@ public void testShibbolethSessionId() { EasyMock.expect(req.getHeaderNames()).andAnswer(new IAnswer() { @Override - public Enumeration answer() throws Throwable { + public Enumeration answer() { return RequestSupportForTests.getLoggedShibTable().keys(); } }); @@ -118,21 +118,21 @@ public void testValidateSessionId() { HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class); EasyMock.expect(req.getHeaderNames()).andAnswer(new IAnswer() { @Override - public Enumeration answer() throws Throwable { + public Enumeration answer() { return RequestSupportForTests.getLoggedShibTable().keys(); } }).anyTimes(); EasyMock.expect(req.getSession(true)).andAnswer(new IAnswer() { @Override - public HttpSession answer() throws Throwable { + public HttpSession answer() { return session; } }).anyTimes(); EasyMock.expect(session.getAttribute("Shib-Session-ID")).andAnswer(new IAnswer() { @Override - public String answer() throws Throwable { + public String answer() { return "_8b58b975229f61df5d9389b8f2d0d8d8"; } }).anyTimes(); @@ -149,21 +149,21 @@ public void testValidateSessionId2() { HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class); EasyMock.expect(req.getHeaderNames()).andAnswer(new IAnswer() { @Override - public Enumeration answer() throws Throwable { + public Enumeration answer() { return RequestSupportForTests.getLoggedShibTable().keys(); } }).anyTimes(); EasyMock.expect(req.getSession(true)).andAnswer(new IAnswer() { @Override - public HttpSession answer() throws Throwable { + public HttpSession answer() { return session; } }).anyTimes(); EasyMock.expect(session.getAttribute("Shib-Session-ID")).andAnswer(new IAnswer() { @Override - public String answer() throws Throwable { + public String answer() { return "_abe8b975229f61df5d9389b8f2d0d8d8"; } }).anyTimes(); @@ -180,21 +180,21 @@ public void testValidateSessionId3() { HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class); EasyMock.expect(req.getHeaderNames()).andAnswer(new IAnswer() { @Override - public Enumeration answer() throws Throwable { + public Enumeration answer() { return RequestSupportForTests.getLoggedShibLowerCaseTable().keys(); } }).anyTimes(); EasyMock.expect(req.getSession(true)).andAnswer(new IAnswer() { @Override - public HttpSession answer() throws Throwable { + public HttpSession answer() { return session; } }).anyTimes(); EasyMock.expect(session.getAttribute("Shib-Session-ID")).andAnswer(new IAnswer() { @Override - public String answer() throws Throwable { + public String answer() { return "_abe8b975229f61df5d9389b8f2d0d8d8"; } }).anyTimes(); diff --git a/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaDNNTUtils.java b/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaDNNTUtils.java index b7ed0316eb..2feb51e4eb 100644 --- a/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaDNNTUtils.java +++ b/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaDNNTUtils.java @@ -8,6 +8,7 @@ import cz.incad.kramerius.security.impl.criteria.PDFDNNTFlag; import cz.incad.kramerius.security.impl.criteria.ReadDNNTFlag; import cz.incad.kramerius.security.impl.criteria.ReadDNNTFlagIPFiltered; +import cz.incad.kramerius.statistics.impl.DatabaseStatisticsAccessLogImpl; import cz.incad.kramerius.utils.solr.SolrUtils; import org.json.JSONArray; import org.json.JSONObject; @@ -30,92 +31,11 @@ public class CriteriaDNNTUtils { public static ThreadLocal currentThreadReturnObject = new ThreadLocal<>(); - // DNNT logger - public static Logger DNNT_LOGGER = Logger.getLogger("dnnt.access"); - public static Logger LOGGER = Logger.getLogger(CriteriaDNNTUtils.class.getName()); - public static void logDnntAccess(String pid, - String stream, - String rootTitle, - String dcTitle, - String remoteAddr, - String username, - String email, - - Map sessionAttributes, - List dcAuthors, - ObjectPidsPath[] paths, - ObjectModelsPath[] mpaths) throws IOException { - - LocalDateTime date = LocalDateTime.now(); - String timestamp = date.format(DateTimeFormatter.ISO_DATE_TIME); - - JSONObject jObject = new JSONObject(); - - jObject.put("pid",pid); - jObject.put("remoteAddr",remoteAddr); - jObject.put("username",username); - jObject.put("email",email); - - jObject.put("rootTitle",rootTitle); - jObject.put("dcTitle",dcTitle); - - jObject.put("date",timestamp); - - sessionAttributes.keySet().stream().forEach(key->{ jObject.put(key, sessionAttributes.get(key)); }); - - - if (!dcAuthors.isEmpty()) { - JSONArray authorsArray = new JSONArray(); - for (int i=0,ll=dcAuthors.size();i 0) { - String[] pathFromRootToLeaf = paths[0].getPathFromRootToLeaf(); - if (pathFromRootToLeaf.length > 0) { - jObject.put("rootPid",pathFromRootToLeaf[0]); - } - } - - if (mpaths.length > 0) { - String[] mpathFromRootToLeaf = mpaths[0].getPathFromRootToLeaf(); - if (mpathFromRootToLeaf.length > 0) { - jObject.put("rootModel",mpathFromRootToLeaf[0]); - } - } - - DNNT_LOGGER.log(Level.INFO, jObject.toString()); - } - - private static String pathToString(String[] pArray) { - return Arrays.stream(pArray).reduce("/", (identity, v) -> { - if (!identity.equals("/")) { - return identity + "/" + v; - } else { - return identity + v; - } - }); - } - - -// public static void logDnntAccess(RightCriteriumContext ctx) throws IOException { -// logDnntAccess(ctx.getRequestedPid(), + // public static void toJSON(RightCriteriumContext ctx) throws IOException { +// toJSON(ctx.getRequestedPid(), // ctx.getRemoteAddr(), // ctx.getRequestedStream(), // ctx.getUser().getLoginname(), diff --git a/common/src/main/java/cz/incad/kramerius/statistics/impl/DatabaseStatisticsAccessLogImpl.java b/common/src/main/java/cz/incad/kramerius/statistics/impl/DatabaseStatisticsAccessLogImpl.java index 3bad7c1d42..a1b54b5d28 100644 --- a/common/src/main/java/cz/incad/kramerius/statistics/impl/DatabaseStatisticsAccessLogImpl.java +++ b/common/src/main/java/cz/incad/kramerius/statistics/impl/DatabaseStatisticsAccessLogImpl.java @@ -26,12 +26,11 @@ import java.sql.Date; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; @@ -46,6 +45,8 @@ import org.antlr.stringtemplate.StringTemplate; import org.antlr.stringtemplate.StringTemplateGroup; import org.antlr.stringtemplate.language.DefaultTemplateLexer; +import org.json.JSONArray; +import org.json.JSONObject; import org.w3c.dom.Document; import com.google.inject.Inject; @@ -77,6 +78,12 @@ */ public class DatabaseStatisticsAccessLogImpl implements StatisticsAccessLog { + // DNNT logger + //public static Logger KRAMERIUS_LOGGER_FOR_KIBANA = Logger.getLogger("dnnt.access"); + + // access logger for kibana processing + public static Logger KRAMERIUS_LOGGER_FOR_KIBANA = Logger.getLogger("kramerius.access"); + static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(DatabaseStatisticsAccessLogImpl.class.getName()); @@ -112,7 +119,10 @@ public class DatabaseStatisticsAccessLogImpl implements StatisticsAccessLog { @Inject Set reports; - + + + + @Override public void reportAccess(final String pid, final String streamName) throws IOException { ObjectPidsPath[] paths = this.solrAccess.getPath(pid); @@ -124,43 +134,9 @@ public void reportAccess(final String pid, final String streamName) throws IOExc if (connection == null) throw new NotReadyException("connection not ready"); - - Document solrDoc = this.solrAccess.getSolrDataDocument(pid); - - String rootTitle = titleElement("str", "root_title", solrDoc); - String rootPid = titleElement("str", "root_pid", solrDoc); - String dctitle = titleElement("str", "dc.title", solrDoc); - //String dctitle = titleElement("str", "dc.title", solrDoc); - List authors = new ArrayList<>(); - if (rootPid != null) { - Document rootSolrDoc = this.solrAccess.getSolrDataDocument(rootPid); - Element array = XMLUtils.findElement(rootSolrDoc.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - String nodeName = element.getNodeName(); - String attr = element.getAttribute("name"); - if (nodeName.equals("arr") && StringUtils.isAnyString(attr) && attr.equals("dc.creator")) - return true; - return false; - } - }); - if (array != null) { - authors = XMLUtils.getElements(array).stream().map(it -> it.getTextContent()).filter(it -> it != null).map(String::trim).collect(Collectors.toList()); - } - } - - - - // REPORT DNNT - if (reportedAction.get() == null || reportedAction.get().equals(ReportedAction.READ)) { - reportDNNT(pid,rootTitle,dctitle,authors, paths, mpaths, userProvider.get()); - } - - - + Map dcMap = new HashMap<>(); List commands = new ArrayList(); commands.add(new InsertRecord(pid, loggedUsersSingleton, requestProvider, userProvider, this.reportedAction.get())); - for (int i = 0, ll = paths.length; i < ll; i++) { if (paths[i].contains(SpecialObjects.REPOSITORY.getPid())) { @@ -169,11 +145,17 @@ public boolean acceptElement(Element element) { final int pathIndex = i; String[] pathFromLeafToRoot = paths[i].getPathFromLeafToRoot(); + for (int j = 0; j < pathFromLeafToRoot.length; j++) { + final String detailPid = pathFromLeafToRoot[j]; + Document dc = fedoraAccess.getDC(detailPid); + dcMap.put(detailPid, dc); + } + for (int j = 0; j < pathFromLeafToRoot.length; j++) { final String detailPid = pathFromLeafToRoot[j]; String kModel = fedoraAccess.getKrameriusModelName(detailPid); - Document dc = fedoraAccess.getDC(detailPid); + Document dc = dcMap.containsKey(detailPid) ? dcMap.get(detailPid) : fedoraAccess.getDC(detailPid); Object dateFromDC = DCUtils.dateFromDC(dc); dateFromDC = dateFromDC != null ? dateFromDC : new JDBCUpdateTemplate.NullObject(String.class); @@ -197,7 +179,62 @@ public boolean acceptElement(Element element) { } } } - + + + // Prepare data from solr + Document solrDoc = this.solrAccess.getSolrDataDocument(pid); + String rootTitle = selem("str", "root_title", solrDoc); + String rootPid = selem("str", "root_pid", solrDoc); + String dctitle = selem("str", "dc.title", solrDoc); + String publishedDate = selem("str", "datum_str", solrDoc); + String dnnt = selem("bool", "dnnt", solrDoc); + String policy = selem("str", "dostupnost", solrDoc); + + List dcAuthors = new ArrayList<>(); + if (rootPid != null) { + Document rootSolrDoc = this.solrAccess.getSolrDataDocument(rootPid); + Element array = XMLUtils.findElement(rootSolrDoc.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String nodeName = element.getNodeName(); + String attr = element.getAttribute("name"); + if (nodeName.equals("arr") && StringUtils.isAnyString(attr) && attr.equals("dc.creator")) + return true; + return false; + } + }); + if (array != null) { + dcAuthors = XMLUtils.getElements(array).stream().map(it -> it.getTextContent()).filter(it -> it != null).map(String::trim).collect(Collectors.toList()); + } + } + + List dcPublishers = new ArrayList<>(); + for (int i = 0, ll = paths.length; i < ll; i++) { + if (paths[i].contains(SpecialObjects.REPOSITORY.getPid())) { + paths[i] = paths[i].cutHead(0); + } + final int pathIndex = i; + String[] pathFromLeafToRoot = paths[i].getPathFromLeafToRoot(); + for (int j = 0; j < pathFromLeafToRoot.length; j++) { + final String detailPid = pathFromLeafToRoot[j]; + Document document = dcMap.get(detailPid); + + List collected = Arrays.stream(DCUtils.publishersFromDC(document)).map(it -> { + return it.replaceAll("\\r?\\n", " "); + }).collect(Collectors.toList()); + + dcPublishers.addAll(collected); + } + } + + + // WRITE TO LOG - kibana processing + if (reportedAction.get() == null || reportedAction.get().equals(ReportedAction.READ)) { + logKibanaAccess(pid,rootTitle,dctitle,publishedDate, dnnt,policy, dcPublishers ,dcAuthors, paths, mpaths, userProvider.get()); + } + + + // WRITE TO DATABASE JDBCTransactionTemplate transactionTemplate = new JDBCTransactionTemplate(connection, true); transactionTemplate.updateWithTransaction(commands); } catch (SQLException e) { @@ -206,8 +243,8 @@ public boolean acceptElement(Element element) { } - private String titleElement(String type, String attrVal, Document solrDoc) { - Element titleElm = XMLUtils.findElement(solrDoc.getDocumentElement(), new XMLUtils.ElementsFilter() { + private String selem(String type, String attrVal, Document solrDoc) { + Element dcElm = XMLUtils.findElement(solrDoc.getDocumentElement(), new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { String nodeName = element.getNodeName(); @@ -216,25 +253,29 @@ public boolean acceptElement(Element element) { return false; } }); - return titleElm != null ? titleElm.getTextContent() : null; + return dcElm != null ? dcElm.getTextContent() : null; } - private void reportDNNT(String pid, String rootTitle, String dcTitle, List dcAuthors, ObjectPidsPath[] paths, ObjectModelsPath[] mpaths, User user) throws IOException { + private void logKibanaAccess(String pid, String rootTitle, String dcTitle, String publishedDate, String dnntFlag, String policy, List dcPublishers, List dcAuthors, ObjectPidsPath[] paths, ObjectModelsPath[] mpaths, User user) throws IOException { RightsReturnObject rightsReturnObject = CriteriaDNNTUtils.currentThreadReturnObject.get(); - if (rightsReturnObject == null) return; - if (CriteriaDNNTUtils.checkContainsCriteriumReadDNNT(rightsReturnObject)) { - - CriteriaDNNTUtils.logDnntAccess(pid, - null,rootTitle,dcTitle, - IPAddressUtils.getRemoteAddress(requestProvider.get(), KConfiguration.getInstance().getConfiguration()), - user!= null ? user.getLoginname() : null, - user != null ? user.getEmail(): null, - user.getSessionAttributes(), - dcAuthors, - paths, - mpaths - ); - } + boolean providedByDnnt = rightsReturnObject != null ? CriteriaDNNTUtils.checkContainsCriteriumReadDNNT(rightsReturnObject) : false; + + JSONObject jObject = toJSON(pid, rootTitle, dcTitle, + IPAddressUtils.getRemoteAddress(requestProvider.get(), KConfiguration.getInstance().getConfiguration()), + user != null ? user.getLoginname() : null, + user != null ? user.getEmail() : null, + publishedDate, + dnntFlag, + providedByDnnt, + policy, + user.getSessionAttributes(), + dcAuthors, + dcPublishers, + paths, + mpaths + ); + + KRAMERIUS_LOGGER_FOR_KIBANA.log(Level.INFO, jObject.toString()); } @@ -453,4 +494,103 @@ public Object executeJDBCCommand(Connection con) throws SQLException { InputStream is = DatabaseStatisticsAccessLogImpl.class.getResourceAsStream("res/statistics.stg"); stGroup = new StringTemplateGroup(new InputStreamReader(is), DefaultTemplateLexer.class); } + + + + + + public static JSONObject toJSON(String pid, + String rootTitle, + String dcTitle, + String remoteAddr, + String username, + String email, + String publishedDate, + String dnntFlag, + boolean providedByDnnt, + String policy, + + Map sessionAttributes, + List dcAuthors, + List dcPublishers, + ObjectPidsPath[] paths, + ObjectModelsPath[] mpaths) throws IOException { + + LocalDateTime date = LocalDateTime.now(); + String timestamp = date.format(DateTimeFormatter.ISO_DATE_TIME); + + JSONObject jObject = new JSONObject(); + + jObject.put("pid",pid); + jObject.put("remoteAddr",remoteAddr); + jObject.put("username",username); + jObject.put("email",email); + + jObject.put("rootTitle",rootTitle); + jObject.put("dcTitle",dcTitle); + + if (dnntFlag != null ) jObject.put("dnnt", dnntFlag.trim().toLowerCase().equals("true")); + jObject.put("providedByDnnt", providedByDnnt); + jObject.put("policy", policy); + + jObject.put("publishedDate", publishedDate); + + jObject.put("date",timestamp); + + sessionAttributes.keySet().stream().forEach(key->{ jObject.put(key, sessionAttributes.get(key)); }); + + + if (!dcAuthors.isEmpty()) { + JSONArray authorsArray = new JSONArray(); + for (int i=0,ll=dcAuthors.size();i 0) { + String[] pathFromRootToLeaf = paths[0].getPathFromRootToLeaf(); + if (pathFromRootToLeaf.length > 0) { + jObject.put("rootPid",pathFromRootToLeaf[0]); + } + } + + if (mpaths.length > 0) { + String[] mpathFromRootToLeaf = mpaths[0].getPathFromRootToLeaf(); + if (mpathFromRootToLeaf.length > 0) { + jObject.put("rootModel",mpathFromRootToLeaf[0]); + } + } + return jObject; + } + + private static String pathToString(String[] pArray) { + return Arrays.stream(pArray).reduce("/", (identity, v) -> { + if (!identity.equals("/")) { + return identity + "/" + v; + } else { + return identity + v; + } + }); + } + } diff --git a/mw-process/src/main/java/cz/cas/lib/knav/ApplyMWUtils.java b/mw-process/src/main/java/cz/cas/lib/knav/ApplyMWUtils.java index 16e3290453..3d803f760b 100644 --- a/mw-process/src/main/java/cz/cas/lib/knav/ApplyMWUtils.java +++ b/mw-process/src/main/java/cz/cas/lib/knav/ApplyMWUtils.java @@ -95,11 +95,8 @@ public boolean acceptElement(Element element) { "str"); boolean hasGoodAttr = nameAttr != null && nameAttr.equals("dc.title"); - - if (isElmStr && hasGoodAttr) { - return true; - } else - return false; + + return isElmStr && hasGoodAttr; } }); @@ -234,8 +231,7 @@ public static String disectFlagFromRELSEXT(String onePid, FedoraAccess fa) * @throws IOException */ public static void setPolicyFlag(String pid, boolean b, FedoraAccess fa, - String previousState, CollectPidForIndexing coll) - throws IOException { + String previousState, CollectPidForIndexing coll) { if (ApplyMWUtils.detectChange(b, previousState)) { PolicyServiceImpl policy = new PolicyServiceImpl(); policy.setFedoraAccess(fa); @@ -292,8 +288,7 @@ public static int configuredWall(SolrAccess sa, String onePid, Configuration con return wall; } - public static int defaultConfiguredWall( Configuration conf) - throws IOException { + public static int defaultConfiguredWall( Configuration conf) { int wall = conf.getInt("mwprocess.wall", 70); return wall; diff --git a/mw-process/src/main/java/cz/cas/lib/knav/ParametrizedIterateRepoMovingWall.java b/mw-process/src/main/java/cz/cas/lib/knav/ParametrizedIterateRepoMovingWall.java index 8fc38ef1ee..57af669430 100644 --- a/mw-process/src/main/java/cz/cas/lib/knav/ParametrizedIterateRepoMovingWall.java +++ b/mw-process/src/main/java/cz/cas/lib/knav/ParametrizedIterateRepoMovingWall.java @@ -16,7 +16,7 @@ public class ParametrizedIterateRepoMovingWall { @Process - public static void process(@ParameterName("userValue") String uVal) throws XPathExpressionException, IOException, RightCriteriumException { + public static void process(@ParameterName("userValue") String uVal) { List topLevelModels = KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"); RepositoryItemsSupport repoItems = new SolrRepoItemsSupport("PID"); for (String m : topLevelModels) { diff --git a/solr-migration/src/test/java/cz/incad/kramerius/services/NKPIterations.java b/solr-migration/src/test/java/cz/incad/kramerius/services/NKPIterations.java deleted file mode 100644 index e5c9f2ad78..0000000000 --- a/solr-migration/src/test/java/cz/incad/kramerius/services/NKPIterations.java +++ /dev/null @@ -1,35 +0,0 @@ -package cz.incad.kramerius.services; - -import com.sun.jersey.api.client.Client; -import cz.incad.kramerius.service.MigrateSolrIndexException; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; - -import javax.xml.parsers.ParserConfigurationException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BrokenBarrierException; - -public class NKPIterations { - - public static void main(String[] args) throws InterruptedException, ParserConfigurationException, IOException, BrokenBarrierException, SAXException, MigrateSolrIndexException { - final List list = new ArrayList<>(); - long startTime = System.currentTimeMillis(); - Client client = Client.create(); - IterationUtils.queryPaginationIteration(client, "http://10.10.0.94:8080/solr/","*:*", (Element element, String t) ->{ - List allPids = MigrationUtils.findAllPids(element); - list.add(allPids.size()); - int sum = sum(list); - System.out.println(sum(list)); - },() ->{ - System.out.println("Konec"); - long stopTime = System.currentTimeMillis(); - System.out.println("It took "+(stopTime - startTime)); - }); - } - - public static final int sum(List sum) { - return sum.stream().mapToInt(Integer::intValue).sum(); - } -}