From a9f9104d2863320216af5db9c3403da02ebcc0df Mon Sep 17 00:00:00 2001 From: cloudAndMonkey Date: Sun, 11 Dec 2022 09:51:48 +0800 Subject: [PATCH 1/3] Add files via upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、 支持多数据源 2、统一关闭connection连接 3、支持elasticSearch-sql 4、子查询、join 主表和附表使用相同数据源 5、相关小bug修复 --- APIJSONORM/src/main/java/apijson/JSONObject.java | 1 + APIJSONORM/src/main/java/apijson/StringUtil.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/APIJSONORM/src/main/java/apijson/JSONObject.java b/APIJSONORM/src/main/java/apijson/JSONObject.java index db21693cf..a73203596 100755 --- a/APIJSONORM/src/main/java/apijson/JSONObject.java +++ b/APIJSONORM/src/main/java/apijson/JSONObject.java @@ -173,6 +173,7 @@ public JSONObject setUserIdIn(List list) { TABLE_KEY_LIST.add(KEY_ORDER); TABLE_KEY_LIST.add(KEY_RAW); TABLE_KEY_LIST.add(KEY_JSON); + TABLE_KEY_LIST.add(KEY_METHOD); } //@key关键字都放这个类 >>>>>>>>>>>>>>>>>>>>>> diff --git a/APIJSONORM/src/main/java/apijson/StringUtil.java b/APIJSONORM/src/main/java/apijson/StringUtil.java index 67ca446f6..5e22b184c 100755 --- a/APIJSONORM/src/main/java/apijson/StringUtil.java +++ b/APIJSONORM/src/main/java/apijson/StringUtil.java @@ -354,7 +354,7 @@ public static boolean isNotEmpty(String s, boolean trim) { PATTERN_ALPHA = Pattern.compile("^[a-zA-Z]+$"); PATTERN_ALPHA_BIG = Pattern.compile("^[A-Z]+$"); PATTERN_ALPHA_SMALL = Pattern.compile("^[a-z]+$"); - PATTERN_NAME = Pattern.compile("^[0-9a-zA-Z_:]+$");//已用55个中英字符测试通过 + PATTERN_NAME = Pattern.compile("^[0-9a-zA-Z_.:]+$");//已用55个中英字符测试通过 //newest phone regex expression reference https://github.com/VincentSit/ChinaMobilePhoneNumberRegex PATTERN_PHONE = Pattern.compile("^1(?:3\\d{3}|5[^4\\D]\\d{2}|8\\d{3}|7(?:[0-35-9]\\d{2}|4(?:0\\d|1[0-2]|9\\d))|9[0-35-9]\\d{2}|6[2567]\\d{2}|4(?:(?:10|4[01])\\d{3}|[68]\\d{4}|[579]\\d{2}))\\d{6}$"); PATTERN_EMAIL = Pattern.compile("^([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)$"); From d9c74739c6c2f0f60da75c5ba37458fd867b1935 Mon Sep 17 00:00:00 2001 From: cloudAndMonkey Date: Sun, 11 Dec 2022 09:55:01 +0800 Subject: [PATCH 2/3] Add files via upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、 支持多数据源 2、统一关闭connection连接 3、支持elasticSearch-sql 4、子查询、join 主表和附表使用相同数据源 5、相关小bug修复 --- .../main/java/apijson/orm/AbstractParser.java | 128 ++++++++++++++---- .../java/apijson/orm/AbstractSQLConfig.java | 17 ++- .../java/apijson/orm/AbstractSQLExecutor.java | 69 ++++++++-- 3 files changed, 180 insertions(+), 34 deletions(-) diff --git a/APIJSONORM/src/main/java/apijson/orm/AbstractParser.java b/APIJSONORM/src/main/java/apijson/orm/AbstractParser.java index b217c8d69..dd932b8b7 100755 --- a/APIJSONORM/src/main/java/apijson/orm/AbstractParser.java +++ b/APIJSONORM/src/main/java/apijson/orm/AbstractParser.java @@ -48,7 +48,11 @@ */ public abstract class AbstractParser implements Parser, ParserCreator, VerifierCreator, SQLCreator { protected static final String TAG = "AbstractParser"; - protected Map keyMethodMap = new HashMap<>(); + + /** + * json对象、数组对应的数据源、版本、角色、method等 + */ + protected Map> keyObjectAttributesMap = new HashMap<>(); /** * 可以通过切换该变量来控制是否打印关键的接口请求内容。保守起见，该值默认为false。 * 与 {@link Log#DEBUG} 任何一个为 true 都会打印关键的接口请求内容。 @@ -1158,7 +1162,8 @@ public JSONArray onArrayParse(JSONObject request, String parentPath, String name } //不能允许GETS，否则会被通过"[]":{"@role":"ADMIN"},"Table":{},"tag":"Table"绕过权限并能批量查询 - if (isSubquery == false && RequestMethod.isGetMethod(requestMethod, true) == false) { + RequestMethod _method = request.get(apijson.JSONObject.KEY_METHOD) == null ? requestMethod : RequestMethod.valueOf(request.getString(apijson.JSONObject.KEY_METHOD)); + if (isSubquery == false && RequestMethod.isGetMethod(_method, true) == false) { throw new UnsupportedOperationException("key[]:{} 只支持 GET, GETS 方法！其它方法不允许传 " + name + ":{} 等这种 key[]:{} 格式！"); } if (request == null || request.isEmpty()) { // jsonKey-jsonValue 条件 @@ -1913,7 +1918,7 @@ public JSONObject executeSQL(SQLConfig config, boolean isSubquery) throws Except JSONObject res = getSQLExecutor().execute(config, false); //如果是查询方法，才能执行explain - if (RequestMethod.isQueryMethod(config.getMethod())){ + if (RequestMethod.isQueryMethod(config.getMethod()) && config.isElasticsearch() == false){ config.setExplain(explain); JSONObject explainResult = config.isMain() && config.getPosition() != 0 ? null : getSQLExecutor().execute(config, false); @@ -2083,6 +2088,7 @@ protected JSONObject getRequestStructure(RequestMethod method, String tag, int v private JSONObject batchVerify(RequestMethod method, String tag, int version, String name, @NotNull JSONObject request, int maxUpdateCount, SQLCreator creator) throws Exception { JSONObject jsonObject = new JSONObject(true); + List removeTmpKeys = new ArrayList<>(); // 请求json里面的临时变量,不需要带入后面的业务中,比如 @post、@get等 if (request.keySet() == null || request.keySet().size() == 0) { throw new IllegalArgumentException("json对象格式不正确 ！,例如 \"User\": {}"); } @@ -2098,59 +2104,117 @@ private JSONObject batchVerify(RequestMethod method, String tag, int version, St if (key.startsWith("@")) { try { // 如果不匹配,异常不处理即可 - RequestMethod l_method = RequestMethod.valueOf(key.substring(1).toUpperCase()); - for(String objKey : StringUtil.split(request.getString(key))) { - keyMethodMap.put(objKey, l_method); + RequestMethod _method = RequestMethod.valueOf(key.substring(1).toUpperCase()); + removeTmpKeys.add(key); + for (String objKey : request.getJSONObject(key).keySet()) { + Map object_attributes_map = new HashMap<>(); + object_attributes_map.put(apijson.JSONObject.KEY_METHOD, _method); + keyObjectAttributesMap.put(objKey, object_attributes_map); + JSONObject objAttrJson = request.getJSONObject(key).getJSONObject(objKey); + for (String objAttr : objAttrJson.keySet()) { + switch (objAttr) { + case apijson.JSONObject.KEY_DATASOURCE: + object_attributes_map.put(apijson.JSONObject.KEY_DATASOURCE, objAttrJson.getString(objAttr)); + break; + case apijson.JSONObject.KEY_SCHEMA: + object_attributes_map.put(apijson.JSONObject.KEY_SCHEMA, objAttrJson.getString(objAttr)); + break; + case apijson.JSONObject.KEY_DATABASE: + object_attributes_map.put(apijson.JSONObject.KEY_DATABASE, objAttrJson.getString(objAttr)); + break; + case apijson.JSONObject.VERSION: + object_attributes_map.put(apijson.JSONObject.VERSION, objAttrJson.getString(objAttr)); + break; + case apijson.JSONObject.KEY_ROLE: + object_attributes_map.put(apijson.JSONObject.KEY_ROLE, objAttrJson.getString(objAttr)); + break; + default: + break; + } + } } + continue; } catch (Exception e) { } } - - // + // 1、非crud,对于没有显式声明操作方法的，直接用 URL(/get, /post 等) 对应的默认操作方法 // 2、crud, 没有声明就用 GET // 3、兼容 sql@ JSONObject,设置 GET方法 // 将method 设置到每个object, op执行会解析 if (request.get(key) instanceof JSONObject) { - if (keyMethodMap.get(key) == null) { + if (keyObjectAttributesMap.get(key) == null) { // 数组会解析为对象进行校验,做一下兼容 - if (keyMethodMap.get(key + apijson.JSONObject.KEY_ARRAY) == null) { - if (method == RequestMethod.CRUD || (key.endsWith("@") && request.get(key) instanceof JSONObject)) { + if (keyObjectAttributesMap.get(key + apijson.JSONObject.KEY_ARRAY) == null) { + if (method == RequestMethod.CRUD || key.endsWith("@")) { request.getJSONObject(key).put(apijson.JSONObject.KEY_METHOD, GET); - keyMethodMap.put(key, GET); + if(keyObjectAttributesMap.get(key) == null) { + Map object_attributes_map = new HashMap<>(); + object_attributes_map.put(apijson.JSONObject.KEY_METHOD, GET); + keyObjectAttributesMap.put(key, object_attributes_map); + }else { + keyObjectAttributesMap.get(key).put(apijson.JSONObject.KEY_METHOD, GET); + } } else { request.getJSONObject(key).put(apijson.JSONObject.KEY_METHOD, method); - keyMethodMap.put(key, method); + if(keyObjectAttributesMap.get(key) == null) { + Map object_attributes_map = new HashMap<>(); + object_attributes_map.put(apijson.JSONObject.KEY_METHOD, method); + keyObjectAttributesMap.put(key, object_attributes_map); + }else { + keyObjectAttributesMap.get(key).put(apijson.JSONObject.KEY_METHOD, method); + } } } else { - request.getJSONObject(key).put(apijson.JSONObject.KEY_METHOD, keyMethodMap.get(key + apijson.JSONObject.KEY_ARRAY)); + setRequestAttribute(key, true, apijson.JSONObject.KEY_METHOD, request); + setRequestAttribute(key, true, apijson.JSONObject.KEY_DATASOURCE, request); + setRequestAttribute(key, true, apijson.JSONObject.KEY_SCHEMA, request); + setRequestAttribute(key, true, apijson.JSONObject.KEY_DATABASE, request); + setRequestAttribute(key, true, apijson.JSONObject.VERSION, request); + setRequestAttribute(key, true, apijson.JSONObject.KEY_ROLE, request); } } else { - request.getJSONObject(key).put(apijson.JSONObject.KEY_METHOD, keyMethodMap.get(key)); + setRequestAttribute(key, false, apijson.JSONObject.KEY_METHOD, request); + setRequestAttribute(key, false, apijson.JSONObject.KEY_DATASOURCE, request); + setRequestAttribute(key, false, apijson.JSONObject.KEY_SCHEMA, request); + setRequestAttribute(key, false, apijson.JSONObject.KEY_DATABASE, request); + setRequestAttribute(key, false, apijson.JSONObject.VERSION, request); + setRequestAttribute(key, false, apijson.JSONObject.KEY_ROLE, request); } } - + if (key.startsWith("@") || key.endsWith("@")) { jsonObject.put(key, request.get(key)); continue; } - if (request.get(key) instanceof JSONObject || request.get(key) instanceof JSONArray) { RequestMethod _method = null; if (request.get(key) instanceof JSONObject) { _method = RequestMethod.valueOf(request.getJSONObject(key).getString(apijson.JSONObject.KEY_METHOD).toUpperCase()); } else { - if (keyMethodMap.get(key) == null) { + if (keyObjectAttributesMap.get(key) == null) { if (method == RequestMethod.CRUD) { _method = GET; - keyMethodMap.put(key, GET); + if(keyObjectAttributesMap.get(key) == null) { + Map object_attributes_map = new HashMap<>(); + object_attributes_map.put(apijson.JSONObject.KEY_METHOD, GET); + keyObjectAttributesMap.put(key, object_attributes_map); + }else { + keyObjectAttributesMap.get(key).put(apijson.JSONObject.KEY_METHOD, GET); + } } else { _method = method; - keyMethodMap.put(key, method); + if(keyObjectAttributesMap.get(key) == null) { + Map object_attributes_map = new HashMap<>(); + object_attributes_map.put(apijson.JSONObject.KEY_METHOD, method); + keyObjectAttributesMap.put(key, object_attributes_map); + }else { + keyObjectAttributesMap.get(key).put(apijson.JSONObject.KEY_METHOD, method); + } } } else { - _method = keyMethodMap.get(key); + _method = (RequestMethod) keyObjectAttributesMap.get(key).get(apijson.JSONObject.KEY_METHOD); } } @@ -2179,10 +2243,26 @@ private JSONObject batchVerify(RequestMethod method, String tag, int version, St throw new Exception(e); } } - + // 这里是requestObject ref request 的引用, 删除不需要的临时变量 + for(String removeKey : removeTmpKeys) { + request.remove(removeKey); + } return jsonObject; } + private void setRequestAttribute(String key, boolean isArray, String attrKey, @NotNull JSONObject request) { + Object attrVal = null; + if(isArray) { + attrVal = keyObjectAttributesMap.get(key + apijson.JSONObject.KEY_ARRAY).get(attrKey); + }else { + attrVal = keyObjectAttributesMap.get(key).get(attrKey); + } + + if(attrVal != null && request.getJSONObject(key).get(attrKey) == null) { + // 如果对象内部已经包含该属性,不覆盖 + request.getJSONObject(key).put(attrKey, attrVal); + } + } /** * { "xxx:aa":{ "@tag": "" }} * 生成规则: @@ -2231,8 +2311,8 @@ protected JSONObject objectVerify(RequestMethod method, String tag, int version, * @return */ public RequestMethod getRealMethod(RequestMethod method, String key, Object value) { - if(method == CRUD && (value instanceof JSONObject || value instanceof JSONArray)) { - return this.keyMethodMap.get(key); + if(method == CRUD && key.startsWith("@") == false && (value instanceof JSONObject || value instanceof JSONArray)) { + return (RequestMethod)this.keyObjectAttributesMap.get(key).get(apijson.JSONObject.KEY_METHOD); } return method; } diff --git a/APIJSONORM/src/main/java/apijson/orm/AbstractSQLConfig.java b/APIJSONORM/src/main/java/apijson/orm/AbstractSQLConfig.java index 12ecbd63c..a8e6885f0 100755 --- a/APIJSONORM/src/main/java/apijson/orm/AbstractSQLConfig.java +++ b/APIJSONORM/src/main/java/apijson/orm/AbstractSQLConfig.java @@ -73,6 +73,7 @@ import static apijson.RequestMethod.HEADS; import static apijson.RequestMethod.POST; import static apijson.RequestMethod.PUT; +import static apijson.JSONObject.KEY_METHOD; import static apijson.SQL.AND; import static apijson.SQL.NOT; import static apijson.SQL.ON; @@ -1119,6 +1120,9 @@ public static boolean isTDengine(String db) { @Override public String getQuote() { + if(isElasticsearch()) { + return ""; + } return isMySQL() || isMariaDB() || isTiDB() || isClickHouse() || isTDengine() ? "`" : "\""; } @@ -3967,6 +3971,10 @@ public String getSubqueryString(Subquery subquery) throws Exception { String range = subquery.getRange(); SQLConfig cfg = subquery.getConfig(); + // 子查询 = 主语句 datasource + if(StringUtil.equals(this.getTable(), subquery.getFrom() ) == false && cfg.hasJoin() == false) { + cfg.setDatasource(this.getDatasource()); + } cfg.setPreparedValueList(new ArrayList<>()); String withAsExpreSql = withAsExpreSubqueryString(cfg, subquery); String sql = (range == null || range.isEmpty() ? "" : range) + "(" + withAsExpreSql + ") "; @@ -4213,6 +4221,9 @@ public static String getSQL(AbstractSQLConfig config) throws Exception { cSql = "SELECT " + (config.getCache() == JSONRequest.CACHE_RAM ? "SQL_NO_CACHE " : "") + column + " FROM " + getConditionString(tablePath, config) + config.getLimitString(); cSql = buildWithAsExpreSql(config, cSql); + if(config.isElasticsearch()) { // elasticSearch 不支持 explain + return cSql; + } return explain + cSql; } } @@ -4354,7 +4365,8 @@ public String getJoinString() throws Exception { // <"INNER JOIN User ON User.id = Moment.userId", UserConfig> TODO AS 放 getSQLTable 内 SQLConfig jc = j.getJoinConfig(); jc.setPrepared(isPrepared()); - + // 将关联表所属数据源配置为主表数据源 + jc.setDatasource(this.getDatasource()); String jt = StringUtil.isEmpty(jc.getAlias(), true) ? jc.getTable() : jc.getAlias(); List onList = j.getOnList(); @@ -4648,7 +4660,7 @@ public static SQLConfig newSQLConfig(RequestMethod method, St boolean explain = request.getBooleanValue(KEY_EXPLAIN); if (explain && Log.DEBUG == false) { //不在 config.setExplain 抛异常，一方面处理更早性能更好，另一方面为了内部调用可以绕过这个限制 - throw new UnsupportedOperationException("DEBUG 模式下不允许传 " + KEY_EXPLAIN + " ！"); + throw new UnsupportedOperationException("INFO 模式下不允许传 " + KEY_EXPLAIN + " ！"); } String database = request.getString(KEY_DATABASE); @@ -4835,6 +4847,7 @@ else if (userId instanceof Subquery) {} request.remove(KEY_ORDER); request.remove(KEY_RAW); request.remove(KEY_JSON); + request.remove(KEY_METHOD); // @null <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< diff --git a/APIJSONORM/src/main/java/apijson/orm/AbstractSQLExecutor.java b/APIJSONORM/src/main/java/apijson/orm/AbstractSQLExecutor.java index 9bbd76d71..bfe7fed26 100755 --- a/APIJSONORM/src/main/java/apijson/orm/AbstractSQLExecutor.java +++ b/APIJSONORM/src/main/java/apijson/orm/AbstractSQLExecutor.java @@ -6,7 +6,6 @@ package apijson.orm; import java.io.BufferedReader; -import java.rmi.ServerError; import java.sql.Blob; import java.sql.Clob; import java.sql.Connection; @@ -18,7 +17,6 @@ import java.sql.SQLException; import java.sql.Savepoint; import java.sql.Statement; -import java.sql.Time; import java.sql.Timestamp; import java.time.DayOfWeek; import java.time.LocalDateTime; @@ -27,7 +25,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -294,7 +291,12 @@ public JSONObject execute(@NotNull SQLConfig config, boolean unknownType) throws } result = AbstractParser.newSuccessResult(); - result.put(JSONResponse.KEY_COUNT, rs.getLong(1)); + // 兼容nosql,比如 elasticSearch-sql + if(config.isElasticsearch()) { + result.put(JSONResponse.KEY_COUNT, rs.getObject(1)); + }else { + result.put(JSONResponse.KEY_COUNT, rs.getLong(1)); + } resultList = new ArrayList<>(1); resultList.add(result); } @@ -1047,7 +1049,10 @@ public boolean isJSONType(@NotNull SQLConfig config, ResultSetMetaData rsmd, int long startTime = System.currentTimeMillis(); String column = rsmd.getColumnTypeName(position); sqlResultDuration += System.currentTimeMillis() - startTime; - + // nosql elasticSearch jdbc获取不到 字段类型 + if(StringUtil.isEmpty(column)) { + return false; + } //TODO CHAR和JSON类型的字段，getColumnType返回值都是1 ，如果不用CHAR，改用VARCHAR，则可以用上面这行来提高性能。 //return rsmd.getColumnType(position) == 1; @@ -1187,7 +1192,21 @@ public void rollback() throws SQLException { if (connection == null) { // || connection.isClosed()) { return; } - connection.rollback(); + // 将所有连接进行回滚 + Collection connections = connectionMap.values(); + + if (connections != null) { + for (Connection connection : connections) { + try { + if (connection != null && connection.isClosed() == false) { + connection.rollback(); + } + } + catch (SQLException e) { + e.printStackTrace(); + } + } + } } @Override public void rollback(Savepoint savepoint) throws SQLException { @@ -1196,7 +1215,26 @@ public void rollback(Savepoint savepoint) throws SQLException { if (connection == null) { // || connection.isClosed()) { return; } - connection.rollback(savepoint); + + if(StringUtil.isEmpty(savepoint)) { + // 将所有连接进行回滚 + Collection connections = connectionMap.values(); + + if (connections != null) { + for (Connection connection : connections) { + try { + if (connection != null && connection.isClosed() == false) { + connection.rollback(); + } + } + catch (SQLException e) { + e.printStackTrace(); + } + } + } + } else { + connection.rollback(savepoint); + } } @Override public void commit() throws SQLException { @@ -1205,7 +1243,22 @@ public void commit() throws SQLException { if (connection == null) { // || connection.isClosed()) { return; } - connection.commit(); + + // 将所有连接进行提交 + Collection connections = connectionMap.values(); + + if (connections != null) { + for (Connection connection : connections) { + try { + if (connection != null && connection.isClosed() == false) { + connection.commit(); + } + } + catch (SQLException e) { + e.printStackTrace(); + } + } + } } //事务处理 >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> From 4af63224663c2ae9e2979028437349dfc1060739 Mon Sep 17 00:00:00 2001 From: cloudAndMonkey Date: Sun, 11 Dec 2022 10:00:38 +0800 Subject: [PATCH 3/3] Add files via upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、 支持多数据源 2、统一关闭connection连接 3、支持elasticSearch-sql 4、解决子查询、join 主表和附表使用相同数据源 5、相关小bug修复 --- APIJSONORM/src/main/java/apijson/orm/AbstractSQLConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/APIJSONORM/src/main/java/apijson/orm/AbstractSQLConfig.java b/APIJSONORM/src/main/java/apijson/orm/AbstractSQLConfig.java index a8e6885f0..707147c18 100755 --- a/APIJSONORM/src/main/java/apijson/orm/AbstractSQLConfig.java +++ b/APIJSONORM/src/main/java/apijson/orm/AbstractSQLConfig.java @@ -4660,7 +4660,7 @@ public static SQLConfig newSQLConfig(RequestMethod method, St boolean explain = request.getBooleanValue(KEY_EXPLAIN); if (explain && Log.DEBUG == false) { //不在 config.setExplain 抛异常，一方面处理更早性能更好，另一方面为了内部调用可以绕过这个限制 - throw new UnsupportedOperationException("INFO 模式下不允许传 " + KEY_EXPLAIN + " ！"); + throw new UnsupportedOperationException("非DEBUG模式, 不允许传 " + KEY_EXPLAIN + " ！"); } String database = request.getString(KEY_DATABASE);