From cc5095ab99554148ab6c5035c91a309d5fde4ccc Mon Sep 17 00:00:00 2001 From: Cyber-SiKu Date: Fri, 13 Oct 2023 14:16:46 +0800 Subject: [PATCH 01/27] [feat]curvefs/client kvcache monitor 1. fix base metric(bps/qps/latnecy) error 2. add hit and miss metric 3. change metric prefix to curvefs_kvclient_manager 4. add memcache client metric Signed-off-by: Cyber-SiKu --- .../provisioning/dashboards/client.json | 2285 +++++++++++++++-- curvefs/src/client/fuse_s3_client.cpp | 4 +- curvefs/src/client/kvclient/kvclient.h | 7 +- .../src/client/kvclient/kvclient_manager.cpp | 72 +- .../src/client/kvclient/kvclient_manager.h | 26 +- curvefs/src/client/kvclient/memcache_client.h | 36 +- curvefs/src/client/metric/client_metric.cpp | 5 +- curvefs/src/client/metric/client_metric.h | 40 +- curvefs/test/client/client_memcache_test.cpp | 7 +- .../client/client_s3_adaptor_Integration.cpp | 8 +- .../test/client/metric/client_metric_test.cpp | 16 +- curvefs/test/client/mock_kvclient.h | 8 +- 12 files changed, 2179 insertions(+), 335 deletions(-) diff --git a/curvefs/monitor/grafana/provisioning/dashboards/client.json b/curvefs/monitor/grafana/provisioning/dashboards/client.json index 870a2b5bad..2bdfcddad8 100644 --- a/curvefs/monitor/grafana/provisioning/dashboards/client.json +++ b/curvefs/monitor/grafana/provisioning/dashboards/client.json @@ -25,6 +25,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, + "id": 7, "links": [], "liveNow": false, "panels": [ @@ -59,7 +60,7 @@ "h": 8, "w": 12, "x": 0, - "y": 1 + "y": 25 }, "hiddenSeries": false, "id": 116, @@ -155,7 +156,7 @@ "h": 8, "w": 12, "x": 12, - "y": 1 + "y": 25 }, "hiddenSeries": false, "id": 117, @@ -251,7 +252,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 33 }, "hiddenSeries": false, "id": 118, @@ -347,7 +348,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 33 }, "hiddenSeries": false, "id": 119, @@ -457,7 +458,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 2 }, "hiddenSeries": false, "id": 202, @@ -479,7 +480,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -555,7 +556,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 2 }, "hiddenSeries": false, "id": 203, @@ -577,7 +578,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -653,7 +654,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 10 }, "hiddenSeries": false, "id": 204, @@ -675,7 +676,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -751,7 +752,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 10 }, "hiddenSeries": false, "id": 205, @@ -773,7 +774,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -849,7 +850,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 18 }, "hiddenSeries": false, "id": 206, @@ -871,7 +872,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -948,7 +949,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 18 }, "hiddenSeries": false, "id": 207, @@ -970,7 +971,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1061,7 +1062,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 3 }, "hiddenSeries": false, "id": 195, @@ -1159,7 +1160,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 3 }, "hiddenSeries": false, "id": 196, @@ -1257,7 +1258,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 11 }, "hiddenSeries": false, "id": 197, @@ -1355,7 +1356,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 11 }, "hiddenSeries": false, "id": 198, @@ -1453,7 +1454,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 19 }, "hiddenSeries": false, "id": 199, @@ -1552,7 +1553,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 19 }, "hiddenSeries": false, "id": 200, @@ -1665,7 +1666,7 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 4 }, "hiddenSeries": false, "id": 188, @@ -1763,7 +1764,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 4 }, "hiddenSeries": false, "id": 189, @@ -1861,7 +1862,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 12 }, "hiddenSeries": false, "id": 190, @@ -1959,7 +1960,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 12 }, "hiddenSeries": false, "id": 191, @@ -2057,7 +2058,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 20 }, "hiddenSeries": false, "id": 192, @@ -2156,7 +2157,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 20 }, "hiddenSeries": false, "id": 193, @@ -2267,7 +2268,7 @@ "h": 8, "w": 12, "x": 0, - "y": 161 + "y": 193 }, "hiddenSeries": false, "id": 4, @@ -2381,7 +2382,7 @@ "h": 8, "w": 12, "x": 12, - "y": 161 + "y": 193 }, "hiddenSeries": false, "id": 2, @@ -2529,7 +2530,7 @@ "h": 8, "w": 12, "x": 0, - "y": 6 + "y": 7 }, "hiddenSeries": false, "id": 107, @@ -2623,7 +2624,7 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 7 }, "hiddenSeries": false, "id": 108, @@ -2717,7 +2718,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 15 }, "hiddenSeries": false, "id": 87, @@ -2811,7 +2812,7 @@ "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 15 }, "hiddenSeries": false, "id": 91, @@ -2905,7 +2906,7 @@ "h": 8, "w": 12, "x": 0, - "y": 22 + "y": 23 }, "hiddenSeries": false, "id": 88, @@ -2999,7 +3000,7 @@ "h": 8, "w": 12, "x": 12, - "y": 22 + "y": 23 }, "hiddenSeries": false, "id": 92, @@ -3093,7 +3094,7 @@ "h": 8, "w": 12, "x": 0, - "y": 30 + "y": 31 }, "hiddenSeries": false, "id": 89, @@ -3187,7 +3188,7 @@ "h": 8, "w": 12, "x": 12, - "y": 30 + "y": 31 }, "hiddenSeries": false, "id": 90, @@ -3281,7 +3282,7 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 39 }, "hiddenSeries": false, "id": 93, @@ -3375,7 +3376,7 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 39 }, "hiddenSeries": false, "id": 94, @@ -3469,7 +3470,7 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 47 }, "hiddenSeries": false, "id": 99, @@ -3563,7 +3564,7 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 47 }, "hiddenSeries": false, "id": 100, @@ -3657,7 +3658,7 @@ "h": 8, "w": 12, "x": 0, - "y": 54 + "y": 55 }, "hiddenSeries": false, "id": 95, @@ -3751,7 +3752,7 @@ "h": 8, "w": 12, "x": 12, - "y": 54 + "y": 55 }, "hiddenSeries": false, "id": 96, @@ -3845,7 +3846,7 @@ "h": 8, "w": 12, "x": 0, - "y": 62 + "y": 63 }, "hiddenSeries": false, "id": 97, @@ -3939,7 +3940,7 @@ "h": 8, "w": 12, "x": 12, - "y": 62 + "y": 63 }, "hiddenSeries": false, "id": 98, @@ -4033,7 +4034,7 @@ "h": 8, "w": 12, "x": 0, - "y": 70 + "y": 71 }, "hiddenSeries": false, "id": 101, @@ -4127,7 +4128,7 @@ "h": 8, "w": 12, "x": 12, - "y": 70 + "y": 71 }, "hiddenSeries": false, "id": 102, @@ -4221,7 +4222,7 @@ "h": 8, "w": 12, "x": 0, - "y": 78 + "y": 79 }, "hiddenSeries": false, "id": 103, @@ -4315,7 +4316,7 @@ "h": 8, "w": 12, "x": 12, - "y": 78 + "y": 79 }, "hiddenSeries": false, "id": 104, @@ -4409,7 +4410,7 @@ "h": 8, "w": 12, "x": 0, - "y": 86 + "y": 87 }, "hiddenSeries": false, "id": 105, @@ -4503,7 +4504,7 @@ "h": 8, "w": 12, "x": 12, - "y": 86 + "y": 87 }, "hiddenSeries": false, "id": 106, @@ -4597,7 +4598,7 @@ "h": 8, "w": 12, "x": 0, - "y": 94 + "y": 95 }, "hiddenSeries": false, "id": 109, @@ -4718,7 +4719,7 @@ "h": 8, "w": 24, "x": 0, - "y": 203 + "y": 235 }, "hiddenSeries": false, "id": 177, @@ -4839,7 +4840,7 @@ "h": 8, "w": 12, "x": 0, - "y": 164 + "y": 196 }, "hiddenSeries": false, "id": 48, @@ -4935,7 +4936,7 @@ "h": 8, "w": 12, "x": 12, - "y": 164 + "y": 196 }, "hiddenSeries": false, "id": 49, @@ -5031,7 +5032,7 @@ "h": 8, "w": 12, "x": 0, - "y": 172 + "y": 204 }, "hiddenSeries": false, "id": 50, @@ -5127,7 +5128,7 @@ "h": 8, "w": 12, "x": 12, - "y": 172 + "y": 204 }, "hiddenSeries": false, "id": 51, @@ -5223,7 +5224,7 @@ "h": 8, "w": 12, "x": 0, - "y": 180 + "y": 212 }, "hiddenSeries": false, "id": 52, @@ -5313,7 +5314,7 @@ "h": 8, "w": 12, "x": 12, - "y": 180 + "y": 212 }, "hiddenSeries": false, "id": 53, @@ -5409,7 +5410,7 @@ "h": 8, "w": 12, "x": 0, - "y": 188 + "y": 220 }, "hiddenSeries": false, "id": 54, @@ -5505,7 +5506,7 @@ "h": 8, "w": 12, "x": 12, - "y": 188 + "y": 220 }, "hiddenSeries": false, "id": 43, @@ -5601,7 +5602,7 @@ "h": 8, "w": 12, "x": 0, - "y": 196 + "y": 228 }, "hiddenSeries": false, "id": 56, @@ -5697,7 +5698,7 @@ "h": 8, "w": 12, "x": 12, - "y": 196 + "y": 228 }, "hiddenSeries": false, "id": 57, @@ -5793,7 +5794,7 @@ "h": 8, "w": 12, "x": 0, - "y": 204 + "y": 236 }, "hiddenSeries": false, "id": 58, @@ -5890,7 +5891,7 @@ "h": 8, "w": 12, "x": 12, - "y": 204 + "y": 236 }, "hiddenSeries": false, "id": 59, @@ -6014,7 +6015,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 41 }, "hiddenSeries": false, "id": 10, @@ -6110,7 +6111,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 41 }, "hiddenSeries": false, "id": 11, @@ -6206,7 +6207,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 49 }, "hiddenSeries": false, "id": 12, @@ -6302,7 +6303,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 49 }, "hiddenSeries": false, "id": 13, @@ -6398,7 +6399,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 57 }, "hiddenSeries": false, "id": 14, @@ -6488,7 +6489,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 57 }, "hiddenSeries": false, "id": 15, @@ -6584,7 +6585,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 65 }, "hiddenSeries": false, "id": 42, @@ -6680,7 +6681,7 @@ "h": 8, "w": 12, "x": 12, - "y": 33 + "y": 65 }, "hiddenSeries": false, "id": 55, @@ -6776,7 +6777,7 @@ "h": 8, "w": 12, "x": 0, - "y": 41 + "y": 73 }, "hiddenSeries": false, "id": 60, @@ -6872,7 +6873,7 @@ "h": 8, "w": 12, "x": 12, - "y": 41 + "y": 73 }, "hiddenSeries": false, "id": 61, @@ -6968,7 +6969,7 @@ "h": 8, "w": 12, "x": 0, - "y": 49 + "y": 81 }, "hiddenSeries": false, "id": 26, @@ -7065,7 +7066,7 @@ "h": 8, "w": 12, "x": 12, - "y": 49 + "y": 81 }, "hiddenSeries": false, "id": 27, @@ -7189,7 +7190,7 @@ "h": 8, "w": 12, "x": 0, - "y": 206 + "y": 238 }, "hiddenSeries": false, "id": 73, @@ -7285,7 +7286,7 @@ "h": 8, "w": 12, "x": 12, - "y": 206 + "y": 238 }, "hiddenSeries": false, "id": 74, @@ -7381,7 +7382,7 @@ "h": 8, "w": 12, "x": 0, - "y": 214 + "y": 246 }, "hiddenSeries": false, "id": 75, @@ -7477,7 +7478,7 @@ "h": 8, "w": 12, "x": 12, - "y": 214 + "y": 246 }, "hiddenSeries": false, "id": 76, @@ -7573,7 +7574,7 @@ "h": 8, "w": 12, "x": 0, - "y": 222 + "y": 254 }, "hiddenSeries": false, "id": 77, @@ -7670,7 +7671,7 @@ "h": 8, "w": 12, "x": 12, - "y": 222 + "y": 254 }, "hiddenSeries": false, "id": 78, @@ -7794,7 +7795,7 @@ "h": 8, "w": 12, "x": 0, - "y": 65 + "y": 97 }, "hiddenSeries": false, "id": 62, @@ -7890,7 +7891,7 @@ "h": 8, "w": 12, "x": 12, - "y": 65 + "y": 97 }, "hiddenSeries": false, "id": 63, @@ -7986,7 +7987,7 @@ "h": 8, "w": 12, "x": 0, - "y": 73 + "y": 105 }, "hiddenSeries": false, "id": 66, @@ -8082,7 +8083,7 @@ "h": 8, "w": 12, "x": 12, - "y": 73 + "y": 105 }, "hiddenSeries": false, "id": 65, @@ -8178,7 +8179,7 @@ "h": 8, "w": 12, "x": 0, - "y": 81 + "y": 113 }, "hiddenSeries": false, "id": 64, @@ -8275,7 +8276,7 @@ "h": 8, "w": 12, "x": 12, - "y": 81 + "y": 113 }, "hiddenSeries": false, "id": 67, @@ -8399,7 +8400,7 @@ "h": 8, "w": 12, "x": 0, - "y": 256 + "y": 288 }, "hiddenSeries": false, "id": 81, @@ -8495,7 +8496,7 @@ "h": 8, "w": 12, "x": 12, - "y": 256 + "y": 288 }, "hiddenSeries": false, "id": 82, @@ -8591,7 +8592,7 @@ "h": 8, "w": 12, "x": 0, - "y": 264 + "y": 296 }, "hiddenSeries": false, "id": 83, @@ -8715,7 +8716,7 @@ "h": 8, "w": 12, "x": 0, - "y": 273 + "y": 305 }, "hiddenSeries": false, "id": 112, @@ -8811,7 +8812,7 @@ "h": 8, "w": 12, "x": 12, - "y": 273 + "y": 305 }, "hiddenSeries": false, "id": 113, @@ -8934,7 +8935,7 @@ "h": 8, "w": 12, "x": 0, - "y": 210 + "y": 47 }, "hiddenSeries": false, "id": 122, @@ -8956,7 +8957,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -9030,7 +9031,7 @@ "h": 8, "w": 12, "x": 12, - "y": 210 + "y": 47 }, "hiddenSeries": false, "id": 123, @@ -9052,7 +9053,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -9126,7 +9127,7 @@ "h": 8, "w": 12, "x": 0, - "y": 218 + "y": 55 }, "hiddenSeries": false, "id": 124, @@ -9148,7 +9149,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -9222,7 +9223,7 @@ "h": 8, "w": 12, "x": 12, - "y": 218 + "y": 55 }, "hiddenSeries": false, "id": 125, @@ -9244,7 +9245,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -9318,7 +9319,7 @@ "h": 8, "w": 12, "x": 0, - "y": 226 + "y": 63 }, "hiddenSeries": false, "id": 126, @@ -9340,7 +9341,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -9414,7 +9415,7 @@ "h": 8, "w": 12, "x": 12, - "y": 226 + "y": 63 }, "hiddenSeries": false, "id": 127, @@ -9436,7 +9437,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -9525,7 +9526,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "", + "description": "Total cache disk capacity", "fieldConfig": { "defaults": { "unit": "bytes" @@ -9576,7 +9577,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_disk_cache_[[fs:regex]]_diskcache_totalbytes\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_diskcache_totalbytes\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "range": true, @@ -9624,6 +9625,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "The size occupied by S3 objects cached in the cache disk.", "fieldConfig": { "defaults": { "unit": "bytes" @@ -9672,10 +9674,12 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_disk_cache_[[fs:regex]]_diskcache_usedbytes\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_diskcache_usedbytes\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], @@ -9819,10 +9823,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Cache hit rate", + "description": "diskcache usage in lru", "fieldConfig": { "defaults": { - "unit": "percentunit" + "unit": "bytes" }, "overrides": [] }, @@ -9835,7 +9839,7 @@ "y": 23 }, "hiddenSeries": false, - "id": 213, + "id": 217, "interval": "1s", "legend": { "alignAsTable": true, @@ -9868,44 +9872,16 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "exemplar": true, - "expr": "{__name__=~\"diskcache_cache_hit\", instance=~\"$instance\"}", - "hide": true, - "interval": "", - "legendFormat": "", - "range": true, - "refId": "hit" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"diskcache_cache_miss\", instance=~\"$instance\"}", - "hide": true, + "expr": "{__name__=~\"diskcache_cache_bytes\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", - "range": true, - "refId": "miss" - }, - { - "datasource": { - "name": "Expression", - "type": "__expr__", - "uid": "__expr__" - }, - "expression": "$hit/($hit+$miss)", - "hide": false, - "refId": "A", - "type": "math" + "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "cache hit rate", + "title": "cache bytes", "tooltip": { "shared": true, "sort": 0, @@ -9920,7 +9896,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "percentunit", + "format": "bytes", "logBase": 1, "show": true }, @@ -10043,10 +10019,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Number of cache misses", + "description": "Cache hit rate", "fieldConfig": { "defaults": { - "unit": "short" + "unit": "percentunit" }, "overrides": [] }, @@ -10059,7 +10035,7 @@ "y": 31 }, "hiddenSeries": false, - "id": 212, + "id": 213, "interval": "1s", "legend": { "alignAsTable": true, @@ -10087,6 +10063,20 @@ "stack": false, "steppedLine": false, "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"diskcache_cache_hit\", instance=~\"$instance\"}", + "hide": true, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "hit" + }, { "datasource": { "type": "prometheus", @@ -10095,15 +10085,27 @@ "editorMode": "code", "exemplar": true, "expr": "{__name__=~\"diskcache_cache_miss\", instance=~\"$instance\"}", + "hide": true, "interval": "", "legendFormat": "", "range": true, - "refId": "A" + "refId": "miss" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$hit/($hit+$miss)", + "hide": false, + "refId": "A", + "type": "math" } ], "thresholds": [], "timeRegions": [], - "title": "cache miss", + "title": "cache hit rate", "tooltip": { "shared": true, "sort": 0, @@ -10118,7 +10120,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "short", + "format": "percentunit", "logBase": 1, "show": true }, @@ -10142,7 +10144,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "This metric measures the efficiency of evicting objects from the cache disk, recorded as the number of objects evicted from the cache per second", + "description": "Number of cache misses", "fieldConfig": { "defaults": { "unit": "short" @@ -10158,7 +10160,7 @@ "y": 39 }, "hiddenSeries": false, - "id": 214, + "id": 212, "interval": "1s", "legend": { "alignAsTable": true, @@ -10193,7 +10195,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_disk_cache_[[fs:regex]]_diskcache_trim_qps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"diskcache_cache_miss\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "range": true, @@ -10202,7 +10204,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "trim qps", + "title": "cache miss", "tooltip": { "shared": true, "sort": 0, @@ -10340,10 +10342,10 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "This metric tracks the cache eviction latency. It measures how long it takes to evict objects from the cache disk, recorded in units like milliseconds or microseconds per eviction.", + "description": "This metric measures the efficiency of evicting objects from the cache disk, recorded as the number of objects evicted from the cache per second", "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "short" }, "overrides": [] }, @@ -10356,7 +10358,7 @@ "y": 47 }, "hiddenSeries": false, - "id": 216, + "id": 214, "interval": "1s", "legend": { "alignAsTable": true, @@ -10391,7 +10393,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_disk_cache_[[fs:regex]]_diskcache_trim_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_diskcache_trim_qps\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", "range": true, @@ -10400,7 +10402,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "trim latency", + "title": "trim qps", "tooltip": { "shared": true, "sort": 0, @@ -10415,7 +10417,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "µs", + "format": "short", "logBase": 1, "show": true }, @@ -10429,34 +10431,7 @@ "yaxis": { "align": false } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refId": "A" - } - ], - "title": "diskcache cache", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 32, - "panels": [ + }, { "aliasColors": {}, "bars": false, @@ -10466,22 +10441,23 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "This metric tracks the cache eviction latency. It measures how long it takes to evict objects from the cache disk, recorded in units like milliseconds or microseconds per eviction.", "fieldConfig": { "defaults": { - "unit": "short" + "unit": "µs" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 0, - "y": 332 + "x": 12, + "y": 47 }, "hiddenSeries": false, - "id": 36, + "id": 216, "interval": "1s", "legend": { "alignAsTable": true, @@ -10500,7 +10476,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -10514,16 +10490,18 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", "exemplar": true, - "expr": "{__name__=~\"curvefs_mds_client.*mount_fs_qps\", instance=~\"$instance\"}", + "expr": "{__name__=~\"curvefs_disk_cache.*[[fs:regex]]_diskcache_trim_lat_[[quantile:regex]]\", instance=~\"$instance\"}", "interval": "", "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "mountfs_qps", + "title": "trim latency", "tooltip": { "shared": true, "sort": 0, @@ -10538,7 +10516,7 @@ "yaxes": [ { "$$hashKey": "object:212", - "format": "short", + "format": "µs", "logBase": 1, "show": true }, @@ -10552,18 +10530,1785 @@ "yaxis": { "align": false } - }, + } + ], + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { + "refId": "A" + } + ], + "title": "diskcache cache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 218, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of times using kvcache cache (set)", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 219, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_count\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "set count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "kvcache cache hit rate", + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 220, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_hit\", instance=~\"$instance\"}", + "hide": true, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "hit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_miss\", instance=~\"$instance\"}", + "hide": true, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "miss" + }, + { + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$hit/($hit+$miss)", + "hide": false, + "refId": "hit rate", + "type": "math" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "hit rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "percentunit", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of kvcache cache hits", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 221, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_hit\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "hit" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "hit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Number of kvcache cache misses", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 222, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_miss\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "miss" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "miss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "kvcache get qps", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 223, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_get_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "get qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "kvcache set qps", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "hiddenSeries": false, + "id": 226, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_set_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "set qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "kvcache get latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 225, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_get_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "get latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "kvcache set latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 227, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_set_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "set latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "kvcache get bps", + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 224, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_get_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "get bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "kvcache set bps", + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "hiddenSeries": false, + "id": 228, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_kvclient_manager.*[[fs:regex]]_set_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "set bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "kvcache manager", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 229, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "memcache client get latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 232, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_get_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "get latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "memcache client set qps", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 231, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_set_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "set qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "memcache client get bps", + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 234, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_get_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "get bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "memcache client set latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "hiddenSeries": false, + "id": 233, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_set_lat_[[quantile:regex]]\", job=\"client\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "set latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "memcache get qps", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 230, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_get_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "get qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "memcache client set bps", + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "hiddenSeries": false, + "id": 235, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_memcache_client.*[[fs:regex]]_set_bps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "set bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "memcache client", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 32, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 364 + }, + "hiddenSeries": false, + "id": 36, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_mds_client.*mount_fs_qps\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "mountfs_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { "unit": "µs" }, "overrides": [] @@ -10574,7 +12319,7 @@ "h": 6, "w": 12, "x": 12, - "y": 332 + "y": 364 }, "hiddenSeries": false, "id": 128, @@ -10670,7 +12415,7 @@ "h": 6, "w": 12, "x": 0, - "y": 338 + "y": 370 }, "hiddenSeries": false, "id": 129, @@ -10766,7 +12511,7 @@ "h": 6, "w": 12, "x": 12, - "y": 338 + "y": 370 }, "hiddenSeries": false, "id": 130, @@ -10862,7 +12607,7 @@ "h": 6, "w": 12, "x": 0, - "y": 344 + "y": 376 }, "hiddenSeries": false, "id": 131, @@ -10958,7 +12703,7 @@ "h": 6, "w": 12, "x": 12, - "y": 344 + "y": 376 }, "hiddenSeries": false, "id": 132, @@ -11054,7 +12799,7 @@ "h": 6, "w": 12, "x": 0, - "y": 350 + "y": 382 }, "hiddenSeries": false, "id": 133, @@ -11150,7 +12895,7 @@ "h": 6, "w": 12, "x": 12, - "y": 350 + "y": 382 }, "hiddenSeries": false, "id": 134, @@ -11246,7 +12991,7 @@ "h": 6, "w": 12, "x": 0, - "y": 356 + "y": 388 }, "hiddenSeries": false, "id": 135, @@ -11342,7 +13087,7 @@ "h": 6, "w": 12, "x": 12, - "y": 356 + "y": 388 }, "hiddenSeries": false, "id": 136, @@ -11438,7 +13183,7 @@ "h": 6, "w": 12, "x": 0, - "y": 362 + "y": 394 }, "hiddenSeries": false, "id": 137, @@ -11534,7 +13279,7 @@ "h": 6, "w": 12, "x": 12, - "y": 362 + "y": 394 }, "hiddenSeries": false, "id": 138, @@ -11630,7 +13375,7 @@ "h": 6, "w": 12, "x": 0, - "y": 368 + "y": 400 }, "hiddenSeries": false, "id": 139, @@ -11726,7 +13471,7 @@ "h": 6, "w": 12, "x": 12, - "y": 368 + "y": 400 }, "hiddenSeries": false, "id": 140, @@ -11821,7 +13566,7 @@ "h": 6, "w": 12, "x": 0, - "y": 374 + "y": 406 }, "hiddenSeries": false, "id": 141, @@ -11916,7 +13661,7 @@ "h": 6, "w": 12, "x": 12, - "y": 374 + "y": 406 }, "hiddenSeries": false, "id": 142, @@ -12012,7 +13757,7 @@ "h": 6, "w": 12, "x": 0, - "y": 380 + "y": 412 }, "hiddenSeries": false, "id": 143, @@ -12108,7 +13853,7 @@ "h": 6, "w": 12, "x": 12, - "y": 380 + "y": 412 }, "hiddenSeries": false, "id": 144, @@ -12204,7 +13949,7 @@ "h": 6, "w": 12, "x": 0, - "y": 386 + "y": 418 }, "hiddenSeries": false, "id": 145, @@ -12300,7 +14045,7 @@ "h": 6, "w": 12, "x": 12, - "y": 386 + "y": 418 }, "hiddenSeries": false, "id": 146, @@ -12396,7 +14141,7 @@ "h": 6, "w": 12, "x": 0, - "y": 392 + "y": 424 }, "hiddenSeries": false, "id": 147, @@ -12492,7 +14237,7 @@ "h": 6, "w": 12, "x": 12, - "y": 392 + "y": 424 }, "hiddenSeries": false, "id": 148, @@ -12588,7 +14333,7 @@ "h": 6, "w": 12, "x": 0, - "y": 398 + "y": 430 }, "hiddenSeries": false, "id": 149, @@ -12684,7 +14429,7 @@ "h": 6, "w": 12, "x": 12, - "y": 398 + "y": 430 }, "hiddenSeries": false, "id": 150, @@ -12782,7 +14527,7 @@ "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 18 }, "id": 34, "panels": [ @@ -12807,7 +14552,7 @@ "h": 6, "w": 12, "x": 0, - "y": 405 + "y": 50 }, "hiddenSeries": false, "id": 155, @@ -12829,7 +14574,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -12903,7 +14648,7 @@ "h": 6, "w": 12, "x": 12, - "y": 405 + "y": 50 }, "hiddenSeries": false, "id": 152, @@ -12925,7 +14670,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -12999,7 +14744,7 @@ "h": 6, "w": 12, "x": 0, - "y": 411 + "y": 56 }, "hiddenSeries": false, "id": 153, @@ -13021,7 +14766,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -13095,7 +14840,7 @@ "h": 6, "w": 12, "x": 12, - "y": 411 + "y": 56 }, "hiddenSeries": false, "id": 156, @@ -13117,7 +14862,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -13191,7 +14936,7 @@ "h": 6, "w": 12, "x": 0, - "y": 417 + "y": 62 }, "hiddenSeries": false, "id": 157, @@ -13213,7 +14958,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -13287,7 +15032,7 @@ "h": 6, "w": 12, "x": 12, - "y": 417 + "y": 62 }, "hiddenSeries": false, "id": 154, @@ -13309,7 +15054,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -13383,7 +15128,7 @@ "h": 6, "w": 12, "x": 0, - "y": 423 + "y": 68 }, "hiddenSeries": false, "id": 151, @@ -13479,7 +15224,7 @@ "h": 6, "w": 12, "x": 12, - "y": 423 + "y": 68 }, "hiddenSeries": false, "id": 158, @@ -13575,7 +15320,7 @@ "h": 6, "w": 12, "x": 0, - "y": 429 + "y": 74 }, "hiddenSeries": false, "id": 159, @@ -13671,7 +15416,7 @@ "h": 6, "w": 12, "x": 12, - "y": 429 + "y": 74 }, "hiddenSeries": false, "id": 160, @@ -13767,7 +15512,7 @@ "h": 6, "w": 12, "x": 0, - "y": 435 + "y": 80 }, "hiddenSeries": false, "id": 161, @@ -13863,7 +15608,7 @@ "h": 6, "w": 12, "x": 12, - "y": 435 + "y": 80 }, "hiddenSeries": false, "id": 162, @@ -13959,7 +15704,7 @@ "h": 6, "w": 12, "x": 0, - "y": 441 + "y": 86 }, "hiddenSeries": false, "id": 163, @@ -14055,7 +15800,7 @@ "h": 6, "w": 12, "x": 12, - "y": 441 + "y": 86 }, "hiddenSeries": false, "id": 164, @@ -14151,7 +15896,7 @@ "h": 6, "w": 12, "x": 0, - "y": 447 + "y": 92 }, "hiddenSeries": false, "id": 165, @@ -14247,7 +15992,7 @@ "h": 6, "w": 12, "x": 12, - "y": 447 + "y": 92 }, "hiddenSeries": false, "id": 166, @@ -14343,7 +16088,7 @@ "h": 6, "w": 12, "x": 0, - "y": 453 + "y": 98 }, "hiddenSeries": false, "id": 167, @@ -14439,7 +16184,7 @@ "h": 6, "w": 12, "x": 12, - "y": 453 + "y": 98 }, "hiddenSeries": false, "id": 168, @@ -14535,7 +16280,7 @@ "h": 6, "w": 12, "x": 0, - "y": 459 + "y": 104 }, "hiddenSeries": false, "id": 169, @@ -14631,7 +16376,7 @@ "h": 6, "w": 12, "x": 12, - "y": 459 + "y": 104 }, "hiddenSeries": false, "id": 170, @@ -14727,7 +16472,7 @@ "h": 6, "w": 12, "x": 0, - "y": 465 + "y": 110 }, "hiddenSeries": false, "id": 171, @@ -14823,7 +16568,7 @@ "h": 6, "w": 12, "x": 12, - "y": 465 + "y": 110 }, "hiddenSeries": false, "id": 172, @@ -14919,7 +16664,7 @@ "h": 6, "w": 12, "x": 0, - "y": 471 + "y": 116 }, "hiddenSeries": false, "id": 173, @@ -15015,7 +16760,7 @@ "h": 6, "w": 12, "x": 12, - "y": 471 + "y": 116 }, "hiddenSeries": false, "id": 174, @@ -15109,7 +16854,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 19 }, "id": 180, "panels": [ @@ -15174,7 +16919,7 @@ "h": 8, "w": 12, "x": 0, - "y": 174 + "y": 206 }, "id": 182, "options": { @@ -15269,7 +17014,7 @@ "h": 8, "w": 12, "x": 12, - "y": 174 + "y": 206 }, "id": 184, "options": { @@ -15365,7 +17110,7 @@ "h": 8, "w": 12, "x": 0, - "y": 182 + "y": 214 }, "id": 186, "options": { @@ -15410,9 +17155,13 @@ "list": [ { "current": { - "selected": false, - "text": "All", - "value": "$__all" + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] }, "datasource": { "type": "prometheus", @@ -15437,7 +17186,7 @@ }, { "current": { - "selected": true, + "selected": false, "text": [ "All" ], @@ -15500,7 +17249,7 @@ }, { "current": { - "selected": true, + "selected": false, "text": [ "All" ], diff --git a/curvefs/src/client/fuse_s3_client.cpp b/curvefs/src/client/fuse_s3_client.cpp index e94fe31e2c..829f00e1d3 100644 --- a/curvefs/src/client/fuse_s3_client.cpp +++ b/curvefs/src/client/fuse_s3_client.cpp @@ -116,14 +116,14 @@ bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) { // init kvcache client auto memcacheClient = std::make_shared(); - if (!memcacheClient->Init(kvcachecluster)) { + if (!memcacheClient->Init(kvcachecluster, fsInfo_->fsname())) { LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache << ", but init memcache client fail"; return false; } kvClientManager_ = std::make_shared(); - if (!kvClientManager_->Init(opt, memcacheClient)) { + if (!kvClientManager_->Init(opt, memcacheClient, fsInfo_->fsname())) { LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache << ", but init kvClientManager fail"; return false; diff --git a/curvefs/src/client/kvclient/kvclient.h b/curvefs/src/client/kvclient/kvclient.h index 9d7b96d857..a96c733762 100644 --- a/curvefs/src/client/kvclient/kvclient.h +++ b/curvefs/src/client/kvclient/kvclient.h @@ -22,6 +22,8 @@ #ifndef CURVEFS_SRC_CLIENT_KVCLIENT_KVCLIENT_H_ #define CURVEFS_SRC_CLIENT_KVCLIENT_KVCLIENT_H_ +#include + #include namespace curvefs { @@ -49,8 +51,9 @@ class KVClient { virtual bool Set(const std::string &key, const char *value, const uint64_t value_len, std::string *errorlog) = 0; - virtual bool Get(const std::string &key, char *value, uint64_t offset, - uint64_t length, std::string *errorlog) = 0; + virtual bool Get(const std::string& key, char* value, uint64_t offset, + uint64_t length, std::string* errorlog, + uint64_t* actLength, memcached_return_t* retCod) = 0; }; } // namespace client diff --git a/curvefs/src/client/kvclient/kvclient_manager.cpp b/curvefs/src/client/kvclient/kvclient_manager.cpp index a4a349080b..4ce8f6bd78 100644 --- a/curvefs/src/client/kvclient/kvclient_manager.cpp +++ b/curvefs/src/client/kvclient/kvclient_manager.cpp @@ -21,27 +21,37 @@ */ #include "curvefs/src/client/kvclient/kvclient_manager.h" + #include + +#include "absl/memory/memory.h" +#include "curvefs/src/client/metric/client_metric.h" #include "src/client/client_metric.h" #include "src/common/concurrent/count_down_event.h" using curve::client::LatencyGuard; using curve::common::CountDownEvent; -using curvefs::client::metric::KVClientMetric; +using curvefs::client::metric::KVClientManagerMetric; namespace curvefs { namespace client { -#define ONRETURN(TYPE, RES) \ - if (RES) { \ - kvClientMetric_.kvClient##TYPE.qps.count << 1; \ - } else { \ - kvClientMetric_.kvClient##TYPE.eps.count << 1; \ - } \ +template +void OnReturn(Metric* metric, const TaskSharePtr task) { + task->timer.stop(); + if (task->res) { + metric::CollectMetrics(metric, task->length, task->timer.u_elapsed()); + } else { + metric->eps.count << 1; + } + task->done(task); +} -bool KVClientManager::Init(const KVClientManagerOpt &config, - const std::shared_ptr &kvclient) { +bool KVClientManager::Init(const KVClientManagerOpt& config, + const std::shared_ptr& kvclient, + const std::string& fsName) { client_ = kvclient; + kvClientManagerMetric_ = absl::make_unique(fsName); return threadPool_.Start(config.setThreadPooln) == 0; } @@ -52,29 +62,47 @@ void KVClientManager::Uninit() { void KVClientManager::Set(std::shared_ptr task) { threadPool_.Enqueue([task, this]() { - LatencyGuard guard(&kvClientMetric_.kvClientSet.latency); - std::string error_log; task->res = client_->Set(task->key, task->value, task->length, &error_log); - task->timer.stop(); - ONRETURN(Set, task->res); - - task->done(task); + if (task->res) { + kvClientManagerMetric_->count << 1; + } + OnReturn(&kvClientManagerMetric_->set, task); }); } +void UpdateHitMissMetric(memcached_return_t retCode, + KVClientManagerMetric* metric) { + // https://awesomized.github.io/libmemcached/libmemcached/memcached_return_t.html + switch (retCode) { + case MEMCACHED_SUCCESS: + metric->hit << 1; + break; + case MEMCACHED_DATA_DOES_NOT_EXIST: + // The data requested with the key given was not found. + case MEMCACHED_DATA_EXISTS: + // The data requested with the key given was not found. + case MEMCACHED_DELETED: + // The object requested by the key has been deleted. + case MEMCACHED_NOTFOUND: + // The object requested was not found. + metric->miss << 1; + break; + default: + break; + } +} + void KVClientManager::Get(std::shared_ptr task) { threadPool_.Enqueue([task, this]() { - LatencyGuard guard(&kvClientMetric_.kvClientGet.latency); - std::string error_log; + memcached_return_t retCode; task->res = client_->Get(task->key, task->value, task->offset, - task->length, &error_log); - task->timer.stop(); - ONRETURN(Get, task->res); - - task->done(task); + task->valueLength, &error_log, &task->length, + &retCode); + UpdateHitMissMetric(retCode, kvClientManagerMetric_.get()); + OnReturn(&kvClientManagerMetric_->get, task); }); } diff --git a/curvefs/src/client/kvclient/kvclient_manager.h b/curvefs/src/client/kvclient/kvclient_manager.h index de2ae588a6..ad8701ec84 100644 --- a/curvefs/src/client/kvclient/kvclient_manager.h +++ b/curvefs/src/client/kvclient/kvclient_manager.h @@ -25,19 +25,20 @@ #include -#include +#include #include -#include #include +#include +#include #include "absl/strings/string_view.h" -#include "curvefs/src/client/kvclient/kvclient.h" #include "curvefs/src/client/common/config.h" +#include "curvefs/src/client/kvclient/kvclient.h" #include "curvefs/src/client/metric/client_metric.h" #include "src/common/concurrent/task_thread_pool.h" #include "src/common/s3_adapter.h" -using curvefs::client::metric::KVClientMetric; +using curvefs::client::metric::KVClientManagerMetric; namespace curvefs { namespace client { @@ -76,7 +77,8 @@ struct GetKVCacheTask { const std::string& key; char* value; uint64_t offset; - uint64_t length; + uint64_t valueLength; + uint64_t length; // actual length of value bool res; GetKVCacheDone done; butil::Timer timer; @@ -87,7 +89,8 @@ struct GetKVCacheTask { : key(k), value(v), offset(off), - length(len), + valueLength(len), + length(0), res(false), done(std::move(done)), timer(butil::Timer::STARTED) {} @@ -98,8 +101,9 @@ class KVClientManager { KVClientManager() = default; ~KVClientManager() { Uninit(); } - bool Init(const KVClientManagerOpt &config, - const std::shared_ptr &kvclient); + bool Init(const KVClientManagerOpt& config, + const std::shared_ptr& kvclient, + const std::string& fsName); /** * It will get a db client and set the key value asynchronusly. @@ -110,7 +114,9 @@ class KVClientManager { void Get(std::shared_ptr task); - KVClientMetric *GetClientMetricForTesting() { return &kvClientMetric_; } + KVClientManagerMetric* GetMetricForTesting() { + return kvClientManagerMetric_.get(); + } private: void Uninit(); @@ -118,7 +124,7 @@ class KVClientManager { private: TaskThreadPool threadPool_; std::shared_ptr client_; - KVClientMetric kvClientMetric_; + std::unique_ptr kvClientManagerMetric_; }; } // namespace client diff --git a/curvefs/src/client/kvclient/memcache_client.h b/curvefs/src/client/kvclient/memcache_client.h index 82c11347ae..ab4e939a74 100644 --- a/curvefs/src/client/kvclient/memcache_client.h +++ b/curvefs/src/client/kvclient/memcache_client.h @@ -27,10 +27,14 @@ #include #include +#include +#include #include -#include "curvefs/src/client/kvclient/kvclient.h" +#include "absl/memory/memory.h" #include "curvefs/proto/topology.pb.h" +#include "curvefs/src/client/kvclient/kvclient.h" +#include "curvefs/src/client/metric/client_metric.h" namespace curvefs { @@ -83,7 +87,9 @@ class MemCachedClient : public KVClient { explicit MemCachedClient(memcached_st *cli) : client_(cli) {} ~MemCachedClient() { UnInit(); } - bool Init(const MemcacheClusterInfo &kvcachecluster) { + bool Init(const MemcacheClusterInfo& kvcachecluster, + const std::string& fsName) { + metric_ = absl::make_unique(fsName); client_ = memcached(nullptr, 0); for (int i = 0; i < kvcachecluster.servers_size(); i++) { @@ -106,8 +112,9 @@ class MemCachedClient : public KVClient { } } - bool Set(const std::string &key, const char *value, - const uint64_t value_len, std::string *errorlog) override { + bool Set(const std::string& key, const char* value, + const uint64_t value_len, std::string* errorlog) override { + uint64_t start = butil::cpuwide_time_us(); if (nullptr == tcli) { tcli = memcached_clone(nullptr, client_); } @@ -115,17 +122,22 @@ class MemCachedClient : public KVClient { value_len, 0, 0); if (MEMCACHED_SUCCESS == res) { VLOG(9) << "Set key = " << key << " OK"; + metric::CollectMetrics(&metric_->set, value_len, + butil::cpuwide_time_us() - start); return true; } *errorlog = ResError(res); memcached_free(tcli); tcli = nullptr; LOG(ERROR) << "Set key = " << key << " error = " << *errorlog; + metric_->set.eps.count << 1; return false; } - bool Get(const std::string &key, char *value, uint64_t offset, - uint64_t length, std::string *errorlog) override { + bool Get(const std::string& key, char* value, uint64_t offset, + uint64_t length, std::string* errorlog, uint64_t* actLength, + memcached_return_t* retCode) override { + uint64_t start = butil::cpuwide_time_us(); if (nullptr == tcli) { // multi thread use a memcached_st* client is unsafe. // should clone it or use memcached_st_pool. @@ -136,11 +148,19 @@ class MemCachedClient : public KVClient { memcached_return_t ue; char *res = memcached_get(tcli, key.c_str(), key.length(), &value_length, &flags, &ue); + if (actLength != nullptr) { + (*actLength) = value_length; + } + if (retCode != nullptr) { + (*retCode) = ue; + } if (MEMCACHED_SUCCESS == ue && res != nullptr && value && value_length >= length) { VLOG(9) << "Get key = " << key << " OK"; memcpy(value, res + offset, length); free(res); + metric::CollectMetrics(&metric_->get, value_length, + butil::cpuwide_time_us() - start); return true; } @@ -153,6 +173,7 @@ class MemCachedClient : public KVClient { tcli = nullptr; } + metric_->get.eps.count << 1; return false; } @@ -198,7 +219,8 @@ class MemCachedClient : public KVClient { private: memcached_server_st *server_; - memcached_st *client_; + memcached_st* client_; + std::unique_ptr metric_; }; } // namespace client diff --git a/curvefs/src/client/metric/client_metric.cpp b/curvefs/src/client/metric/client_metric.cpp index 6ae27d357f..8e9939413d 100644 --- a/curvefs/src/client/metric/client_metric.cpp +++ b/curvefs/src/client/metric/client_metric.cpp @@ -35,7 +35,10 @@ const std::string S3MultiManagerMetric::prefix = "curvefs_client_manager"; // N const std::string FSMetric::prefix = "curvefs_client"; // NOLINT const std::string S3Metric::prefix = "curvefs_s3"; // NOLINT const std::string DiskCacheMetric::prefix = "curvefs_disk_cache"; // NOLINT -const std::string KVClientMetric::prefix = "curvefs_kvclient"; // NOLINT +const std::string KVClientManagerMetric::prefix = // NOLINT + "curvefs_kvclient_manager"; // NOLINT +const std::string MemcacheClientMetric::prefix = // NOLINT + "curvefs_memcache_client"; // NOLINT const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info"; // NOLINT const std::string WarmupManagerS3Metric::prefix = "curvefs_warmup"; // NOLINT diff --git a/curvefs/src/client/metric/client_metric.h b/curvefs/src/client/metric/client_metric.h index 40b235a729..b8a7896f1f 100644 --- a/curvefs/src/client/metric/client_metric.h +++ b/curvefs/src/client/metric/client_metric.h @@ -304,13 +304,41 @@ struct DiskCacheMetric { trim_(prefix, fsName + "_diskcache_trim") {} }; -struct KVClientMetric { +struct KVClientManagerMetric { static const std::string prefix; - InterfaceMetric kvClientGet; - InterfaceMetric kvClientSet; - KVClientMetric() - : kvClientGet(prefix, "get"), kvClientSet(prefix, "set") {} + std::string fsName; + InterfaceMetric get; + InterfaceMetric set; + // kvcache count + bvar::Adder count; + // kvcache hit + bvar::Adder hit; + // kvcache miss + bvar::Adder miss; + + explicit KVClientManagerMetric(const std::string& name = "") + : fsName(!name.empty() ? name + : prefix + curve::common::ToHexString(this)), + get(prefix, fsName + "_get"), + set(prefix, fsName + "_set"), + count(prefix, fsName + "_count"), + hit(prefix, fsName + "_hit"), + miss(prefix, fsName + "_miss") {} +}; + +struct MemcacheClientMetric { + static const std::string prefix; + + std::string fsName; + InterfaceMetric get; + InterfaceMetric set; + + explicit MemcacheClientMetric(const std::string& name = "") + : fsName(!name.empty() ? name + : prefix + curve::common::ToHexString(this)), + get(prefix, fsName + "_get"), + set(prefix, fsName + "_set") {} }; struct S3ChunkInfoMetric { @@ -332,7 +360,7 @@ struct WarmupManagerS3Metric { warmupS3CacheSize(prefix, "s3_cache_size") {} }; -void CollectMetrics(InterfaceMetric* interface, int count, uint64_t start); +void CollectMetrics(InterfaceMetric* interface, int count, uint64_t u_elapsed); void AsyncContextCollectMetrics( std::shared_ptr s3Metric, diff --git a/curvefs/test/client/client_memcache_test.cpp b/curvefs/test/client/client_memcache_test.cpp index c0ce49e033..d898c429b7 100644 --- a/curvefs/test/client/client_memcache_test.cpp +++ b/curvefs/test/client/client_memcache_test.cpp @@ -55,12 +55,14 @@ class MemCachedTest : public ::testing::Test { } std::shared_ptr client(new MemCachedClient()); + MemcacheClusterInfo info; + client->Init(info, "test"); ASSERT_EQ(true, client->AddServer(hostname, port)); ASSERT_EQ(true, client->PushServer()); KVClientManagerOpt opt; opt.setThreadPooln = 2; opt.getThreadPooln = 2; - ASSERT_EQ(true, manager_.Init(opt, client)); + ASSERT_EQ(true, manager_.Init(opt, client, "test")); // wait memcached server start std::string errorlog; @@ -114,8 +116,7 @@ TEST_F(MemCachedTest, MultiThreadTask) { }); } taskEvent.Wait(); - ASSERT_EQ( - 5, manager_.GetClientMetricForTesting()->kvClientSet.latency.count()); + ASSERT_EQ(5, manager_.GetMetricForTesting()->set.latency.count()); // get for (int i = 0; i < 5; i++) { diff --git a/curvefs/test/client/client_s3_adaptor_Integration.cpp b/curvefs/test/client/client_s3_adaptor_Integration.cpp index 204637965c..522cfeb525 100644 --- a/curvefs/test/client/client_s3_adaptor_Integration.cpp +++ b/curvefs/test/client/client_s3_adaptor_Integration.cpp @@ -167,7 +167,7 @@ class ClientS3IntegrationTest : public testing::Test { KVClientManagerOpt opt; std::shared_ptr mockKVClient(&mockKVClient_); - kvClientManager_->Init(opt, mockKVClient); + kvClientManager_->Init(opt, mockKVClient, "test"); } protected: @@ -2896,7 +2896,7 @@ TEST_F(ClientS3IntegrationTest, test_write_read_remotekvcache) { EXPECT_CALL(mockInodeManager_, GetInode(_, _)) .WillOnce( DoAll(SetArgReferee<1>(inode), Return(CURVEFS_ERROR::OK))); - EXPECT_CALL(mockKVClient_, Get(_, _, 0, len, _)) + EXPECT_CALL(mockKVClient_, Get(_, _, 0, len, _, _, _)) .WillOnce(DoAll(SetArrayArgument<1>(buf, buf + len), Return(true))); int readLen = s3ClientAdaptor_->Read(inodeId, offset_0, len, readBuf); EXPECT_EQ(readLen, len); @@ -2910,7 +2910,7 @@ TEST_F(ClientS3IntegrationTest, test_write_read_remotekvcache) { EXPECT_CALL(mockInodeManager_, GetInode(_, _)) .WillOnce( DoAll(SetArgReferee<1>(inode), Return(CURVEFS_ERROR::OK))); - EXPECT_CALL(mockKVClient_, Get(_, _, 0, len, _)) + EXPECT_CALL(mockKVClient_, Get(_, _, 0, len, _, _, _)) .WillOnce(DoAll(SetArrayArgument<1>(buf, buf + len), Return(true))); int readLen = s3ClientAdaptor_->Read(inodeId, offset_4M, len, readBuf); EXPECT_EQ(readLen, len); @@ -2924,7 +2924,7 @@ TEST_F(ClientS3IntegrationTest, test_write_read_remotekvcache) { EXPECT_CALL(mockInodeManager_, GetInode(_, _)) .WillOnce( DoAll(SetArgReferee<1>(inode), Return(CURVEFS_ERROR::OK))); - EXPECT_CALL(mockKVClient_, Get(_, _, 0, len, _)) + EXPECT_CALL(mockKVClient_, Get(_, _, 0, len, _, _, _)) .WillOnce(Return(false)); EXPECT_CALL(mockS3Client_, Download(_, _, _, _)) .WillOnce(DoAll(SetArrayArgument<1>(buf, buf + len), Return(true))); diff --git a/curvefs/test/client/metric/client_metric_test.cpp b/curvefs/test/client/metric/client_metric_test.cpp index a7bd171477..9e159d27d1 100644 --- a/curvefs/test/client/metric/client_metric_test.cpp +++ b/curvefs/test/client/metric/client_metric_test.cpp @@ -31,15 +31,15 @@ #include "src/common/s3_adapter.h" -using ::curvefs::client::metric::MDSClientMetric; -using ::curvefs::client::metric::MetaServerClientMetric; using ::curvefs::client::metric::ClientOpMetric; -using ::curvefs::client::metric::S3MultiManagerMetric; -using ::curvefs::client::metric::FSMetric; -using ::curvefs::client::metric::S3Metric; using ::curvefs::client::metric::DiskCacheMetric; -using ::curvefs::client::metric::KVClientMetric; +using ::curvefs::client::metric::FSMetric; +using ::curvefs::client::metric::KVClientManagerMetric; +using ::curvefs::client::metric::MDSClientMetric; +using ::curvefs::client::metric::MetaServerClientMetric; using ::curvefs::client::metric::S3ChunkInfoMetric; +using ::curvefs::client::metric::S3Metric; +using ::curvefs::client::metric::S3MultiManagerMetric; using ::curvefs::client::metric::WarmupManagerS3Metric; @@ -89,8 +89,8 @@ TEST_F(ClientMetricTest, test_prefix) { } { - const char* prefix = "curvefs_kvclient"; - ASSERT_EQ(0, ::strcmp(KVClientMetric::prefix.c_str(), prefix)); + const char* prefix = "curvefs_kvclient_manager"; + ASSERT_EQ(0, ::strcmp(KVClientManagerMetric::prefix.c_str(), prefix)); } { diff --git a/curvefs/test/client/mock_kvclient.h b/curvefs/test/client/mock_kvclient.h index 2a1e6c9594..8ce35e08c8 100644 --- a/curvefs/test/client/mock_kvclient.h +++ b/curvefs/test/client/mock_kvclient.h @@ -24,7 +24,10 @@ #define CURVEFS_TEST_CLIENT_MOCK_KVCLIENT_H_ #include + +#include #include + #include "curvefs/src/client/kvclient/kvclient.h" namespace curvefs { @@ -36,8 +39,9 @@ class MockKVClient : public KVClient { MOCK_METHOD4(Set, bool(const std::string &, const char *, const uint64_t, std::string *)); - MOCK_METHOD5(Get, bool(const std::string &, char *, uint64_t, uint64_t, - std::string *)); + MOCK_METHOD7(Get, + bool(const std::string&, char*, uint64_t, uint64_t, + std::string*, uint64_t*, memcached_return_t* retCod)); }; } // namespace client From 59c07ec5ea50a8f54523a96b1488d4467b195ac4 Mon Sep 17 00:00:00 2001 From: Cyber-SiKu Date: Thu, 19 Oct 2023 18:02:02 +0800 Subject: [PATCH 02/27] [feat]curvefs/metaserver: add delete inode monitor Signed-off-by: Cyber-SiKu --- .../provisioning/dashboards/metaserver.json | 2319 +++++++++++------ .../src/client/kvclient/kvclient_manager.cpp | 3 +- curvefs/src/client/kvclient/memcache_client.h | 8 +- curvefs/src/client/metric/client_metric.cpp | 10 +- curvefs/src/client/metric/client_metric.h | 2 - curvefs/src/client/s3/client_s3_adaptor.cpp | 8 +- .../src/client/s3/client_s3_cache_manager.cpp | 34 +- curvefs/src/client/s3/disk_cache_manager.cpp | 5 +- .../src/client/s3/disk_cache_manager_impl.cpp | 5 +- curvefs/src/client/s3/disk_cache_write.cpp | 12 +- curvefs/src/client/warmup/warmup_manager.cpp | 6 +- .../metaserver/s3/metaserver_s3_adaptor.cpp | 26 +- .../src/metaserver/s3/metaserver_s3_adaptor.h | 16 +- src/client/client_metric.cpp | 34 + src/client/client_metric.h | 2 + 15 files changed, 1620 insertions(+), 870 deletions(-) create mode 100644 src/client/client_metric.cpp diff --git a/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json b/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json index c21f1bf870..a4395d2af3 100644 --- a/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json +++ b/curvefs/monitor/grafana/provisioning/dashboards/metaserver.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -20,543 +23,1019 @@ }, "description": "Curvefs mds server", "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 5, - "iteration": 1660902423620, + "id": 8, "links": [], + "liveNow": false, "panels": [ { - "collapsed": false, - "datasource": null, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 6, - "panels": [], - "title": "process usage", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 4, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "id": 70, + "panels": [ { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": { - "type": "prometheus" + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "exemplar": true, - "expr": "process_memory_resident{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "process_memory_resident {{hostname}}", - "refId": "process_memory_resident" - }, - { - "datasource": { - "type": "prometheus" - }, - "exemplar": true, - "expr": "process_memory_virtual{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_memory_virtual {{hostname}}", - "refId": "process_memory_virtual" - }, - { - "datasource": { - "type": "prometheus" + "fieldConfig": { + "defaults": { + "unit": "binBps" + }, + "overrides": [] }, - "exemplar": true, - "expr": "process_memory_shared{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_memory_shared {{hostname}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "process memory usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:63", - "format": "decbytes", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:64", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 2, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus" + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 }, - "exemplar": true, - "expr": "process_cpu_usage{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "process_cpu_usage {{hostname}}", - "refId": "process_cpu_usage" - }, - { - "datasource": { - "type": "prometheus" - }, - "exemplar": true, - "expr": "process_cpu_usage_system{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_cpu_usage_system {{hostname}}", - "refId": "process_cpu_usage_system" - }, - { - "datasource": { - "type": "prometheus" - }, - "exemplar": true, - "expr": "process_cpu_usage_user{instance=~\"$instance\"}", - "hide": false, - "interval": "", - "legendFormat": "process_cpu_usage_user {{hostname}}", - "refId": "process_cpu_usage_user" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "process cpu usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:495", - "format": "percentunit", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:496", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 14, - "panels": [], - "title": "metaserver config", - "type": "row" - }, - { - "datasource": null, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "hiddenSeries": false, + "id": 69, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - { - "color": "red", - "value": 80 - } - ] + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_s3_client_adaptor_delete_inode_bps\",job=\"metaserver\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "delete inode bps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "binBps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false } }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 16, - "interval": "1s", - "options": { - "showHeader": true - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "exemplar": true, - "expr": "{__name__=~\"curvefs_metaserver_config.*\", job=\"metaserver\", instance=~\"$instance\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "metaserver config", - "transformations": [ { - "id": "organize", + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 71, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", "options": { - "excludeByName": { - "Time": true, - "Value": true, - "__name__": true, - "hostname": true, - "job": true + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_s3_client_adaptor_delete_inode_qps\",job=\"metaserver\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "delete inode qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "short", + "logBase": 1, + "show": true }, - "indexByName": {}, - "renameByName": {} + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false } - } - ], - "type": "table" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 30, - "panels": [], - "title": "service total performance", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "unit": "qps" }, - "overrides": [] + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 72, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_s3_client_adaptor_delete_inode_lat_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "delete inode latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 73, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_s3_client_adaptor_delete_inode_eps\",job=\"metaserver\", instance=~\"$instance\"}", + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "delete inode eps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "s3 client adaptor", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 19 - }, - "hiddenSeries": false, - "id": 31, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "y": 1 }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "id": 6, + "panels": [ { - "exemplar": true, - "expr": "sum({__name__=~\".*curvefs_metaserver_meta_server_service.*_qps\",job=\"metaserver\", instance=~\"$instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 4, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_memory_resident{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "process_memory_resident {{hostname}}", + "refId": "process_memory_resident" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_memory_virtual{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_memory_virtual {{hostname}}", + "refId": "process_memory_virtual" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_memory_shared{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_memory_shared {{hostname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "process memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "decbytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_cpu_usage{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "process_cpu_usage {{hostname}}", + "refId": "process_cpu_usage" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_cpu_usage_system{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_cpu_usage_system {{hostname}}", + "refId": "process_cpu_usage_system" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_cpu_usage_user{instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "process_cpu_usage_user {{hostname}}", + "refId": "process_cpu_usage_user" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "process cpu usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:495", + "format": "percentunit", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:496", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "total_qps", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:63", - "format": "qps", - "logBase": 1, - "show": true - }, + "targets": [ { - "$$hashKey": "object:64", - "format": "short", - "logBase": 1, - "show": true + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" } ], - "yaxis": { - "align": false - } + "title": "process usage", + "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": { - "unit": "none" - }, - "overrides": [] + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 - }, - "hiddenSeries": false, - "id": 32, - "interval": "1s", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "h": 1, + "w": 24, + "x": 0, + "y": 2 }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "id": 14, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 16, + "interval": "1s", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "{__name__=~\"curvefs_metaserver_config.*\", job=\"metaserver\", instance=~\"$instance\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "metaserver config", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "hostname": true, + "job": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], "targets": [ { - "exemplar": true, - "expr": "sum({__name__=~\".*curvefs_metaserver_meta_server_service.*_error\",job=\"metaserver\", instance=~\"$instance\"}) by (instance)", - "interval": "", - "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "total_error_count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "title": "metaserver config", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 }, - "yaxes": [ + "id": 30, + "panels": [ { - "$$hashKey": "object:63", - "format": "none", - "logBase": 1, - "show": true + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "qps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 31, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum({__name__=~\".*curvefs_metaserver_meta_server_service.*_qps\",job=\"metaserver\", instance=~\"$instance\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "total_qps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "qps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { - "$$hashKey": "object:64", - "format": "short", - "logBase": 1, - "show": true + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 32, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "sum({__name__=~\".*curvefs_metaserver_meta_server_service.*_error\",job=\"metaserver\", instance=~\"$instance\"}) by (instance)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "total_error_count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:63", + "format": "none", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:64", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" } ], - "yaxis": { - "align": false - } + "title": "service total performance", + "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 4 }, "id": 8, "panels": [ @@ -565,7 +1044,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -578,7 +1060,7 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 21 }, "hiddenSeries": false, "id": 9, @@ -600,7 +1082,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -610,6 +1092,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -618,9 +1104,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "create_inode", "tooltip": { "shared": true, @@ -629,9 +1113,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -658,7 +1140,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -671,7 +1156,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 21 }, "hiddenSeries": false, "id": 10, @@ -693,7 +1178,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -703,6 +1188,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -711,9 +1200,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "create_dentry", "tooltip": { "shared": true, @@ -722,9 +1209,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -751,7 +1236,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -764,7 +1252,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 29 }, "hiddenSeries": false, "id": 17, @@ -786,7 +1274,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -796,6 +1284,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -804,9 +1296,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "delete_inode", "tooltip": { "shared": true, @@ -815,9 +1305,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -844,7 +1332,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -857,7 +1348,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 29 }, "hiddenSeries": false, "id": 18, @@ -879,7 +1370,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -889,6 +1380,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -897,9 +1392,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "delete_dentry", "tooltip": { "shared": true, @@ -908,9 +1401,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -937,7 +1428,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -950,7 +1444,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 37 }, "hiddenSeries": false, "id": 11, @@ -972,7 +1466,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -982,6 +1476,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -990,9 +1488,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "get_inode", "tooltip": { "shared": true, @@ -1001,9 +1497,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1030,7 +1524,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1043,7 +1540,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 37 }, "hiddenSeries": false, "id": 12, @@ -1065,7 +1562,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1075,6 +1572,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1083,9 +1584,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "get_dentry", "tooltip": { "shared": true, @@ -1094,9 +1593,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1123,7 +1620,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1136,7 +1636,7 @@ "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 45 }, "hiddenSeries": false, "id": 19, @@ -1158,7 +1658,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1168,6 +1668,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_batch_get_inode_attr\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1176,9 +1680,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "batch_get_inode_attr", "tooltip": { "shared": true, @@ -1187,9 +1689,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1216,7 +1716,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1229,7 +1732,7 @@ "h": 8, "w": 12, "x": 12, - "y": 52 + "y": 45 }, "hiddenSeries": false, "id": 20, @@ -1251,7 +1754,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1261,6 +1764,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_list_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1269,9 +1776,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "list_dentry", "tooltip": { "shared": true, @@ -1280,9 +1785,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1309,7 +1812,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1322,7 +1828,7 @@ "h": 8, "w": 12, "x": 0, - "y": 60 + "y": 53 }, "hiddenSeries": false, "id": 21, @@ -1344,7 +1850,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1354,6 +1860,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_update_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1362,9 +1872,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "update_inode", "tooltip": { "shared": true, @@ -1373,9 +1881,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1402,7 +1908,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1415,7 +1924,7 @@ "h": 8, "w": 12, "x": 12, - "y": 60 + "y": 53 }, "hiddenSeries": false, "id": 22, @@ -1437,7 +1946,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1447,6 +1956,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_batch_get_xattr\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1455,9 +1968,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "batch_get_xattr", "tooltip": { "shared": true, @@ -1466,9 +1977,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1495,7 +2004,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1508,7 +2020,7 @@ "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 61 }, "hiddenSeries": false, "id": 23, @@ -1530,7 +2042,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1540,6 +2052,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_prepare_rename_tx\", job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1548,9 +2064,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "prepare_rename_tx", "tooltip": { "shared": true, @@ -1559,9 +2073,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1588,7 +2100,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1601,7 +2116,7 @@ "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 61 }, "hiddenSeries": false, "id": 24, @@ -1623,7 +2138,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1633,6 +2148,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_or_modify_s3_chunk_info\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1641,9 +2160,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "get_or_modify_s3_chunk_info", "tooltip": { "shared": true, @@ -1652,9 +2169,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1681,7 +2196,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1694,7 +2212,7 @@ "h": 8, "w": 12, "x": 0, - "y": 76 + "y": 69 }, "hiddenSeries": false, "id": 25, @@ -1716,7 +2234,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1726,6 +2244,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_partition\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1734,9 +2256,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "create_partition", "tooltip": { "shared": true, @@ -1745,9 +2265,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1774,7 +2292,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1787,7 +2308,7 @@ "h": 8, "w": 12, "x": 12, - "y": 76 + "y": 69 }, "hiddenSeries": false, "id": 26, @@ -1809,7 +2330,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1819,6 +2340,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_partition\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1827,9 +2352,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "delete_partition", "tooltip": { "shared": true, @@ -1838,9 +2361,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1867,7 +2388,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1880,7 +2404,7 @@ "h": 8, "w": 12, "x": 0, - "y": 84 + "y": 77 }, "hiddenSeries": false, "id": 27, @@ -1902,7 +2426,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1912,6 +2436,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_volume_extent\",job=\"metaserver\", instance=~\"$instance\",quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -1920,9 +2448,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "get_volume_extent", "tooltip": { "shared": true, @@ -1931,9 +2457,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1960,7 +2484,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -1973,7 +2500,7 @@ "h": 8, "w": 12, "x": 12, - "y": 84 + "y": 77 }, "hiddenSeries": false, "id": 28, @@ -1995,7 +2522,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2005,6 +2532,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_update_volume_extent\",job=\"metaserver\", instance=~\"$instance\",quantile=~\"[[service_quantile]]\"}", "interval": "", @@ -2013,9 +2544,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "update_volume_extent", "tooltip": { "shared": true, @@ -2024,9 +2553,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2049,17 +2576,29 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "service latency", "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 5 }, "id": 34, "panels": [ @@ -2068,7 +2607,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "qps" @@ -2081,7 +2623,7 @@ "h": 8, "w": 8, "x": 0, - "y": 29 + "y": 13 }, "hiddenSeries": false, "id": 35, @@ -2103,7 +2645,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2113,6 +2655,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_qps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2121,9 +2667,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_qps", "tooltip": { "shared": true, @@ -2132,9 +2676,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2161,7 +2703,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "reqps" @@ -2174,7 +2719,7 @@ "h": 8, "w": 8, "x": 8, - "y": 29 + "y": 13 }, "hiddenSeries": false, "id": 36, @@ -2196,7 +2741,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2206,6 +2751,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_rps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2214,9 +2763,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_rps", "tooltip": { "shared": true, @@ -2225,9 +2772,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2254,7 +2799,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "eps" @@ -2267,7 +2815,7 @@ "h": 8, "w": 8, "x": 16, - "y": 29 + "y": 13 }, "hiddenSeries": false, "id": 37, @@ -2289,7 +2837,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2299,6 +2847,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_eps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2307,9 +2859,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_eps", "tooltip": { "shared": true, @@ -2318,9 +2868,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2347,7 +2895,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -2360,7 +2911,7 @@ "h": 8, "w": 8, "x": 0, - "y": 37 + "y": 21 }, "hiddenSeries": false, "id": 38, @@ -2382,7 +2933,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2392,6 +2943,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2400,9 +2955,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_apply_latency", "tooltip": { "shared": true, @@ -2411,9 +2964,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2440,7 +2991,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -2453,7 +3007,7 @@ "h": 8, "w": 8, "x": 8, - "y": 37 + "y": 21 }, "hiddenSeries": false, "id": 39, @@ -2475,7 +3029,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2485,6 +3039,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_execute_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2493,9 +3051,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_apply_execute_latency", "tooltip": { "shared": true, @@ -2504,9 +3060,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2533,7 +3087,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -2546,7 +3103,7 @@ "h": 8, "w": 8, "x": 16, - "y": 37 + "y": 21 }, "hiddenSeries": false, "id": 40, @@ -2568,7 +3125,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2578,6 +3135,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"op_apply_pool_[[pool:regex]]_copyset_[[copyset:regex]]_[[op:regex]]_wait_in_queue_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2586,9 +3147,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "op_apply_wait_in_queue_latency", "tooltip": { "shared": true, @@ -2597,9 +3156,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2622,17 +3179,29 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "raft op metric", "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 6 }, "id": 42, "panels": [ @@ -2641,7 +3210,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -2654,7 +3226,7 @@ "h": 8, "w": 12, "x": 0, - "y": 30 + "y": 14 }, "hiddenSeries": false, "id": 43, @@ -2676,7 +3248,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2686,6 +3258,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"copyset_snapshot_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2694,9 +3270,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "copyset_snapshot_latency", "tooltip": { "shared": true, @@ -2705,9 +3279,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2734,7 +3306,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -2747,7 +3322,7 @@ "h": 8, "w": 12, "x": 12, - "y": 30 + "y": 14 }, "hiddenSeries": false, "id": 44, @@ -2769,7 +3344,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2779,6 +3354,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"copyset_snapshot_error_count\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2787,9 +3366,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "copyset_snapshot_error_count", "tooltip": { "shared": true, @@ -2798,9 +3375,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2827,7 +3402,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -2840,7 +3418,7 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 22 }, "hiddenSeries": false, "id": 45, @@ -2862,7 +3440,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2872,6 +3450,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"copyset_snapshot_flying_count\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2880,9 +3462,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "copyset_snapshot_flying_count", "tooltip": { "shared": true, @@ -2891,9 +3471,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -2916,17 +3494,29 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "raft snapshot metric", "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 7 }, "id": 47, "panels": [ @@ -2935,7 +3525,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -2948,7 +3541,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 15 }, "hiddenSeries": false, "id": 48, @@ -2970,7 +3563,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2980,6 +3573,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_flushing\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -2988,9 +3585,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_flushing", "tooltip": { "shared": true, @@ -2999,9 +3594,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3028,7 +3621,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -3041,7 +3637,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 15 }, "hiddenSeries": false, "id": 49, @@ -3063,7 +3659,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3073,6 +3669,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_flush_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3081,9 +3681,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_flush_latency", "tooltip": { "shared": true, @@ -3092,9 +3690,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3121,7 +3717,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3134,7 +3733,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 23 }, "hiddenSeries": false, "id": 50, @@ -3156,7 +3755,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3166,6 +3765,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_compacting\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3174,9 +3777,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_compacting", "tooltip": { "shared": true, @@ -3185,9 +3786,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3214,7 +3813,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -3227,7 +3829,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 23 }, "hiddenSeries": false, "id": 51, @@ -3249,7 +3851,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3259,6 +3861,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_compaction_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3267,9 +3873,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_compaction_latency", "tooltip": { "shared": true, @@ -3278,9 +3882,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3307,7 +3909,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3320,7 +3925,7 @@ "h": 8, "w": 8, "x": 0, - "y": 47 + "y": 31 }, "hiddenSeries": false, "id": 52, @@ -3342,7 +3947,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3352,6 +3957,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_sealed_memtable\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3360,9 +3969,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_sealed_memtable", "tooltip": { "shared": true, @@ -3371,9 +3978,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3400,7 +4005,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3413,7 +4021,7 @@ "h": 8, "w": 8, "x": 8, - "y": 47 + "y": 31 }, "hiddenSeries": false, "id": 53, @@ -3435,7 +4043,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3445,6 +4053,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_delayed_write\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3453,9 +4065,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_delayed_write", "tooltip": { "shared": true, @@ -3464,9 +4074,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3493,7 +4101,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3506,7 +4117,7 @@ "h": 8, "w": 8, "x": 16, - "y": 47 + "y": 31 }, "hiddenSeries": false, "id": 54, @@ -3528,7 +4139,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3538,6 +4149,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"rocksdb_stopped_write\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3546,9 +4161,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "rocksdb_stopped_write", "tooltip": { "shared": true, @@ -3557,9 +4170,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3582,17 +4193,29 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "rocksdb metric", "type": "row" }, { "collapsed": true, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 8 }, "id": 56, "panels": [ @@ -3601,7 +4224,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3614,7 +4240,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 16 }, "hiddenSeries": false, "id": 57, @@ -3636,7 +4262,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3646,6 +4272,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_apply_tasks_batch_avg_counter\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3653,6 +4283,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_apply_tasks_batch_counter_.*\",job=\"metaserver\", instance=~\"$instance\"}", "hide": false, @@ -3662,9 +4296,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_apply_tasks_batch_counter", "tooltip": { "shared": true, @@ -3673,9 +4305,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3702,7 +4332,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -3715,7 +4348,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 16 }, "hiddenSeries": false, "id": 59, @@ -3737,7 +4370,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3747,6 +4380,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_send_entries_batch_avg_counter\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3754,6 +4391,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_send_entries_batch_counter_.*\",job=\"metaserver\", instance=~\"$instance\"}", "hide": false, @@ -3763,9 +4404,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_send_entries_batch_counter", "tooltip": { "shared": true, @@ -3774,9 +4413,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3803,7 +4440,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "qps" @@ -3816,7 +4456,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 24 }, "hiddenSeries": false, "id": 61, @@ -3838,7 +4478,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3848,6 +4488,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_send_entries_qps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3856,9 +4500,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_send_entries_qps", "tooltip": { "shared": true, @@ -3867,9 +4509,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3896,7 +4536,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -3909,7 +4552,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 24 }, "hiddenSeries": false, "id": 60, @@ -3931,7 +4574,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3941,6 +4584,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_send_entries_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -3949,9 +4596,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_send_entries_latency", "tooltip": { "shared": true, @@ -3960,9 +4605,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -3989,7 +4632,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "qps" @@ -4002,7 +4648,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 32 }, "hiddenSeries": false, "id": 62, @@ -4024,7 +4670,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4034,6 +4680,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*raft_service_append_entries_qps\",job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4042,9 +4692,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_service_append_entries_qps", "tooltip": { "shared": true, @@ -4053,9 +4701,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4082,7 +4728,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -4095,7 +4744,7 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 32 }, "hiddenSeries": false, "id": 63, @@ -4117,7 +4766,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4127,6 +4776,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*raft_service_append_entries\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4135,9 +4788,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_service_append_entries_latency", "tooltip": { "shared": true, @@ -4146,9 +4797,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4175,7 +4824,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -4188,7 +4840,7 @@ "h": 8, "w": 12, "x": 0, - "y": 56 + "y": 40 }, "hiddenSeries": false, "id": 64, @@ -4210,7 +4862,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4220,6 +4872,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*raft_storage_append_entries_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4228,9 +4884,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_storage_append_entries_latency", "tooltip": { "shared": true, @@ -4239,9 +4893,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4268,7 +4920,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "µs" @@ -4281,7 +4936,7 @@ "h": 8, "w": 12, "x": 12, - "y": 56 + "y": 40 }, "hiddenSeries": false, "id": 65, @@ -4303,7 +4958,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4313,6 +4968,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_storage_append_entries_normalized_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4321,9 +4980,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_storage_append_entries_normalized_latency", "tooltip": { "shared": true, @@ -4332,9 +4989,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4361,7 +5016,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -4374,7 +5032,7 @@ "h": 8, "w": 12, "x": 0, - "y": 64 + "y": 48 }, "hiddenSeries": false, "id": 66, @@ -4396,7 +5054,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4406,6 +5064,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_storage_flush_batch_avg_counter\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4413,6 +5075,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_storage_flush_batch_counter_[[quantile:regex]]\", job=\"metaserver\", instance=~\"$instance\"}", "hide": false, @@ -4422,9 +5088,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_storage_flush_batch_counter", "tooltip": { "shared": true, @@ -4433,9 +5097,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4462,7 +5124,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "short" @@ -4475,7 +5140,7 @@ "h": 8, "w": 12, "x": 12, - "y": 64 + "y": 48 }, "hiddenSeries": false, "id": 67, @@ -4497,7 +5162,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4507,6 +5172,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\"raft_num_log_entries\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4515,9 +5184,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_num_log_entries", "tooltip": { "shared": true, @@ -4526,9 +5193,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4555,7 +5220,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "qps" @@ -4568,7 +5236,7 @@ "h": 8, "w": 12, "x": 0, - "y": 72 + "y": 56 }, "hiddenSeries": false, "id": 68, @@ -4600,6 +5268,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "exemplar": true, "expr": "{__name__=~\".*raft_service_install_snapshot_qps\", job=\"metaserver\", instance=~\"$instance\"}", "interval": "", @@ -4608,9 +5280,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "raft_service_install_snapshot_qps", "tooltip": { "shared": true, @@ -4619,9 +5289,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -4644,20 +5312,28 @@ } } ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], "title": "raft builtin metric", "type": "row" } ], - "refresh": "", - "schemaVersion": 30, + "refresh": "5s", + "schemaVersion": 38, "style": "dark", "tags": [], "templating": { "list": [ { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -4665,10 +5341,11 @@ "$__all" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\"bthread_count\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "Addr", @@ -4686,9 +5363,8 @@ "type": "query" }, { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -4696,10 +5372,11 @@ "$__all" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\"op_apply_pool_.*_copyset.*create_inode_count\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "pool", @@ -4717,9 +5394,8 @@ "type": "query" }, { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -4727,10 +5403,11 @@ "$__all" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\".*copyset_.*_create_inode_count\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "copyset", @@ -4750,7 +5427,7 @@ { "allValue": "", "current": { - "selected": false, + "selected": true, "text": [ "latency" ], @@ -4758,10 +5435,11 @@ "latency" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\".*(latency_[0-9]*|latency)\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "latency_quantile", @@ -4779,9 +5457,8 @@ "type": "query" }, { - "allValue": null, "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -4789,10 +5466,11 @@ "$__all" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\".*curvefs_metaserver_meta_server_service_.*_qps\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "op", @@ -4812,7 +5490,7 @@ { "allValue": "", "current": { - "selected": false, + "selected": true, "text": [ "1" ], @@ -4820,10 +5498,11 @@ "1" ] }, - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "{__name__=~\".*curvefs_metaserver_meta_server_service.*\", job=\"metaserver\"}", - "description": null, - "error": null, "hide": 0, "includeAll": true, "label": "service_quantile", @@ -4851,4 +5530,4 @@ "title": "metaserver", "uid": "d12ivuiVz", "version": 8 -} +} \ No newline at end of file diff --git a/curvefs/src/client/kvclient/kvclient_manager.cpp b/curvefs/src/client/kvclient/kvclient_manager.cpp index 4ce8f6bd78..a808580c8d 100644 --- a/curvefs/src/client/kvclient/kvclient_manager.cpp +++ b/curvefs/src/client/kvclient/kvclient_manager.cpp @@ -40,7 +40,8 @@ template void OnReturn(Metric* metric, const TaskSharePtr task) { task->timer.stop(); if (task->res) { - metric::CollectMetrics(metric, task->length, task->timer.u_elapsed()); + curve::client::CollectMetrics(metric, task->length, + task->timer.u_elapsed()); } else { metric->eps.count << 1; } diff --git a/curvefs/src/client/kvclient/memcache_client.h b/curvefs/src/client/kvclient/memcache_client.h index ab4e939a74..57e82a7f44 100644 --- a/curvefs/src/client/kvclient/memcache_client.h +++ b/curvefs/src/client/kvclient/memcache_client.h @@ -122,8 +122,8 @@ class MemCachedClient : public KVClient { value_len, 0, 0); if (MEMCACHED_SUCCESS == res) { VLOG(9) << "Set key = " << key << " OK"; - metric::CollectMetrics(&metric_->set, value_len, - butil::cpuwide_time_us() - start); + curve::client::CollectMetrics(&metric_->set, value_len, + butil::cpuwide_time_us() - start); return true; } *errorlog = ResError(res); @@ -159,8 +159,8 @@ class MemCachedClient : public KVClient { VLOG(9) << "Get key = " << key << " OK"; memcpy(value, res + offset, length); free(res); - metric::CollectMetrics(&metric_->get, value_length, - butil::cpuwide_time_us() - start); + curve::client::CollectMetrics(&metric_->get, value_length, + butil::cpuwide_time_us() - start); return true; } diff --git a/curvefs/src/client/metric/client_metric.cpp b/curvefs/src/client/metric/client_metric.cpp index 8e9939413d..f97e7ef87a 100644 --- a/curvefs/src/client/metric/client_metric.cpp +++ b/curvefs/src/client/metric/client_metric.cpp @@ -24,10 +24,14 @@ #include +#include "src/client/client_metric.h" + namespace curvefs { namespace client { namespace metric { +using curve::client::CollectMetrics; + const std::string MDSClientMetric::prefix = "curvefs_mds_client"; // NOLINT const std::string MetaServerClientMetric::prefix = "curvefs_metaserver_client"; // NOLINT const std::string ClientOpMetric::prefix = "curvefs_client"; // NOLINT @@ -42,12 +46,6 @@ const std::string MemcacheClientMetric::prefix = // NOLINT const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info"; // NOLINT const std::string WarmupManagerS3Metric::prefix = "curvefs_warmup"; // NOLINT -void CollectMetrics(InterfaceMetric* interface, int count, uint64_t u_elapsed) { - interface->bps.count << count; - interface->qps.count << 1; - interface->latency << u_elapsed; -} - void AsyncContextCollectMetrics( std::shared_ptr s3Metric, const std::shared_ptr& context) { diff --git a/curvefs/src/client/metric/client_metric.h b/curvefs/src/client/metric/client_metric.h index b8a7896f1f..e5819cfa8a 100644 --- a/curvefs/src/client/metric/client_metric.h +++ b/curvefs/src/client/metric/client_metric.h @@ -360,8 +360,6 @@ struct WarmupManagerS3Metric { warmupS3CacheSize(prefix, "s3_cache_size") {} }; -void CollectMetrics(InterfaceMetric* interface, int count, uint64_t u_elapsed); - void AsyncContextCollectMetrics( std::shared_ptr s3Metric, const std::shared_ptr& context); diff --git a/curvefs/src/client/s3/client_s3_adaptor.cpp b/curvefs/src/client/s3/client_s3_adaptor.cpp index d6ce3a8bc9..85d94c874e 100644 --- a/curvefs/src/client/s3/client_s3_adaptor.cpp +++ b/curvefs/src/client/s3/client_s3_adaptor.cpp @@ -145,8 +145,8 @@ int S3ClientAdaptorImpl::Write(uint64_t inodeId, uint64_t offset, int ret = fileCacheManager->Write(offset, length, buf); fsCacheManager_->DataCacheByteDec(length); if (s3Metric_ != nullptr) { - metric::CollectMetrics(&s3Metric_->adaptorWrite, ret, - butil::cpuwide_time_us() - start); + curve::client::CollectMetrics(&s3Metric_->adaptorWrite, ret, + butil::cpuwide_time_us() - start); s3Metric_->writeSize.set_value(length); } VLOG(6) << "write end inodeId: " << inodeId << ", ret: " << ret; @@ -167,8 +167,8 @@ int S3ClientAdaptorImpl::Read(uint64_t inodeId, uint64_t offset, return ret; } if (s3Metric_.get() != nullptr) { - metric::CollectMetrics(&s3Metric_->adaptorRead, ret, - butil::cpuwide_time_us() - start); + curve::client::CollectMetrics(&s3Metric_->adaptorRead, ret, + butil::cpuwide_time_us() - start); s3Metric_->readSize.set_value(length); } VLOG(6) << "read end offset:" << offset << ", len:" << length diff --git a/curvefs/src/client/s3/client_s3_cache_manager.cpp b/curvefs/src/client/s3/client_s3_cache_manager.cpp index 4eab99161e..1532a9282e 100644 --- a/curvefs/src/client/s3/client_s3_cache_manager.cpp +++ b/curvefs/src/client/s3/client_s3_cache_manager.cpp @@ -496,10 +496,12 @@ bool FileCacheManager::ReadKVRequestFromLocalCache(const std::string &name, } if (s3ClientAdaptor_->s3Metric_) { - metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->adaptorReadS3, len, - butil::cpuwide_time_us() - start); - metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->readFromDiskCache, - len, butil::cpuwide_time_us() - start); + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->adaptorReadS3, len, + butil::cpuwide_time_us() - start); + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->readFromDiskCache, len, + butil::cpuwide_time_us() - start); } return true; } @@ -515,7 +517,7 @@ bool FileCacheManager::ReadKVRequestFromRemoteCache(const std::string &name, CountDownEvent event(1); GetKVCacheDone cb = [&](const std::shared_ptr& task) { if (task->res && s3ClientAdaptor_->s3Metric_ != nullptr) { - metric::CollectMetrics( + curve::client::CollectMetrics( &s3ClientAdaptor_->s3Metric_->readFromKVCache, task->length, task->timer.u_elapsed()); } @@ -543,10 +545,11 @@ bool FileCacheManager::ReadKVRequestFromS3(const std::string &name, } if (s3ClientAdaptor_->s3Metric_) { - metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->adaptorReadS3, - length, butil::cpuwide_time_us() - start); - metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->readFromS3, length, - butil::cpuwide_time_us() - start); + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->adaptorReadS3, length, + butil::cpuwide_time_us() - start); + curve::client::CollectMetrics(&s3ClientAdaptor_->s3Metric_->readFromS3, + length, butil::cpuwide_time_us() - start); } return true; @@ -736,9 +739,9 @@ class AsyncPrefetchCallback { } if (s3Client_->s3Metric_ != nullptr) { // prefetch to disk - metric::CollectMetrics(&s3Client_->s3Metric_->writeToDiskCache, - context->actualLen, - butil::cpuwide_time_us() - start); + curve::client::CollectMetrics( + &s3Client_->s3Metric_->writeToDiskCache, context->actualLen, + butil::cpuwide_time_us() - start); } { curve::common::LockGuard lg(fileCache->downloadMtx_); @@ -2369,7 +2372,7 @@ CURVEFS_ERROR DataCache::PrepareFlushTasks( [this](const std::shared_ptr& setTask) { if (setTask->res && s3ClientAdaptor_->s3Metric_ != nullptr) { - metric::CollectMetrics( + curve::client::CollectMetrics( &s3ClientAdaptor_->s3Metric_->writeToKVCache, setTask->length, setTask->timer.u_elapsed()); } @@ -2448,8 +2451,9 @@ void DataCache::FlushTaskExecute( SetKVCacheDone kvdone = [&](const std::shared_ptr &task) { kvTaskEvent.Signal(); if (task->res && s3ClientAdaptor_->s3Metric_ != nullptr) { - metric::CollectMetrics(&s3ClientAdaptor_->s3Metric_->writeToKVCache, - task->length, task->timer.u_elapsed()); + curve::client::CollectMetrics( + &s3ClientAdaptor_->s3Metric_->writeToKVCache, task->length, + task->timer.u_elapsed()); } return; }; diff --git a/curvefs/src/client/s3/disk_cache_manager.cpp b/curvefs/src/client/s3/disk_cache_manager.cpp index e394643489..1bc2ac2473 100644 --- a/curvefs/src/client/s3/disk_cache_manager.cpp +++ b/curvefs/src/client/s3/disk_cache_manager.cpp @@ -475,8 +475,9 @@ void DiskCacheManager::TrimCache() { << "error is: " << errno; continue; } - metric::CollectMetrics(&metric_->trim_, statReadFile.st_size, - butil::cpuwide_time_us() - start); + curve::client::CollectMetrics(&metric_->trim_, + statReadFile.st_size, + butil::cpuwide_time_us() - start); UpdateDiskUsedBytes(-statReadFile.st_size); VLOG(6) << "remove disk file success, file is: " << cacheKey; } diff --git a/curvefs/src/client/s3/disk_cache_manager_impl.cpp b/curvefs/src/client/s3/disk_cache_manager_impl.cpp index f99a70e33e..df080f3c5c 100644 --- a/curvefs/src/client/s3/disk_cache_manager_impl.cpp +++ b/curvefs/src/client/s3/disk_cache_manager_impl.cpp @@ -183,8 +183,9 @@ int DiskCacheManagerImpl::Read(const std::string name, char *buf, LOG(ERROR) << "download object fail. object name = " << name; return ret; } - metric::CollectMetrics(&diskCacheManager_->GetS3Metric()->readFromS3, - length, butil::cpuwide_time_us() - start); + curve::client::CollectMetrics( + &diskCacheManager_->GetS3Metric()->readFromS3, length, + butil::cpuwide_time_us() - start); } VLOG(9) << "read success, read name = " << name; return ret; diff --git a/curvefs/src/client/s3/disk_cache_write.cpp b/curvefs/src/client/s3/disk_cache_write.cpp index 5b650bfa83..640c6bca0a 100644 --- a/curvefs/src/client/s3/disk_cache_write.cpp +++ b/curvefs/src/client/s3/disk_cache_write.cpp @@ -143,9 +143,9 @@ int DiskCacheWrite::UploadFile(const std::string &name, name](const std::shared_ptr& context) { if (context->retCode >= 0) { if (metric_ != nullptr) { - metric::CollectMetrics(&metric_->writeS3, - context->bufferSize, - context->timer.u_elapsed()); + curve::client::CollectMetrics(&metric_->writeS3, + context->bufferSize, + context->timer.u_elapsed()); } if (s3Metric_ != nullptr) { metric::AsyncContextCollectMetrics(s3Metric_, context); @@ -403,9 +403,9 @@ int DiskCacheWrite::UploadAllCacheWriteFile() { [&, buffer](const std::shared_ptr& context) { if (context->retCode >= 0) { if (s3Metric_ != nullptr) { - metric::CollectMetrics(&metric_->writeS3, - context->bufferSize, - context->timer.u_elapsed()); + curve::client::CollectMetrics( + &metric_->writeS3, context->bufferSize, + context->timer.u_elapsed()); metric::AsyncContextCollectMetrics(s3Metric_, context); } if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == diff --git a/curvefs/src/client/warmup/warmup_manager.cpp b/curvefs/src/client/warmup/warmup_manager.cpp index 4ffd9d13bd..77182d7bdc 100644 --- a/curvefs/src/client/warmup/warmup_manager.cpp +++ b/curvefs/src/client/warmup/warmup_manager.cpp @@ -536,9 +536,9 @@ void WarmupManagerS3Impl::WarmUpAllObjs( if (context->retCode >= 0) { VLOG(9) << "Get Object success: " << context->key; PutObjectToCache(key, context); - metric::CollectMetrics(&warmupS3Metric_.warmupS3Cached, - context->len, - butil::cpuwide_time_us() - start); + curve::client::CollectMetrics(&warmupS3Metric_.warmupS3Cached, + context->len, + butil::cpuwide_time_us() - start); warmupS3Metric_.warmupS3CacheSize << context->len; if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) { VLOG(6) << "pendingReq is over"; diff --git a/curvefs/src/metaserver/s3/metaserver_s3_adaptor.cpp b/curvefs/src/metaserver/s3/metaserver_s3_adaptor.cpp index b6c42068a8..94a496be8e 100644 --- a/curvefs/src/metaserver/s3/metaserver_s3_adaptor.cpp +++ b/curvefs/src/metaserver/s3/metaserver_s3_adaptor.cpp @@ -21,12 +21,21 @@ */ #include "curvefs/src/metaserver/s3/metaserver_s3_adaptor.h" -#include + #include +#include +#include +#include + #include "curvefs/src/common/s3util.h" +#include "src/client/client_metric.h" namespace curvefs { namespace metaserver { + +const std::string S3ClientAdaptorMetric::prefix = // NOLINT + "curvefs_metaserver_s3_client_adaptor"; // NOLINT + void S3ClientAdaptorImpl::Init(const S3ClientAdaptorOption &option, S3Client *client) { blockSize_ = option.blockSize; @@ -48,12 +57,21 @@ void S3ClientAdaptorImpl::Reinit(const S3ClientAdaptorOption& option, client_->Reinit(ak, sk, endpoint, bucketName); } -int S3ClientAdaptorImpl::Delete(const Inode &inode) { +int S3ClientAdaptorImpl::Delete(const Inode& inode) { + int ret = 0; + uint64_t start = butil::cpuwide_time_us(); if (enableBatchDelete_) { - return DeleteInodeByDeleteBatchChunk(inode); + ret = DeleteInodeByDeleteBatchChunk(inode); } else { - return DeleteInodeByDeleteSingleChunk(inode); + ret = DeleteInodeByDeleteSingleChunk(inode); } + if (ret == 0) { + curve::client::CollectMetrics(&metric_.deleteInode, inode.length(), + butil::cpuwide_time_us() - start); + } else { + metric_.deleteInode.eps.count << 1; + } + return ret; } int S3ClientAdaptorImpl::DeleteInodeByDeleteSingleChunk(const Inode &inode) { diff --git a/curvefs/src/metaserver/s3/metaserver_s3_adaptor.h b/curvefs/src/metaserver/s3/metaserver_s3_adaptor.h index 233887fb9c..6330373afe 100644 --- a/curvefs/src/metaserver/s3/metaserver_s3_adaptor.h +++ b/curvefs/src/metaserver/s3/metaserver_s3_adaptor.h @@ -23,10 +23,14 @@ #ifndef CURVEFS_SRC_METASERVER_S3_METASERVER_S3_ADAPTOR_H_ #define CURVEFS_SRC_METASERVER_S3_METASERVER_S3_ADAPTOR_H_ -#include #include +#include +#include + +#include "absl/memory/memory.h" #include "curvefs/proto/metaserver.pb.h" #include "curvefs/src/metaserver/s3/metaserver_s3.h" +#include "src/client/client_metric.h" namespace curvefs { namespace metaserver { @@ -78,9 +82,18 @@ class S3ClientAdaptor { virtual void GetS3ClientAdaptorOption(S3ClientAdaptorOption *option) = 0; }; +struct S3ClientAdaptorMetric { + static const std::string prefix; + + curve::client::InterfaceMetric deleteInode; + + S3ClientAdaptorMetric() : deleteInode(prefix, "_delete_inode") {} +}; + class S3ClientAdaptorImpl : public S3ClientAdaptor { public: S3ClientAdaptorImpl() {} + ~S3ClientAdaptorImpl() { if (client_ != nullptr) { delete client_; @@ -151,6 +164,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { uint64_t batchSize_; uint32_t objectPrefix_; bool enableBatchDelete_; + S3ClientAdaptorMetric metric_; }; } // namespace metaserver } // namespace curvefs diff --git a/src/client/client_metric.cpp b/src/client/client_metric.cpp new file mode 100644 index 0000000000..8435da33ad --- /dev/null +++ b/src/client/client_metric.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Project: curve + * Created Date: 2023-10-19 + * Author: chengyi01 + */ + +#include "src/client/client_metric.h" + +namespace curve { +namespace client { + +void CollectMetrics(InterfaceMetric* interface, int count, uint64_t u_elapsed) { + interface->bps.count << count; + interface->qps.count << 1; + interface->latency << u_elapsed; +} + +} // namespace client +} // namespace curve diff --git a/src/client/client_metric.h b/src/client/client_metric.h index ba841e8b80..826b8b9b2d 100644 --- a/src/client/client_metric.h +++ b/src/client/client_metric.h @@ -87,6 +87,8 @@ struct InterfaceMetric { latency(prefix, name + "_lat") {} }; +void CollectMetrics(InterfaceMetric* interface, int count, uint64_t u_elapsed); + struct DiscardMetric { explicit DiscardMetric(const std::string& prefix) : totalSuccess(prefix, "discard_total_success"), From 45eca24fbec42183f6b1fb27925590ec5b38b56b Mon Sep 17 00:00:00 2001 From: NopeDl Date: Sat, 14 Oct 2023 02:37:13 +0800 Subject: [PATCH 03/27] [fix]doc:sfix error word Signed-off-by: NopeDl --- docs/cn/clangd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/clangd.md b/docs/cn/clangd.md index bf3976a474..2280caf115 100644 --- a/docs/cn/clangd.md +++ b/docs/cn/clangd.md @@ -1,6 +1,6 @@ ## 生成compile_commands.json -1. 清楚之前的编译缓存，`bazel clean` +1. 清除之前的编译缓存，`bazel clean` 2. 编译整个项目，`bazel build //...` 3. 生成compile_commands.json，`bazel run @hedron_compile_commands//:refresh_all` (更多使用方式可以参考https://github.com/hedronvision/bazel-compile-commands-extractor) From e738ee1068a63ae5c107a1bb3ecd3f0255562a9c Mon Sep 17 00:00:00 2001 From: Xinlong Chen Date: Sat, 21 Oct 2023 09:52:27 +0000 Subject: [PATCH 04/27] [refactor] CurveBS: add some common functions Signed-off-by: Xinlong Chen --- .../concurrent_apply/concurrent_apply.cpp | 32 ++++++++++++++++--- .../concurrent_apply/concurrent_apply.h | 3 +- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/chunkserver/concurrent_apply/concurrent_apply.cpp b/src/chunkserver/concurrent_apply/concurrent_apply.cpp index dd7b219502..48d7130681 100644 --- a/src/chunkserver/concurrent_apply/concurrent_apply.cpp +++ b/src/chunkserver/concurrent_apply/concurrent_apply.cpp @@ -124,8 +124,11 @@ void ConcurrentApplyModule::Run(ApplyTaskType type, int index) { } void ConcurrentApplyModule::Stop() { + if (!start_.exchange(false)) { + return; + } + LOG(INFO) << "stop ConcurrentApplyModule..."; - start_ = false; auto wakeup = []() {}; for (auto iter : rapplyMap_) { iter.second->tq.Push(wakeup); @@ -145,15 +148,36 @@ void ConcurrentApplyModule::Stop() { } void ConcurrentApplyModule::Flush() { + if (!start_.load(std::memory_order_relaxed)) { + return; + } + CountDownEvent event(wconcurrentsize_); - auto flushtask = [&event]() { - event.Signal(); - }; + auto flushtask = [&event]() { event.Signal(); }; + + for (int i = 0; i < wconcurrentsize_; i++) { + wapplyMap_[i]->tq.Push(flushtask); + } + + event.Wait(); +} + +void ConcurrentApplyModule::FlushAll() { + if (!start_.load(std::memory_order_relaxed)) { + return; + } + + CountDownEvent event(wconcurrentsize_ + rconcurrentsize_); + auto flushtask = [&event]() { event.Signal(); }; for (int i = 0; i < wconcurrentsize_; i++) { wapplyMap_[i]->tq.Push(flushtask); } + for (int i = 0; i < rconcurrentsize_; i++) { + rapplyMap_[i]->tq.Push(flushtask); + } + event.Wait(); } diff --git a/src/chunkserver/concurrent_apply/concurrent_apply.h b/src/chunkserver/concurrent_apply/concurrent_apply.h index a4e9712348..60b10a3e55 100644 --- a/src/chunkserver/concurrent_apply/concurrent_apply.h +++ b/src/chunkserver/concurrent_apply/concurrent_apply.h @@ -107,6 +107,7 @@ class CURVE_CACHELINE_ALIGNMENT ConcurrentApplyModule { * Flush: finish all task in write threads */ void Flush(); + void FlushAll(); void Stop(); @@ -128,7 +129,7 @@ class CURVE_CACHELINE_ALIGNMENT ConcurrentApplyModule { explicit TaskThread(size_t capacity) : tq(capacity) {} }; - bool start_; + std::atomic start_; int rconcurrentsize_; int rqueuedepth_; int wconcurrentsize_; From 1341a58393ee9c7296f4ecb9db1df80b24ca13d8 Mon Sep 17 00:00:00 2001 From: Cyber-SiKu Date: Tue, 17 Oct 2023 11:26:20 +0800 Subject: [PATCH 05/27] [feat]curvefs: change some conf on fly 1. use gflag to change mds heartbeatMissTimeOutMs on fly 2. use gflag to change metaserver transh.expiredAfterSec on fly 3. use gflag to change metaserver transh.scanPeriodSec on fly 4. use gflag to change client fs.disableXattr on fly 5. set client s3.maxAsyncRequestInflightBytes default vale to 1GiB Signed-off-by: Cyber-SiKu --- curvefs/conf/client.conf | 4 +-- curvefs/src/client/common/config.cpp | 3 ++ curvefs/src/client/fuse_client.cpp | 5 +-- .../heartbeat/metaserver_healthy_checker.cpp | 21 ++++++++++- .../heartbeat/metaserver_healthy_checker.h | 36 ++++++++++++++----- curvefs/src/metaserver/trash.cpp | 15 ++++++-- curvefs/src/metaserver/trash_manager.cpp | 8 +++-- .../test/client/filesystem/helper/builder.h | 5 ++- 8 files changed, 76 insertions(+), 21 deletions(-) diff --git a/curvefs/conf/client.conf b/curvefs/conf/client.conf index c66477926c..339a08a0d2 100644 --- a/curvefs/conf/client.conf +++ b/curvefs/conf/client.conf @@ -92,8 +92,6 @@ fuseClient.enableMultiMountPointRename=true fuseClient.enableSplice=false # thread number of listDentry when get summary xattr fuseClient.listDentryThreads=10 -# disable xattr on one mountpoint can fast 'ls -l' -fuseClient.disableXattr=false # default data（s3ChunkInfo/volumeExtent） size in inode, if exceed will eliminate and try to get the merged one fuseClient.maxDataSize=1024 # default refresh data interval 30s @@ -217,7 +215,7 @@ s3.logLevel=4 s3.logPrefix=/data/logs/curvefs/aws_ # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__ s3.asyncThreadNum=500 # limit all inflight async requests' bytes, |0| means not limited -s3.maxAsyncRequestInflightBytes=104857600 +s3.maxAsyncRequestInflightBytes=1073741824 s3.chunkFlushThreads=5 # throttle s3.throttle.iopsTotalLimit=0 diff --git a/curvefs/src/client/common/config.cpp b/curvefs/src/client/common/config.cpp index 10d64bea9a..0b987a2cac 100644 --- a/curvefs/src/client/common/config.cpp +++ b/curvefs/src/client/common/config.cpp @@ -39,6 +39,8 @@ namespace curvefs { namespace client { namespace common { DECLARE_bool(useFakeS3); +DEFINE_bool(fs_disableXattr, false, "disable xattr"); +DEFINE_validator(fs_disableXattr, [](const char*, bool value) { return true; }); } // namespace common } // namespace client } // namespace curvefs @@ -303,6 +305,7 @@ void InitFileSystemOption(Configuration* c, FileSystemOption* option) { c->GetValueFatalIfFail("fs.cto", &option->cto); c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto); c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr); + FLAGS_fs_disableXattr = option->disableXattr; c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength); c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging); { // kernel cache option diff --git a/curvefs/src/client/fuse_client.cpp b/curvefs/src/client/fuse_client.cpp index 3dfea2b7c4..0fe8d192da 100644 --- a/curvefs/src/client/fuse_client.cpp +++ b/curvefs/src/client/fuse_client.cpp @@ -88,6 +88,7 @@ DECLARE_uint64(fuseClientBurstReadIopsSecs); DECLARE_uint64(fuseClientAvgReadBytes); DECLARE_uint64(fuseClientBurstReadBytes); DECLARE_uint64(fuseClientBurstReadBytesSecs); +DECLARE_bool(fs_disableXattr); } // namespace common } // namespace client } // namespace curvefs @@ -1043,8 +1044,8 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino, (void)req; VLOG(9) << "FuseOpGetXattr, ino: " << ino << ", name: " << name << ", size = " << size; - if (option_.fileSystemOption.disableXattr) { - return CURVEFS_ERROR::NOSYS; + if (common::FLAGS_fs_disableXattr) { + return CURVEFS_ERROR::NODATA; } InodeAttr inodeAttr; diff --git a/curvefs/src/mds/heartbeat/metaserver_healthy_checker.cpp b/curvefs/src/mds/heartbeat/metaserver_healthy_checker.cpp index 228fba18c8..eb7cd75cba 100644 --- a/curvefs/src/mds/heartbeat/metaserver_healthy_checker.cpp +++ b/curvefs/src/mds/heartbeat/metaserver_healthy_checker.cpp @@ -30,6 +30,24 @@ using ::curvefs::mds::topology::TopoStatusCode; using ::curve::common::WriteLockGuard; using std::chrono::milliseconds; +// the maximun peroid that heartbeat is missed without +// setting the metaserver to offline status and alarm. +// scheduling will depend on this status of metaserver +DEFINE_uint64(heartbeat_offlineTimeoutMs, 1800000, + "the maximun peroid that heartbeat is missed without setting the " + "metaserver to offline status and alarm."); + +// network jitter is unavoidable, and for this reason +// background process will alarm during the inspection once it +// finds out that heartbeat is missed after heartbeatMissTimeOut peroid +DEFINE_uint64( + heartbeat_missTimeoutMs, 30000, + "background process will alarm during the inspection once it finds out " + "that heartbeat is missed after heartbeatMissTimeOut peroid"); +DEFINE_validator(heartbeat_offlineTimeoutMs, [](const char*, uint64_t value) { + return value >= FLAGS_heartbeat_missTimeoutMs; +}); + namespace curvefs { namespace mds { namespace heartbeat { @@ -68,7 +86,8 @@ bool MetaserverHealthyChecker::MetaServerStateNeedUpdate( return false; } - bool shouldUnstable = (timePass < milliseconds(option_.offLineTimeOutMs)); + bool shouldUnstable = + (timePass < milliseconds(FLAGS_heartbeat_offlineTimeoutMs)); if (shouldUnstable) { if (OnlineState::UNSTABLE != info.state) { LOG(WARNING) << "metaserver " << info.msId << " is unstable. " diff --git a/curvefs/src/mds/heartbeat/metaserver_healthy_checker.h b/curvefs/src/mds/heartbeat/metaserver_healthy_checker.h index d1428d7989..e1404342c4 100644 --- a/curvefs/src/mds/heartbeat/metaserver_healthy_checker.h +++ b/curvefs/src/mds/heartbeat/metaserver_healthy_checker.h @@ -23,19 +23,25 @@ #ifndef CURVEFS_SRC_MDS_HEARTBEAT_METASERVER_HEALTHY_CHECKER_H_ #define CURVEFS_SRC_MDS_HEARTBEAT_METASERVER_HEALTHY_CHECKER_H_ -#include //NOLINT -#include +#include + +#include //NOLINT #include -#include "curvefs/src/mds/common/types.h" +#include + +#include "curvefs/proto/topology.pb.h" #include "curvefs/src/mds/common/mds_define.h" +#include "curvefs/src/mds/common/types.h" #include "curvefs/src/mds/topology/topology.h" -#include "curvefs/proto/topology.pb.h" using ::std::chrono::steady_clock; using ::curvefs::mds::topology::MetaServerIdType; using ::curvefs::mds::topology::Topology; using ::curvefs::mds::topology::OnlineState; +DECLARE_uint64(heartbeat_offlineTimeoutMs); +DECLARE_uint64(heartbeat_missTimeoutMs); + namespace curvefs { namespace mds { namespace heartbeat { @@ -45,8 +51,20 @@ struct HeartbeatOption { uint64_t heartbeatMissTimeout, uint64_t offLineTimeout) { this->heartbeatIntervalMs = heartbeatInterval; - this->heartbeatMissTimeOutMs = heartbeatMissTimeout; - this->offLineTimeOutMs = offLineTimeout; + FLAGS_heartbeat_missTimeoutMs = this->heartbeatMissTimeOutMs = + heartbeatMissTimeout; + FLAGS_heartbeat_offlineTimeoutMs = this->offLineTimeOutMs = + offLineTimeout; + } + + explicit HeartbeatOption(const HeartbeatOption& option) + : heartbeatIntervalMs(option.heartbeatIntervalMs), + heartbeatMissTimeOutMs(option.heartbeatMissTimeOutMs), + offLineTimeOutMs(option.offLineTimeOutMs), + cleanFollowerAfterMs(option.cleanFollowerAfterMs), + mdsStartTime(option.mdsStartTime) { + FLAGS_heartbeat_offlineTimeoutMs = offLineTimeOutMs; + FLAGS_heartbeat_missTimeoutMs = heartbeatMissTimeOutMs; } // heartbeatIntervalMs: normal heartbeat interval. @@ -90,9 +108,9 @@ struct HeartbeatInfo { class MetaserverHealthyChecker { public: - MetaserverHealthyChecker( - HeartbeatOption option, const std::shared_ptr &topo) : - option_(option), topo_(topo) {} + MetaserverHealthyChecker(const HeartbeatOption& option, + const std::shared_ptr& topo) + : option_(option), topo_(topo) {} ~MetaserverHealthyChecker() {} /** diff --git a/curvefs/src/metaserver/trash.cpp b/curvefs/src/metaserver/trash.cpp index db44dea151..1175376e72 100644 --- a/curvefs/src/metaserver/trash.cpp +++ b/curvefs/src/metaserver/trash.cpp @@ -21,8 +21,10 @@ */ #include "curvefs/src/metaserver/trash.h" -#include "src/common/timeutility.h" + #include "curvefs/proto/mds.pb.h" +#include "curvefs/src/metaserver/inode_storage.h" +#include "src/common/timeutility.h" using ::curve::common::TimeUtility; @@ -32,9 +34,18 @@ namespace metaserver { using ::curvefs::mds::FsInfo; using ::curvefs::mds::FSStatusCode; +bool pass_uint32(const char*, uint32_t value) { return true; } +DEFINE_uint32(trash_expiredAfterSec, 604800, + "time to delete data in the recycle bin"); +DEFINE_uint32(trash_scanPeriodSec, 600, "recycle bin scan interval"); +DEFINE_validator(trash_expiredAfterSec, pass_uint32); +DEFINE_validator(trash_scanPeriodSec, pass_uint32); + void TrashOption::InitTrashOptionFromConf(std::shared_ptr conf) { conf->GetValueFatalIfFail("trash.scanPeriodSec", &scanPeriodSec); + FLAGS_trash_scanPeriodSec = scanPeriodSec; conf->GetValueFatalIfFail("trash.expiredAfterSec", &expiredAfterSec); + FLAGS_trash_expiredAfterSec = expiredAfterSec; } void TrashImpl::Init(const TrashOption &option) { @@ -129,7 +140,7 @@ bool TrashImpl::NeedDelete(const TrashItem &item) { // if fs recycleTimeHour is not 0, return true uint64_t recycleTimeHour = GetFsRecycleTimeHour(item.fsId); if (recycleTimeHour == 0) { - return ((now - item.dtime) >= options_.expiredAfterSec); + return ((now - item.dtime) >= FLAGS_trash_expiredAfterSec); } else { return true; } diff --git a/curvefs/src/metaserver/trash_manager.cpp b/curvefs/src/metaserver/trash_manager.cpp index 936f820218..7f6341db1c 100644 --- a/curvefs/src/metaserver/trash_manager.cpp +++ b/curvefs/src/metaserver/trash_manager.cpp @@ -27,6 +27,8 @@ namespace curvefs { namespace metaserver { +DECLARE_uint32(trash_scanPeriodSec); + int TrashManager::Run() { if (isStop_.exchange(false)) { recycleThread_ = @@ -48,9 +50,9 @@ void TrashManager::Fini() { } void TrashManager::ScanLoop() { - while (sleeper_.wait_for(std::chrono::seconds(options_.scanPeriodSec))) { - ScanEveryTrash(); - } + while (sleeper_.wait_for(std::chrono::seconds(FLAGS_trash_scanPeriodSec))) { + ScanEveryTrash(); + } } void TrashManager::ScanEveryTrash() { diff --git a/curvefs/test/client/filesystem/helper/builder.h b/curvefs/test/client/filesystem/helper/builder.h index dc7c196adb..56c4b70874 100644 --- a/curvefs/test/client/filesystem/helper/builder.h +++ b/curvefs/test/client/filesystem/helper/builder.h @@ -37,6 +37,9 @@ namespace curvefs { namespace client { +namespace common { +DECLARE_bool(fs_disableXattr); +} namespace filesystem { using ::curvefs::client::common::KernelCacheOption; @@ -197,7 +200,7 @@ class FileSystemBuilder { }; option.cto = true; - option.disableXattr = true; + common::FLAGS_fs_disableXattr = option.disableXattr = true; option.maxNameLength = 255; option.blockSize = 0x10000u; option.kernelCacheOption = kernelCacheOption; From 41e0eb4cfd492537bc4b2dd355591a444a700f85 Mon Sep 17 00:00:00 2001 From: hzwuhongsong Date: Wed, 11 Oct 2023 15:24:37 +0800 Subject: [PATCH 06/27] curvefs/client: optimizing the read amplification problem(especial memcached) --- curvefs/conf/client.conf | 14 +- curvefs/src/client/common/config.cpp | 11 + curvefs/src/client/common/config.h | 5 + .../src/client/kvclient/kvclient_manager.cpp | 21 ++ .../src/client/kvclient/kvclient_manager.h | 37 ++- curvefs/src/client/s3/client_s3_adaptor.cpp | 40 ++- curvefs/src/client/s3/client_s3_adaptor.h | 24 ++ .../src/client/s3/client_s3_cache_manager.cpp | 156 ++++++--- .../src/client/s3/client_s3_cache_manager.h | 21 +- .../src/client/s3/disk_cache_manager_impl.h | 16 +- curvefs/test/client/BUILD | 24 ++ curvefs/test/client/client_prefetch_test.cpp | 310 ++++++++++++++++++ curvefs/test/client/mock_disk_cache_manager.h | 12 +- 13 files changed, 629 insertions(+), 62 deletions(-) create mode 100644 curvefs/test/client/client_prefetch_test.cpp diff --git a/curvefs/conf/client.conf b/curvefs/conf/client.conf index 339a08a0d2..b7b00baceb 100644 --- a/curvefs/conf/client.conf +++ b/curvefs/conf/client.conf @@ -202,6 +202,18 @@ s3.writeCacheMaxByte=838860800 s3.readCacheMaxByte=209715200 # file cache read thread num s3.readCacheThreads=5 + +# The data in the cache cluster download to local +s3.memClusterToLocal=true +# The data in the s3 storage download to local +s3.s3ToLocal=true +# read size bigger than this value will read until prefetch is finished +s3.bigIoSize=131072 +# retry times when read big io failed +s3.bigIoRetryTimes=100 +# retry interval when read big io failed +s3.bigIoRetryIntervalUs=100 + # http = 0, https = 1 s3.http_scheme=0 s3.verify_SSL=False @@ -245,7 +257,7 @@ diskCache.asyncLoadPeriodMs=5 # ok nearfull full # |------------|-------------------|----------------------| # 0 trimRatio*safeRatio safeRatio fullRatio -# +# # 1. 0<=oks3ClientAdaptorOpt.writeCacheMaxByte); conf->GetValueFatalIfFail("s3.readCacheMaxByte", &s3Opt->s3ClientAdaptorOpt.readCacheMaxByte); + conf->GetValueFatalIfFail("s3.memClusterToLocal", + &s3Opt->s3ClientAdaptorOpt.memClusterToLocal); + conf->GetValueFatalIfFail("s3.s3ToLocal", + &s3Opt->s3ClientAdaptorOpt.s3ToLocal); + conf->GetValueFatalIfFail("s3.bigIoSize", + &s3Opt->s3ClientAdaptorOpt.bigIoSize); + conf->GetValueFatalIfFail("s3.bigIoRetryTimes", + &s3Opt->s3ClientAdaptorOpt.bigIoRetryTimes); + conf->GetValueFatalIfFail("s3.bigIoRetryIntervalUs", + &s3Opt->s3ClientAdaptorOpt.bigIoRetryIntervalUs); conf->GetValueFatalIfFail("s3.readCacheThreads", &s3Opt->s3ClientAdaptorOpt.readCacheThreads); conf->GetValueFatalIfFail("s3.nearfullRatio", @@ -240,6 +250,7 @@ void InitS3Option(Configuration *conf, S3Option *s3Opt) { &s3Opt->s3ClientAdaptorOpt.readRetryIntervalMs); ::curve::common::InitS3AdaptorOptionExceptS3InfoOption(conf, &s3Opt->s3AdaptrOpt); + InitDiskCacheOption(conf, &s3Opt->s3ClientAdaptorOpt.diskCacheOpt); } diff --git a/curvefs/src/client/common/config.h b/curvefs/src/client/common/config.h index 6d93f5a059..30c6417ea7 100644 --- a/curvefs/src/client/common/config.h +++ b/curvefs/src/client/common/config.h @@ -127,6 +127,11 @@ struct S3ClientAdaptorOption { uint32_t flushIntervalSec; uint64_t writeCacheMaxByte; uint64_t readCacheMaxByte; + bool memClusterToLocal; + bool s3ToLocal; + uint32_t bigIoSize; + uint32_t bigIoRetryTimes; + uint32_t bigIoRetryIntervalUs; uint32_t readCacheThreads; uint32_t nearfullRatio; uint32_t baseSleepUs; diff --git a/curvefs/src/client/kvclient/kvclient_manager.cpp b/curvefs/src/client/kvclient/kvclient_manager.cpp index a808580c8d..c87630758b 100644 --- a/curvefs/src/client/kvclient/kvclient_manager.cpp +++ b/curvefs/src/client/kvclient/kvclient_manager.cpp @@ -107,5 +107,26 @@ void KVClientManager::Get(std::shared_ptr task) { }); } +void KVClientManager::Enqueue(std::shared_ptr context) { + auto task = [this, context]() { this->GetKvCache(context); }; + threadPool_.Enqueue(task); +} + +int KVClientManager::GetKvCache( + std::shared_ptr context) { + VLOG(9) << "GetKvCache start: " << context->key; + std::string error_log; + memcached_return_t retCode; + uint64_t actLength = 0; + context->retCode = + !client_->Get(context->key, context->buf, context->offset, context->len, + &error_log, &actLength, &retCode); + context->actualLen = actLength; + context->cb(nullptr, context); + VLOG(9) << "GetKvCache end: " << context->key << ", " << context->retCode + << ", " << context->actualLen; + return 0; +} + } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/kvclient/kvclient_manager.h b/curvefs/src/client/kvclient/kvclient_manager.h index ad8701ec84..bad99193a8 100644 --- a/curvefs/src/client/kvclient/kvclient_manager.h +++ b/curvefs/src/client/kvclient/kvclient_manager.h @@ -46,6 +46,11 @@ namespace client { class KVClientManager; struct SetKVCacheTask; struct GetKVCacheTask; + +class GetKvCacheContext; +class SetKvCacheContext; + +using curve::common::GetObjectAsyncContext; using curve::common::TaskThreadPool; using curvefs::client::common::KVClientManagerOpt; @@ -96,6 +101,33 @@ struct GetKVCacheTask { timer(butil::Timer::STARTED) {} }; +using GetKvCacheCallBack = + std::function&)>; + +using SetKvCacheCallBack = + std::function&)>; + +struct KvCacheContext { + std::string key; + uint64_t inodeId; + uint64_t offset; + uint64_t length; + uint64_t chunkIndex; + uint64_t chunkPos; + uint64_t startTime; +}; + +struct GetKvCacheContext : KvCacheContext { + char* value; + bool res; + GetKvCacheCallBack cb; +}; + +struct SetKvCacheContext : KvCacheContext { + const char* value; + SetKvCacheCallBack cb; +}; + class KVClientManager { public: KVClientManager() = default; @@ -118,8 +150,11 @@ class KVClientManager { return kvClientManagerMetric_.get(); } + void Enqueue(std::shared_ptr context); + private: void Uninit(); + int GetKvCache(std::shared_ptr context); private: TaskThreadPool threadPool_; @@ -129,4 +164,4 @@ class KVClientManager { } // namespace client } // namespace curvefs -#endif // CURVEFS_SRC_CLIENT_KVCLIENT_KVCLIENT_MANAGER_H_ +#endif // CURVEFS_SRC_CLIENT_KVCLIENT_KVCLIENT_MANAGER_H_ diff --git a/curvefs/src/client/s3/client_s3_adaptor.cpp b/curvefs/src/client/s3/client_s3_adaptor.cpp index 85d94c874e..89fb0257b6 100644 --- a/curvefs/src/client/s3/client_s3_adaptor.cpp +++ b/curvefs/src/client/s3/client_s3_adaptor.cpp @@ -20,19 +20,42 @@ * Author: huyao */ +#include "curvefs/src/client/s3/client_s3_adaptor.h" + #include #include + #include #include #include #include "absl/memory/memory.h" -#include "curvefs/src/client/s3/client_s3_adaptor.h" #include "curvefs/src/common/s3util.h" namespace curvefs { - namespace client { + +/** + * use curl -L mdsIp:port/flags/memClusterToLocal?setvalue=true + * for dynamic parameter configuration + */ +static bool pass_uint32(const char*, uint32_t) { return true; } +static bool pass_bool(const char*, bool) { return true; } +DEFINE_bool(memClusterToLocal, true, + "The data in the cache cluster download to local"); +DEFINE_validator(memClusterToLocal, &pass_bool); +DEFINE_bool(s3ToLocal, true, "The data in the s3 storage download to local"); +DEFINE_validator(s3ToLocal, &pass_bool); +DEFINE_uint32( + bigIoSize, 131072, + "read size bigger than this value will read until prefetch is finished"); +DEFINE_validator(bigIoSize, &pass_uint32); +DEFINE_uint32(bigIoRetryTimes, 100, "retry times when read big io failed"); +DEFINE_validator(bigIoRetryTimes, &pass_uint32); +DEFINE_uint32(bigIoRetryIntervalUs, 100, + "retry interval when read big io failed"); +DEFINE_validator(bigIoRetryIntervalUs, &pass_uint32); + CURVEFS_ERROR S3ClientAdaptorImpl::Init( const S3ClientAdaptorOption &option, std::shared_ptr client, @@ -55,6 +78,11 @@ S3ClientAdaptorImpl::Init( prefetchExecQueueNum_ = option.prefetchExecQueueNum; diskCacheType_ = option.diskCacheOpt.diskCacheType; memCacheNearfullRatio_ = option.nearfullRatio; + FLAGS_memClusterToLocal = option.memClusterToLocal; + FLAGS_s3ToLocal = option.s3ToLocal; + FLAGS_bigIoSize = option.bigIoSize; + FLAGS_bigIoRetryTimes = option.bigIoRetryTimes; + FLAGS_bigIoRetryIntervalUs = option.bigIoRetryIntervalUs; throttleBaseSleepUs_ = option.baseSleepUs; flushIntervalSec_ = option.flushIntervalSec; chunkFlushThreads_ = option.chunkFlushThreads; @@ -100,7 +128,13 @@ S3ClientAdaptorImpl::Init( << ", readCacheMaxByte: " << option.readCacheMaxByte << ", readCacheThreads: " << option.readCacheThreads << ", nearfullRatio: " << option.nearfullRatio - << ", baseSleepUs: " << option.baseSleepUs; + << ", baseSleepUs: " << option.baseSleepUs + << ", memClusterToLocal: " << FLAGS_memClusterToLocal + << ", s3ToLocal: " << FLAGS_s3ToLocal + << ", bigIoSize: " << FLAGS_bigIoSize + << ", bigIoRetryTimes: " << FLAGS_bigIoRetryTimes + << ", bigIoRetryIntervalUs: " << FLAGS_bigIoRetryIntervalUs; + // start chunk flush threads taskPool_.Start(chunkFlushThreads_); return CURVEFS_ERROR::OK; diff --git a/curvefs/src/client/s3/client_s3_adaptor.h b/curvefs/src/client/s3/client_s3_adaptor.h index 5f9e52a528..dcc25c3795 100644 --- a/curvefs/src/client/s3/client_s3_adaptor.h +++ b/curvefs/src/client/s3/client_s3_adaptor.h @@ -143,12 +143,15 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { CURVEFS_ERROR FlushAllCache(uint64_t inodeId); CURVEFS_ERROR FsSync(); int Stop(); + uint64_t GetBlockSize() { return blockSize_; } + uint64_t GetChunkSize() { return chunkSize_; } + uint32_t GetObjectPrefix() { return objectPrefix_; } @@ -156,32 +159,43 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { std::shared_ptr GetFsCacheManager() { return fsCacheManager_; } + uint32_t GetFlushInterval() { return flushIntervalSec_; } + std::shared_ptr GetS3Client() { return client_; } + uint32_t GetPrefetchBlocks() { return prefetchBlocks_; } + uint32_t GetDiskCacheType() { return diskCacheType_; } + bool DisableDiskCache() { return diskCacheType_ == DiskCacheType::Disable; } + bool HasDiskCache() { return diskCacheType_ != DiskCacheType::Disable; } + bool IsReadCache() { return diskCacheType_ == DiskCacheType::OnlyRead; } + bool IsReadWriteCache() { return diskCacheType_ == DiskCacheType::ReadWrite; } + std::shared_ptr GetInodeCacheManager() { return inodeManager_; } + std::shared_ptr GetDiskCacheManager() { return diskCacheManagerImpl_; } + FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum, uint64_t *chunkId); void FsSyncSignal() { @@ -189,21 +203,26 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { VLOG(3) << "fs sync signal"; cond_.notify_one(); } + void FsSyncSignalAndDataCacheInc() { std::lock_guard lk(mtx_); fsCacheManager_->DataCacheNumInc(); VLOG(3) << "fs sync signal"; cond_.notify_one(); } + void SetFsId(uint32_t fsId) { fsId_ = fsId; } + uint32_t GetFsId() { return fsId_; } + uint32_t GetPageSize() { return pageSize_; } + void InitMetrics(const std::string &fsName); void SetDiskCache(DiskCacheType type) { @@ -253,6 +272,11 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { uint32_t flushIntervalSec_; uint32_t chunkFlushThreads_; uint32_t memCacheNearfullRatio_; + bool memClusterToLocal_; + bool s3ToLocal_; + uint32_t bigIoSize_; + uint32_t bigIoRetryTimes_; + uint32_t bigIoRetryIntervalUs_; uint32_t throttleBaseSleepUs_; uint32_t maxReadRetryIntervalMs_; uint32_t readRetryIntervalMs_; diff --git a/curvefs/src/client/s3/client_s3_cache_manager.cpp b/curvefs/src/client/s3/client_s3_cache_manager.cpp index 1532a9282e..c169c45675 100644 --- a/curvefs/src/client/s3/client_s3_cache_manager.cpp +++ b/curvefs/src/client/s3/client_s3_cache_manager.cpp @@ -28,6 +28,8 @@ #include #include "absl/cleanup/cleanup.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" #include "absl/synchronization/blocking_counter.h" #include "curvefs/src/client/kvclient/kvclient_manager.h" #include "curvefs/src/client/metric/client_metric.h" @@ -37,6 +39,13 @@ namespace curvefs { namespace client { + +DECLARE_bool(memClusterToLocal); +DECLARE_bool(s3ToLocal); +DECLARE_uint32(bigIoSize); +DECLARE_uint32(bigIoRetryTimes); +DECLARE_uint32(bigIoRetryIntervalUs); + namespace common { DECLARE_bool(enableCto); } // namespace common @@ -128,9 +137,9 @@ void FsCacheManager::ReleaseFileCacheManager(uint64_t inodeId) { bool FsCacheManager::Set(DataCachePtr dataCache, std::list::iterator *outIter) { std::lock_guard lk(lruMtx_); - VLOG(3) << "lru current byte:" << lruByte_ - << ",lru max byte:" << readCacheMaxByte_ - << ", dataCache len:" << dataCache->GetLen(); + VLOG(3) << "lru current byte: " << lruByte_ + << ", lru max byte: " << readCacheMaxByte_ + << ", dataCache len: " << dataCache->GetLen(); if (readCacheMaxByte_ == 0) { return false; } @@ -437,7 +446,7 @@ int FileCacheManager::Read(uint64_t inodeId, uint64_t offset, uint64_t length, if (memCacheMissRequest.empty()) { return actualReadLen; } - + VLOG(6) << "memcache miss request size: " << memCacheMissRequest.size(); // 2. read from localcache and remote cluster std::shared_ptr inodeWrapper; @@ -477,15 +486,29 @@ int FileCacheManager::Read(uint64_t inodeId, uint64_t offset, uint64_t length, return actualReadLen; } -bool FileCacheManager::ReadKVRequestFromLocalCache(const std::string &name, - char *databuf, +bool FileCacheManager::ReadKVRequestFromLocalCache(const std::string& name, + char* databuf, uint64_t offset, uint64_t len) { uint64_t start = butil::cpuwide_time_us(); + if (!s3ClientAdaptor_->HasDiskCache()) { + return false; + } + if (!IsCachedInLocal(name) && len >= FLAGS_bigIoSize && + s3ClientAdaptor_->GetPrefetchBlocks()) { + int retry = 0; + do { + VLOG(6) << "wait for download object: " << name; + bthread_usleep(FLAGS_bigIoRetryIntervalUs); + if (++retry >= FLAGS_bigIoRetryTimes) { + LOG(WARNING) << "download object: " << name << " timeout"; + return false; + } + } while (!IsCachedInLocal(name)); + } - bool mayCached = s3ClientAdaptor_->HasDiskCache() && - s3ClientAdaptor_->GetDiskCacheManager()->IsCached(name); - if (!mayCached) { + if (!IsCachedInLocal(name)) { + VLOG(9) << "not cachd in disk, " << name; return false; } @@ -594,8 +617,34 @@ void FileCacheManager::ProcessKVRequest(const S3ReadRequest &req, char *dataBuf, const uint32_t objectPrefix = s3ClientAdaptor_->GetObjectPrefix(); GetBlockLoc(req.offset, &chunkIndex, &chunkPos, &blockIndex, &blockPos); + std::string prefetchName = curvefs::common::s3util::GenObjName( + req.chunkId, blockIndex, req.compaction, req.fsId, req.inodeId, + objectPrefix); + bool waitDownloading = false; + // if obj is in downloading, wait for it. + while (true) { + { + curve::common::LockGuard lg(downloadMtx_); + if (downloadingObj_.find(prefetchName) != downloadingObj_.end()) { + VLOG(9) << "wait for obj is in downloading: " << prefetchName + << ", size: " << downloadingObj_.size() << ", " + << downloadingObj_.size(); + waitDownloading = true; + } else { + VLOG(9) << "obj is not in downloading: " << prefetchName + << ", size: " << downloadingObj_.size() << ", " + << downloadingObj_.size(); + break; + } + } + if (waitDownloading) { + bthread_usleep(FLAGS_bigIoRetryIntervalUs); + } + } + // prefetch - if (s3ClientAdaptor_->HasDiskCache()) { + if (s3ClientAdaptor_->HasDiskCache() && !waitDownloading && + !IsCachedInLocal(prefetchName)) { PrefetchForBlock(req, fileLen, blockSize, chunkSize, blockIndex); } @@ -673,19 +722,21 @@ void FileCacheManager::ProcessKVRequest(const S3ReadRequest &req, char *dataBuf, } } -void FileCacheManager::PrefetchForBlock(const S3ReadRequest &req, +void FileCacheManager::PrefetchForBlock(const S3ReadRequest& req, uint64_t fileLen, uint64_t blockSize, uint64_t chunkSize, uint64_t startBlockIndex) { uint32_t prefetchBlocks = s3ClientAdaptor_->GetPrefetchBlocks(); + if (prefetchBlocks == 0) { + return; + } uint32_t objectPrefix = s3ClientAdaptor_->GetObjectPrefix(); std::vector> prefetchObjs; - uint64_t blockIndex = startBlockIndex; for (uint32_t i = 0; i < prefetchBlocks; i++) { std::string name = curvefs::common::s3util::GenObjName( - req.chunkId, blockIndex, req.compaction, - req.fsId, req.inodeId, objectPrefix); + req.chunkId, blockIndex, req.compaction, req.fsId, req.inodeId, + objectPrefix); uint64_t maxReadLen = (blockIndex + 1) * blockSize; uint64_t needReadLen = maxReadLen > fileLen ? fileLen - blockIndex * blockSize : blockSize; @@ -698,18 +749,26 @@ void FileCacheManager::PrefetchForBlock(const S3ReadRequest &req, } } - PrefetchS3Objs(prefetchObjs); + // It is configurable whether to write to local cache or not + if (!kvClientManager_ && FLAGS_s3ToLocal) { + // get from s3 directly + PrefetchS3Objs(prefetchObjs, true); + } else if (FLAGS_memClusterToLocal) { + // get from memcached first, if failed, get from s3 + PrefetchS3Objs(prefetchObjs, false); + } } class AsyncPrefetchCallback { public: - AsyncPrefetchCallback(uint64_t inode, S3ClientAdaptorImpl *s3Client) - : inode_(inode), s3Client_(s3Client) {} + AsyncPrefetchCallback(uint64_t inode, S3ClientAdaptorImpl* s3Client, + bool fromS3) + : inode_(inode), s3Client_(s3Client), fromS3_(fromS3) {} - void operator()(const S3Adapter *, - const std::shared_ptr &context) { + void operator()(const S3Adapter*, + const std::shared_ptr& context) { VLOG(9) << "prefetch end: " << context->key << ", len " << context->len - << "actual len: " << context->actualLen; + << "actual len: " << context->actualLen << ", " << fromS3_; std::unique_ptr guard(context->buf); auto fileCache = s3Client_->GetFsCacheManager()->FindFileCacheManager(inode_); @@ -721,8 +780,18 @@ class AsyncPrefetchCallback { return; } - if (context->retCode < 0) { - LOG(WARNING) << "prefetch failed, key: " << context->key; + if (context->retCode != 0 && !fromS3_) { + VLOG(6) << "failed and then get from s3, key: " << context->key; + std::vector> prefetchObjs; + prefetchObjs.push_back(std::make_pair(context->key, context->len)); + fileCache->PrefetchS3Objs(prefetchObjs); + curve::common::LockGuard lg(fileCache->downloadMtx_); + fileCache->downloadingObj_.erase(context->key); + return; + } else if (context->retCode != 0 && fromS3_) { + curve::common::LockGuard lg(fileCache->downloadMtx_); + fileCache->downloadingObj_.erase(context->key); + LOG_EVERY_SECOND(INFO) << "prefetch failed, key: " << context->key; return; } if (s3Client_->s3Metric_ != nullptr) { @@ -747,16 +816,19 @@ class AsyncPrefetchCallback { curve::common::LockGuard lg(fileCache->downloadMtx_); fileCache->downloadingObj_.erase(context->key); } + VLOG(9) << "prefetch success: " << context->key; } private: const uint64_t inode_; S3ClientAdaptorImpl *s3Client_; + bool fromS3_; }; void FileCacheManager::PrefetchS3Objs( - const std::vector> &prefetchObjs) { - for (auto &obj : prefetchObjs) { + const std::vector>& prefetchObjs, + bool fromS3) { + for (auto& obj : prefetchObjs) { std::string name = obj.first; uint64_t readLen = obj.second; curve::common::LockGuard lg(downloadMtx_); @@ -765,28 +837,32 @@ void FileCacheManager::PrefetchS3Objs( << ", size: " << downloadingObj_.size(); continue; } - if (s3ClientAdaptor_->GetDiskCacheManager()->IsCached(name)) { + if (IsCachedInLocal(name)) { VLOG(9) << "downloading is exist in cache: " << name << ", size: " << downloadingObj_.size(); continue; } VLOG(9) << "download start: " << name - << ", size: " << downloadingObj_.size(); + << ", size: " << downloadingObj_.size() + << ", from s3: " << fromS3; downloadingObj_.emplace(name); - auto inode = inode_; - auto s3ClientAdaptor = s3ClientAdaptor_; - auto task = [name, inode, s3ClientAdaptor, readLen]() { - char *dataCacheS3 = new char[readLen]; + char* dataCacheS3 = new char[readLen]; + VLOG(9) << "prefetch start: " << name << ", len: " << readLen; + if (fromS3) { auto context = std::make_shared( name, dataCacheS3, 0, readLen, - AsyncPrefetchCallback{inode, s3ClientAdaptor}, - curve::common::ContextType::S3); - VLOG(9) << "prefetch start: " << context->key - << ", len: " << context->len; - s3ClientAdaptor->GetS3Client()->DownloadAsync(context); - }; - s3ClientAdaptor_->PushAsyncTask(task); + AsyncPrefetchCallback{inode_, s3ClientAdaptor_, true}); + auto task = [this, context]() { + s3ClientAdaptor_->GetS3Client()->DownloadAsync(context); + }; + s3ClientAdaptor_->PushAsyncTask(task); + } else { + auto context = std::make_shared( + name, dataCacheS3, 0, readLen, + AsyncPrefetchCallback{inode_, s3ClientAdaptor_, false}); + kvClientManager_->Enqueue(context); + } } return; } @@ -1144,6 +1220,10 @@ CURVEFS_ERROR FileCacheManager::Flush(bool force, bool toS3) { return ret; } +bool FileCacheManager::IsCachedInLocal(const std::string name) { + return s3ClientAdaptor_->GetDiskCacheManager()->IsCached(name); +} + void ChunkCacheManager::ReadChunk(uint64_t index, uint64_t chunkPos, uint64_t readLen, char *dataBuf, uint64_t dataBufOffset, @@ -1301,7 +1381,7 @@ void ChunkCacheManager::ReadByReadCache(uint64_t chunkPos, uint64_t readLen, ReadLockGuard readLockGuard(rwLockRead_); VLOG(9) << "ReadByReadCache chunkPos:" << chunkPos << ",readLen:" << readLen - << ",dataBufOffset:" << dataBufOffset; + << ",dataBufOffset:" << dataBufOffset << "chunkIndex: " << index_; if (dataRCacheMap_.empty()) { VLOG(9) << "dataRCacheMap_ is empty"; ReadRequest request; diff --git a/curvefs/src/client/s3/client_s3_cache_manager.h b/curvefs/src/client/s3/client_s3_cache_manager.h index d27aa308c5..df3d665ac4 100644 --- a/curvefs/src/client/s3/client_s3_cache_manager.h +++ b/curvefs/src/client/s3/client_s3_cache_manager.h @@ -27,19 +27,19 @@ #include #include #include +#include #include #include #include #include -#include #include "curvefs/proto/metaserver.pb.h" #include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/client/inode_wrapper.h" +#include "curvefs/src/client/kvclient/kvclient_manager.h" #include "curvefs/src/client/s3/client_s3.h" #include "src/common/concurrent/concurrent.h" #include "src/common/concurrent/task_thread_pool.h" -#include "curvefs/src/client/kvclient/kvclient_manager.h" -#include "curvefs/src/client/inode_wrapper.h" using curve::common::ReadLockGuard; using curve::common::RWLock; @@ -370,12 +370,13 @@ class FileCacheManager { void WriteChunk(uint64_t index, uint64_t chunkPos, uint64_t writeLen, const char *dataBuf); void GenerateS3Request(ReadRequest request, - const S3ChunkInfoList &s3ChunkInfoList, - char *dataBuf, std::vector *requests, + const S3ChunkInfoList& s3ChunkInfoList, + char* dataBuf, std::vector* requests, uint64_t fsId, uint64_t inodeId); void PrefetchS3Objs( - const std::vector> &prefetchObjs); + const std::vector>& prefetchObjs, + bool fromS3 = true); void HandleReadRequest(const ReadRequest &request, const S3ChunkInfo &s3ChunkInfo, @@ -403,9 +404,11 @@ class FileCacheManager { // miss read from memory read/write cache, need read from // kv(localdisk/remote cache/s3) - int GenerateKVRequest(const std::shared_ptr &inodeWrapper, - const std::vector &readRequest, - char *dataBuf, std::vector *kvRequest); + int GenerateKVRequest(const std::shared_ptr& inodeWrapper, + const std::vector& readRequest, + char* dataBuf, std::vector* kvRequest); + + bool IsCachedInLocal(const std::string name); enum class ReadStatus { OK = 0, diff --git a/curvefs/src/client/s3/disk_cache_manager_impl.h b/curvefs/src/client/s3/disk_cache_manager_impl.h index 372844baef..f6e02deeb1 100644 --- a/curvefs/src/client/s3/disk_cache_manager_impl.h +++ b/curvefs/src/client/s3/disk_cache_manager_impl.h @@ -79,7 +79,7 @@ class DiskCacheManagerImpl { * @param[in] option config option * @return success: 0, fail : < 0 */ - int Init(const S3ClientAdaptorOption option); + virtual int Init(const S3ClientAdaptorOption option); /** * @brief Write obj * @param[in] name obj name @@ -87,13 +87,13 @@ class DiskCacheManagerImpl { * @param[in] length write length * @return success: write length, fail : < 0 */ - int Write(const std::string name, const char *buf, uint64_t length); + virtual int Write(const std::string name, const char* buf, uint64_t length); /** * @brief whether obj is cached in cached disk * @param[in] name obj name * @return cached: true, not cached : < 0 */ - bool IsCached(const std::string name); + virtual bool IsCached(const std::string name); /** * @brief read obj * @param[in] name obj name @@ -102,17 +102,17 @@ class DiskCacheManagerImpl { * @param[in] length read length * @return success: length, fail : < length */ - int Read(const std::string name, char *buf, uint64_t offset, - uint64_t length); + virtual int Read(const std::string name, char* buf, uint64_t offset, + uint64_t length); /** * @brief umount disk cache * @return success: 0, fail : < 0 */ - int UmountDiskCache(); + virtual int UmountDiskCache(); bool IsDiskCacheFull(); - int WriteReadDirect(const std::string fileName, const char *buf, - uint64_t length); + virtual int WriteReadDirect(const std::string fileName, const char* buf, + uint64_t length); void InitMetrics(std::string fsName, std::shared_ptr s3Metric); virtual int UploadWriteCacheByInode(const std::string &inode); diff --git a/curvefs/test/client/BUILD b/curvefs/test/client/BUILD index 82adff0717..65bfb51d50 100644 --- a/curvefs/test/client/BUILD +++ b/curvefs/test/client/BUILD @@ -93,6 +93,7 @@ cc_test( "file_cache_manager_test.cpp", "chunk_cache_manager_test.cpp", "data_cache_test.cpp", + "client_prefetch_test.cpp", "client_s3_adaptor_Integration.cpp", "client_memcache_test.cpp", ], @@ -144,3 +145,26 @@ cc_test( copts = CURVE_TEST_COPTS, visibility = ["//visibility:public"], ) + +cc_test( + name = "curvefs_client_prefetch_test", + srcs = glob([ + "main.cpp", + "client_prefetch_test.cpp", + "*.h", + ]), + copts = CURVE_TEST_COPTS, + defines = ["UNIT_TEST"], + visibility = ["//visibility:public"], + deps = [ + "//external:gtest", + "//curvefs/src/client:fuse_client_lib", + "//curvefs/proto:metaserver_cc_proto", + "//curvefs/proto:mds_cc_proto", + "//curvefs/proto:space_cc_proto", + "//curvefs/test/client/rpcclient:rpcclient_test_mock", + "//test/client/mock:client_mock_lib", + ], + linkopts = [ + "-L/usr/local/lib/x86_64-linux-gnu"], +) diff --git a/curvefs/test/client/client_prefetch_test.cpp b/curvefs/test/client/client_prefetch_test.cpp new file mode 100644 index 0000000000..712fbd208d --- /dev/null +++ b/curvefs/test/client/client_prefetch_test.cpp @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: Wed Mar 23 2023 + * Author: wuhongsong + */ + +#include +#include + +#include "curvefs/src/client/common/common.h" +#include "curvefs/src/client/s3/client_s3_adaptor.h" +#include "curvefs/src/client/s3/client_s3_cache_manager.h" +#include "curvefs/src/client/s3/disk_cache_manager_impl.h" +#include "curvefs/test/client/mock_client_s3.h" +#include "curvefs/test/client/mock_client_s3_cache_manager.h" +#include "curvefs/test/client/mock_disk_cache_base.h" +#include "curvefs/test/client/mock_disk_cache_manager.h" +#include "curvefs/test/client/mock_disk_cache_read.h" +#include "curvefs/test/client/mock_disk_cache_write.h" +#include "curvefs/test/client/mock_inode_cache_manager.h" +#include "curvefs/test/client/mock_kvclient.h" +#include "curvefs/test/client/mock_test_posix_wapper.h" +#include "src/common/concurrent/task_thread_pool.h" + +namespace curvefs { +namespace client { +namespace common { +DECLARE_bool(enableCto); +DECLARE_bool(supportKVcache); +} // namespace common +} // namespace client +} // namespace curvefs + +namespace curvefs { +namespace client { +using curve::common::TaskThreadPool; +using ::testing::_; +using ::testing::DoAll; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::SetArgReferee; +using ::testing::WithArg; + +class FileCacheManagerDiskTest : public testing::Test { + protected: + FileCacheManagerDiskTest() {} + ~FileCacheManagerDiskTest() {} + void SetUp() override { + Aws::InitAPI(awsOptions_); + uint64_t inodeId = 1; + uint64_t fsId = 2; + std::string fsName = "test"; + S3ClientAdaptorOption option; + option.blockSize = 1 * 1024 * 1024; + option.chunkSize = 4 * 1024 * 1024; + option.baseSleepUs = 500; + option.objectPrefix = 0; + option.pageSize = 64 * 1024; + option.intervalSec = 5000; + option.flushIntervalSec = 5000; + option.readCacheMaxByte = 104857600; + option.writeCacheMaxByte = 10485760000; + option.readCacheThreads = 5; + option.diskCacheOpt.diskCacheType = (DiskCacheType)2; + option.chunkFlushThreads = 5; + option.s3ToLocal = true; + s3ClientAdaptor_ = new S3ClientAdaptorImpl(); + fsCacheManager_ = std::make_shared( + s3ClientAdaptor_, option.readCacheMaxByte, option.writeCacheMaxByte, + option.readCacheThreads, nullptr); + mockInodeManager_ = std::make_shared(); + mockS3Client_ = std::make_shared(); + std::shared_ptr client = std::make_shared(); + std::shared_ptr wrapper = + std::make_shared(); + std::shared_ptr diskCacheWrite = + std::make_shared(); + std::shared_ptr diskCacheRead = + std::make_shared(); + std::shared_ptr diskCacheManager = + std::make_shared(wrapper, diskCacheWrite, + diskCacheRead); + mockDiskcacheManagerImpl_ = + std::make_shared(); + mockKVClient_ = std::make_shared(); + KVClientManagerOpt config; + kvClientManager_ = std::make_shared(); + kvClientManager_->Init(config, mockKVClient_, fsName); + s3ClientAdaptor_->Init(option, mockS3Client_, mockInodeManager_, + nullptr, fsCacheManager_, + mockDiskcacheManagerImpl_, kvClientManager_); + s3ClientAdaptor_->SetFsId(fsId); + + threadPool_->Start(option.readCacheThreads); + fileCacheManager_ = std::make_shared( + fsId, inodeId, s3ClientAdaptor_, kvClientManager_, threadPool_); + mockChunkCacheManager_ = std::make_shared(); + curvefs::client::common::FLAGS_enableCto = false; + } + + void TearDown() override { + Aws::ShutdownAPI(awsOptions_); + delete s3ClientAdaptor_; + s3ClientAdaptor_ = nullptr; + } + + protected: + Aws::SDKOptions awsOptions_; + S3ClientAdaptorImpl* s3ClientAdaptor_; + std::shared_ptr fileCacheManager_; + std::shared_ptr fsCacheManager_; + std::shared_ptr mockChunkCacheManager_; + std::shared_ptr mockInodeManager_; + std::shared_ptr mockS3Client_; + + std::shared_ptr kvClientManager_; + std::shared_ptr mockKVClient_; + std::shared_ptr mockDiskcacheManagerImpl_; + + std::shared_ptr> threadPool_ = + std::make_shared>(); +}; + +TEST_F(FileCacheManagerDiskTest, test_read_local_prefetchfail) { + const uint64_t inodeId = 1; + const uint64_t offset = 0; + const uint64_t len = 1024; + + std::vector buf(len); + std::vector tmpBuf(len, 'a'); + ReadRequest req{.index = 0, .chunkPos = offset, .len = len, .bufOffset = 0}; + std::vector requests{req}; + EXPECT_CALL(*mockChunkCacheManager_, ReadByWriteCache(_, _, _, _, _)) + .WillOnce(DoAll(SetArgPointee<4>(requests), Return())); + EXPECT_CALL(*mockChunkCacheManager_, ReadByReadCache(_, _, _, _, _)) + .WillOnce(DoAll(SetArgPointee<4>(requests), Return())); + EXPECT_CALL(*mockChunkCacheManager_, AddReadDataCache(_)) + .WillOnce(Return()); + fileCacheManager_->SetChunkCacheManagerForTest(0, mockChunkCacheManager_); + Inode inode; + inode.set_length(len); + auto* s3ChunkInfoMap = inode.mutable_s3chunkinfomap(); + auto* s3ChunkInfoList = new S3ChunkInfoList(); + auto* s3ChunkInfo = s3ChunkInfoList->add_s3chunks(); + s3ChunkInfo->set_chunkid(25); + s3ChunkInfo->set_compaction(0); + s3ChunkInfo->set_offset(offset); + s3ChunkInfo->set_len(len); + s3ChunkInfo->set_size(len); + s3ChunkInfo->set_zero(false); + s3ChunkInfoMap->insert({0, *s3ChunkInfoList}); + fsCacheManager_->SetFileCacheManagerForTest(inodeId, fileCacheManager_); + auto inodeWrapper = std::make_shared(inode, nullptr); + EXPECT_CALL(*mockInodeManager_, GetInode(_, _)) + .WillOnce( + DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))); + EXPECT_CALL(*mockS3Client_, DownloadAsync(_)) + .WillRepeatedly( + Invoke([&](const std::shared_ptr& context) { + context->retCode = 0; + context->cb(nullptr, context); + })); + EXPECT_CALL(*mockDiskcacheManagerImpl_, IsCached(_)) + .WillOnce(Return(false)) + .WillOnce(Return(false)) + .WillOnce(Return(false)) + .WillOnce(Return(true)); + EXPECT_CALL(*mockKVClient_, Get(_, _, _, _, _, _, _)) + .WillOnce(Return(true)); + EXPECT_CALL(*mockDiskcacheManagerImpl_, Read(_, _, _, _)) + .WillOnce(Return(len)); + EXPECT_CALL(*mockDiskcacheManagerImpl_, WriteReadDirect(_, _, _)) + .WillOnce(Return(-1)); + ASSERT_EQ(len, fileCacheManager_->Read(inodeId, offset, len, buf.data())); + sleep(3); +} + +TEST_F(FileCacheManagerDiskTest, test_read_local_prefetchsuc) { + const uint64_t inodeId = 1; + const uint64_t offset = 0; + const uint64_t len = 1024; + + std::vector buf(len); + std::vector tmpBuf(len, 'a'); + + ReadRequest req{.index = 0, .chunkPos = offset, .len = len, .bufOffset = 0}; + std::vector requests{req}; + EXPECT_CALL(*mockChunkCacheManager_, ReadByWriteCache(_, _, _, _, _)) + .WillOnce(DoAll(SetArgPointee<4>(requests), Return())); + EXPECT_CALL(*mockChunkCacheManager_, ReadByReadCache(_, _, _, _, _)) + .WillOnce(DoAll(SetArgPointee<4>(requests), Return())); + EXPECT_CALL(*mockChunkCacheManager_, AddReadDataCache(_)) + .WillOnce(Return()); + fileCacheManager_->SetChunkCacheManagerForTest(0, mockChunkCacheManager_); + Inode inode; + inode.set_length(len); + auto* s3ChunkInfoMap = inode.mutable_s3chunkinfomap(); + auto* s3ChunkInfoList = new S3ChunkInfoList(); + auto* s3ChunkInfo = s3ChunkInfoList->add_s3chunks(); + s3ChunkInfo->set_chunkid(25); + s3ChunkInfo->set_compaction(0); + s3ChunkInfo->set_offset(offset); + s3ChunkInfo->set_len(len); + s3ChunkInfo->set_size(len); + s3ChunkInfo->set_zero(false); + s3ChunkInfoMap->insert({0, *s3ChunkInfoList}); + + fsCacheManager_->SetFileCacheManagerForTest(inodeId, fileCacheManager_); + auto inodeWrapper = std::make_shared(inode, nullptr); + EXPECT_CALL(*mockInodeManager_, GetInode(_, _)) + .WillOnce( + DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))); + EXPECT_CALL(*mockS3Client_, DownloadAsync(_)) + .WillRepeatedly( + Invoke([&](const std::shared_ptr& context) { + context->retCode = 0; + context->cb(nullptr, context); + })); + EXPECT_CALL(*mockDiskcacheManagerImpl_, IsCached(_)) + .WillOnce(Return(false)) + .WillOnce(Return(false)) + .WillOnce(Return(false)) + .WillOnce(Return(true)); + EXPECT_CALL(*mockDiskcacheManagerImpl_, Read(_, _, _, _)) + .WillOnce(Return(len)); + EXPECT_CALL(*mockDiskcacheManagerImpl_, WriteReadDirect(_, _, _)) + .WillOnce(Return(0)); + EXPECT_CALL(*mockKVClient_, Get(_, _, _, _, _, _, _)) + .WillOnce(Return(true)); + ASSERT_EQ(len, fileCacheManager_->Read(inodeId, offset, len, buf.data())); + sleep(3); +} + +TEST_F(FileCacheManagerDiskTest, test_read_remote) { + const uint64_t inodeId = 1; + const uint64_t offset = 0; + const uint64_t len = 1024; + + std::vector buf(len); + std::vector tmpBuf(len, 'a'); + + ReadRequest req{.index = 0, .chunkPos = offset, .len = len, .bufOffset = 0}; + std::vector requests{req}; + EXPECT_CALL(*mockChunkCacheManager_, ReadByWriteCache(_, _, _, _, _)) + .WillOnce(DoAll(SetArgPointee<4>(requests), Return())); + EXPECT_CALL(*mockChunkCacheManager_, ReadByReadCache(_, _, _, _, _)) + .WillOnce(DoAll(SetArgPointee<4>(requests), Return())); + EXPECT_CALL(*mockChunkCacheManager_, AddReadDataCache(_)) + .WillOnce(Return()); + fileCacheManager_->SetChunkCacheManagerForTest(0, mockChunkCacheManager_); + Inode inode; + inode.set_length(len); + auto* s3ChunkInfoMap = inode.mutable_s3chunkinfomap(); + auto* s3ChunkInfoList = new S3ChunkInfoList(); + auto* s3ChunkInfo = s3ChunkInfoList->add_s3chunks(); + s3ChunkInfo->set_chunkid(25); + s3ChunkInfo->set_compaction(0); + s3ChunkInfo->set_offset(offset); + s3ChunkInfo->set_len(len); + s3ChunkInfo->set_size(len); + s3ChunkInfo->set_zero(false); + s3ChunkInfoMap->insert({0, *s3ChunkInfoList}); + + fsCacheManager_->SetFileCacheManagerForTest(inodeId, fileCacheManager_); + auto inodeWrapper = std::make_shared(inode, nullptr); + EXPECT_CALL(*mockInodeManager_, GetInode(_, _)) + .WillOnce( + DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))); + EXPECT_CALL(*mockS3Client_, DownloadAsync(_)) + .WillRepeatedly( + Invoke([&](const std::shared_ptr& context) { + context->retCode = 0; + context->cb(nullptr, context); + })); + EXPECT_CALL(*mockDiskcacheManagerImpl_, IsCached(_)) + .WillOnce(Return(false)) + .WillOnce(Return(false)) + .WillOnce(Return(false)) + .WillOnce(Return(false)); + + EXPECT_CALL(*mockDiskcacheManagerImpl_, WriteReadDirect(_, _, _)) + .WillOnce(Return(0)); + + EXPECT_CALL(*mockKVClient_, Get(_, _, _, _, _, _, _)) + .WillOnce(Return(true)) + .WillOnce(Return(true)); + + ASSERT_EQ(len, fileCacheManager_->Read(inodeId, offset, len, buf.data())); + sleep(3); +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/client/mock_disk_cache_manager.h b/curvefs/test/client/mock_disk_cache_manager.h index c78a7ff5a1..ef66ab55c0 100644 --- a/curvefs/test/client/mock_disk_cache_manager.h +++ b/curvefs/test/client/mock_disk_cache_manager.h @@ -81,9 +81,17 @@ class MockDiskCacheManagerImpl : public DiskCacheManagerImpl { } ~MockDiskCacheManagerImpl() {} - MOCK_METHOD1(UploadWriteCacheByInode, int(const std::string &inode)); - MOCK_METHOD1(ClearReadCache, int(const std::list &files)); + MOCK_METHOD1(Init, int(const S3ClientAdaptorOption option)); + MOCK_METHOD1(UploadWriteCacheByInode, int(const std::string& inode)); + MOCK_METHOD1(ClearReadCache, int(const std::list& files)); MOCK_METHOD1(IsCached, bool(const std::string)); + MOCK_METHOD0(UmountDiskCache, int()); + MOCK_METHOD3(WriteReadDirect, int(const std::string fileName, + const char* buf, uint64_t length)); + MOCK_METHOD3(Write, + int(const std::string name, const char* buf, uint64_t length)); + MOCK_METHOD4(Read, int(const std::string name, char* buf, uint64_t offset, + uint64_t length)); }; } // namespace client From 5f76eafcde7655bb541bff22fec4a214692b32b6 Mon Sep 17 00:00:00 2001 From: Cyber-SiKu Date: Thu, 19 Oct 2023 21:10:46 +0800 Subject: [PATCH 07/27] [feat]curvefs/client: split io attr/data latency Signed-off-by: Cyber-SiKu --- .../provisioning/dashboards/client.json | 1046 +++++++++++++---- curvefs/src/client/fuse_s3_client.cpp | 35 +- curvefs/src/client/fuse_s3_client.h | 6 +- curvefs/src/client/metric/client_metric.cpp | 3 + curvefs/src/client/metric/client_metric.h | 19 + 5 files changed, 876 insertions(+), 233 deletions(-) diff --git a/curvefs/monitor/grafana/provisioning/dashboards/client.json b/curvefs/monitor/grafana/provisioning/dashboards/client.json index 2bdfcddad8..5303f00661 100644 --- a/curvefs/monitor/grafana/provisioning/dashboards/client.json +++ b/curvefs/monitor/grafana/provisioning/dashboards/client.json @@ -37,6 +37,620 @@ "x": 0, "y": 0 }, + "id": 236, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fuse s3 client FuseOpRead latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 237, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_client_[[fs:regex]]_user_read_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fuse s3 client FuseOpWrite latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 238, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_client_[[fs:regex]]_user_write_lat_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fuse s3 client FuseOpRead read attr latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 239, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_fuse_s3_client_[[fs:regex]]_read_attr_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read attr latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fuse s3 client FuseOpRead write attr latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 240, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_fuse_s3_client_[[fs:regex]]_write_attr_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write attr latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fuse s3 client FuseOpRead read data latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 241, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_fuse_s3_client_[[fs:regex]]_read_data_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "read data latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "fuse s3 client FuseOpRead write data latency", + "fieldConfig": { + "defaults": { + "unit": "µs" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "hiddenSeries": false, + "id": 242, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "{__name__=~\"curvefs_fuse_s3_client_[[fs:regex]]_write_data_[[quantile:regex]]\", instance=~\"$instance\"}", + "hide": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "write data latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:212", + "format": "µs", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:213", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "fuse s3 client", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, "id": 208, "panels": [ { @@ -60,7 +674,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 41 }, "hiddenSeries": false, "id": 116, @@ -156,7 +770,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 41 }, "hiddenSeries": false, "id": 117, @@ -252,7 +866,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 49 }, "hiddenSeries": false, "id": 118, @@ -348,7 +962,7 @@ "h": 8, "w": 12, "x": 12, - "y": 33 + "y": 49 }, "hiddenSeries": false, "id": 119, @@ -433,7 +1047,7 @@ "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 2 }, "id": 201, "panels": [ @@ -458,7 +1072,7 @@ "h": 8, "w": 12, "x": 0, - "y": 2 + "y": 18 }, "hiddenSeries": false, "id": 202, @@ -556,7 +1170,7 @@ "h": 8, "w": 12, "x": 12, - "y": 2 + "y": 18 }, "hiddenSeries": false, "id": 203, @@ -654,7 +1268,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 26 }, "hiddenSeries": false, "id": 204, @@ -752,7 +1366,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 26 }, "hiddenSeries": false, "id": 205, @@ -850,7 +1464,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 34 }, "hiddenSeries": false, "id": 206, @@ -949,7 +1563,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 34 }, "hiddenSeries": false, "id": 207, @@ -1037,7 +1651,7 @@ "h": 1, "w": 24, "x": 0, - "y": 2 + "y": 3 }, "id": 194, "panels": [ @@ -1062,7 +1676,7 @@ "h": 8, "w": 12, "x": 0, - "y": 3 + "y": 19 }, "hiddenSeries": false, "id": 195, @@ -1160,7 +1774,7 @@ "h": 8, "w": 12, "x": 12, - "y": 3 + "y": 19 }, "hiddenSeries": false, "id": 196, @@ -1258,7 +1872,7 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 27 }, "hiddenSeries": false, "id": 197, @@ -1356,7 +1970,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 27 }, "hiddenSeries": false, "id": 198, @@ -1454,7 +2068,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 35 }, "hiddenSeries": false, "id": 199, @@ -1553,7 +2167,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 35 }, "hiddenSeries": false, "id": 200, @@ -1641,7 +2255,7 @@ "h": 1, "w": 24, "x": 0, - "y": 3 + "y": 4 }, "id": 187, "panels": [ @@ -1666,7 +2280,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 20 }, "hiddenSeries": false, "id": 188, @@ -1764,7 +2378,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 20 }, "hiddenSeries": false, "id": 189, @@ -1862,7 +2476,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 28 }, "hiddenSeries": false, "id": 190, @@ -1960,7 +2574,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 28 }, "hiddenSeries": false, "id": 191, @@ -2058,7 +2672,7 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 36 }, "hiddenSeries": false, "id": 192, @@ -2157,7 +2771,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 36 }, "hiddenSeries": false, "id": 193, @@ -2249,7 +2863,7 @@ "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 5 }, "id": 6, "panels": [ @@ -2268,7 +2882,7 @@ "h": 8, "w": 12, "x": 0, - "y": 193 + "y": 209 }, "hiddenSeries": false, "id": 4, @@ -2382,7 +2996,7 @@ "h": 8, "w": 12, "x": 12, - "y": 193 + "y": 209 }, "hiddenSeries": false, "id": 2, @@ -2505,7 +3119,7 @@ "h": 1, "w": 24, "x": 0, - "y": 5 + "y": 6 }, "id": 85, "panels": [ @@ -2530,7 +3144,7 @@ "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 15 }, "hiddenSeries": false, "id": 107, @@ -2624,7 +3238,7 @@ "h": 8, "w": 12, "x": 12, - "y": 7 + "y": 15 }, "hiddenSeries": false, "id": 108, @@ -2718,7 +3332,7 @@ "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 23 }, "hiddenSeries": false, "id": 87, @@ -2812,7 +3426,7 @@ "h": 8, "w": 12, "x": 12, - "y": 15 + "y": 23 }, "hiddenSeries": false, "id": 91, @@ -2906,7 +3520,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 31 }, "hiddenSeries": false, "id": 88, @@ -3000,7 +3614,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 31 }, "hiddenSeries": false, "id": 92, @@ -3094,7 +3708,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 39 }, "hiddenSeries": false, "id": 89, @@ -3188,7 +3802,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 39 }, "hiddenSeries": false, "id": 90, @@ -3282,7 +3896,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 47 }, "hiddenSeries": false, "id": 93, @@ -3376,7 +3990,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 47 }, "hiddenSeries": false, "id": 94, @@ -3470,7 +4084,7 @@ "h": 8, "w": 12, "x": 0, - "y": 47 + "y": 55 }, "hiddenSeries": false, "id": 99, @@ -3564,7 +4178,7 @@ "h": 8, "w": 12, "x": 12, - "y": 47 + "y": 55 }, "hiddenSeries": false, "id": 100, @@ -3658,7 +4272,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 63 }, "hiddenSeries": false, "id": 95, @@ -3752,7 +4366,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 63 }, "hiddenSeries": false, "id": 96, @@ -3846,7 +4460,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 71 }, "hiddenSeries": false, "id": 97, @@ -3940,7 +4554,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 71 }, "hiddenSeries": false, "id": 98, @@ -4034,7 +4648,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 79 }, "hiddenSeries": false, "id": 101, @@ -4128,7 +4742,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 79 }, "hiddenSeries": false, "id": 102, @@ -4222,7 +4836,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 87 }, "hiddenSeries": false, "id": 103, @@ -4316,7 +4930,7 @@ "h": 8, "w": 12, "x": 12, - "y": 79 + "y": 87 }, "hiddenSeries": false, "id": 104, @@ -4410,7 +5024,7 @@ "h": 8, "w": 12, "x": 0, - "y": 87 + "y": 95 }, "hiddenSeries": false, "id": 105, @@ -4504,7 +5118,7 @@ "h": 8, "w": 12, "x": 12, - "y": 87 + "y": 95 }, "hiddenSeries": false, "id": 106, @@ -4598,7 +5212,7 @@ "h": 8, "w": 12, "x": 0, - "y": 95 + "y": 103 }, "hiddenSeries": false, "id": 109, @@ -4694,7 +5308,7 @@ "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 7 }, "id": 176, "panels": [ @@ -4719,7 +5333,7 @@ "h": 8, "w": 24, "x": 0, - "y": 235 + "y": 251 }, "hiddenSeries": false, "id": 177, @@ -4815,7 +5429,7 @@ "h": 1, "w": 24, "x": 0, - "y": 7 + "y": 8 }, "id": 47, "panels": [ @@ -4840,7 +5454,7 @@ "h": 8, "w": 12, "x": 0, - "y": 196 + "y": 17 }, "hiddenSeries": false, "id": 48, @@ -4862,7 +5476,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4936,7 +5550,7 @@ "h": 8, "w": 12, "x": 12, - "y": 196 + "y": 17 }, "hiddenSeries": false, "id": 49, @@ -4958,7 +5572,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5032,7 +5646,7 @@ "h": 8, "w": 12, "x": 0, - "y": 204 + "y": 25 }, "hiddenSeries": false, "id": 50, @@ -5054,7 +5668,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5128,7 +5742,7 @@ "h": 8, "w": 12, "x": 12, - "y": 204 + "y": 25 }, "hiddenSeries": false, "id": 51, @@ -5150,7 +5764,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5224,7 +5838,7 @@ "h": 8, "w": 12, "x": 0, - "y": 212 + "y": 33 }, "hiddenSeries": false, "id": 52, @@ -5246,7 +5860,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5314,7 +5928,7 @@ "h": 8, "w": 12, "x": 12, - "y": 212 + "y": 33 }, "hiddenSeries": false, "id": 53, @@ -5336,7 +5950,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5410,7 +6024,7 @@ "h": 8, "w": 12, "x": 0, - "y": 220 + "y": 41 }, "hiddenSeries": false, "id": 54, @@ -5432,7 +6046,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5506,7 +6120,7 @@ "h": 8, "w": 12, "x": 12, - "y": 220 + "y": 41 }, "hiddenSeries": false, "id": 43, @@ -5528,7 +6142,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5602,7 +6216,7 @@ "h": 8, "w": 12, "x": 0, - "y": 228 + "y": 49 }, "hiddenSeries": false, "id": 56, @@ -5624,7 +6238,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5698,7 +6312,7 @@ "h": 8, "w": 12, "x": 12, - "y": 228 + "y": 49 }, "hiddenSeries": false, "id": 57, @@ -5720,7 +6334,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5794,7 +6408,7 @@ "h": 8, "w": 12, "x": 0, - "y": 236 + "y": 57 }, "hiddenSeries": false, "id": 58, @@ -5816,7 +6430,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5891,7 +6505,7 @@ "h": 8, "w": 12, "x": 12, - "y": 236 + "y": 57 }, "hiddenSeries": false, "id": 59, @@ -5913,7 +6527,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.0.3", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5990,7 +6604,7 @@ "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 9 }, "id": 8, "panels": [ @@ -6015,7 +6629,7 @@ "h": 8, "w": 12, "x": 0, - "y": 41 + "y": 57 }, "hiddenSeries": false, "id": 10, @@ -6111,7 +6725,7 @@ "h": 8, "w": 12, "x": 12, - "y": 41 + "y": 57 }, "hiddenSeries": false, "id": 11, @@ -6207,7 +6821,7 @@ "h": 8, "w": 12, "x": 0, - "y": 49 + "y": 65 }, "hiddenSeries": false, "id": 12, @@ -6303,7 +6917,7 @@ "h": 8, "w": 12, "x": 12, - "y": 49 + "y": 65 }, "hiddenSeries": false, "id": 13, @@ -6399,7 +7013,7 @@ "h": 8, "w": 12, "x": 0, - "y": 57 + "y": 73 }, "hiddenSeries": false, "id": 14, @@ -6489,7 +7103,7 @@ "h": 8, "w": 12, "x": 12, - "y": 57 + "y": 73 }, "hiddenSeries": false, "id": 15, @@ -6585,7 +7199,7 @@ "h": 8, "w": 12, "x": 0, - "y": 65 + "y": 81 }, "hiddenSeries": false, "id": 42, @@ -6681,7 +7295,7 @@ "h": 8, "w": 12, "x": 12, - "y": 65 + "y": 81 }, "hiddenSeries": false, "id": 55, @@ -6777,7 +7391,7 @@ "h": 8, "w": 12, "x": 0, - "y": 73 + "y": 89 }, "hiddenSeries": false, "id": 60, @@ -6873,7 +7487,7 @@ "h": 8, "w": 12, "x": 12, - "y": 73 + "y": 89 }, "hiddenSeries": false, "id": 61, @@ -6969,7 +7583,7 @@ "h": 8, "w": 12, "x": 0, - "y": 81 + "y": 97 }, "hiddenSeries": false, "id": 26, @@ -7066,7 +7680,7 @@ "h": 8, "w": 12, "x": 12, - "y": 81 + "y": 97 }, "hiddenSeries": false, "id": 27, @@ -7165,7 +7779,7 @@ "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 10 }, "id": 72, "panels": [ @@ -7190,7 +7804,7 @@ "h": 8, "w": 12, "x": 0, - "y": 238 + "y": 254 }, "hiddenSeries": false, "id": 73, @@ -7286,7 +7900,7 @@ "h": 8, "w": 12, "x": 12, - "y": 238 + "y": 254 }, "hiddenSeries": false, "id": 74, @@ -7382,7 +7996,7 @@ "h": 8, "w": 12, "x": 0, - "y": 246 + "y": 262 }, "hiddenSeries": false, "id": 75, @@ -7478,7 +8092,7 @@ "h": 8, "w": 12, "x": 12, - "y": 246 + "y": 262 }, "hiddenSeries": false, "id": 76, @@ -7574,7 +8188,7 @@ "h": 8, "w": 12, "x": 0, - "y": 254 + "y": 270 }, "hiddenSeries": false, "id": 77, @@ -7671,7 +8285,7 @@ "h": 8, "w": 12, "x": 12, - "y": 254 + "y": 270 }, "hiddenSeries": false, "id": 78, @@ -7770,7 +8384,7 @@ "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 11 }, "id": 69, "panels": [ @@ -7795,7 +8409,7 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 113 }, "hiddenSeries": false, "id": 62, @@ -7891,7 +8505,7 @@ "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 113 }, "hiddenSeries": false, "id": 63, @@ -7987,7 +8601,7 @@ "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 121 }, "hiddenSeries": false, "id": 66, @@ -8083,7 +8697,7 @@ "h": 8, "w": 12, "x": 12, - "y": 105 + "y": 121 }, "hiddenSeries": false, "id": 65, @@ -8179,7 +8793,7 @@ "h": 8, "w": 12, "x": 0, - "y": 113 + "y": 129 }, "hiddenSeries": false, "id": 64, @@ -8276,7 +8890,7 @@ "h": 8, "w": 12, "x": 12, - "y": 113 + "y": 129 }, "hiddenSeries": false, "id": 67, @@ -8375,7 +8989,7 @@ "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 12 }, "id": 80, "panels": [ @@ -8400,7 +9014,7 @@ "h": 8, "w": 12, "x": 0, - "y": 288 + "y": 304 }, "hiddenSeries": false, "id": 81, @@ -8496,7 +9110,7 @@ "h": 8, "w": 12, "x": 12, - "y": 288 + "y": 304 }, "hiddenSeries": false, "id": 82, @@ -8592,7 +9206,7 @@ "h": 8, "w": 12, "x": 0, - "y": 296 + "y": 312 }, "hiddenSeries": false, "id": 83, @@ -8691,7 +9305,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "id": 111, "panels": [ @@ -8716,7 +9330,7 @@ "h": 8, "w": 12, "x": 0, - "y": 305 + "y": 321 }, "hiddenSeries": false, "id": 112, @@ -8812,7 +9426,7 @@ "h": 8, "w": 12, "x": 12, - "y": 305 + "y": 321 }, "hiddenSeries": false, "id": 113, @@ -8910,7 +9524,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 14 }, "id": 121, "panels": [ @@ -8935,7 +9549,7 @@ "h": 8, "w": 12, "x": 0, - "y": 47 + "y": 63 }, "hiddenSeries": false, "id": 122, @@ -9031,7 +9645,7 @@ "h": 8, "w": 12, "x": 12, - "y": 47 + "y": 63 }, "hiddenSeries": false, "id": 123, @@ -9127,7 +9741,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 71 }, "hiddenSeries": false, "id": 124, @@ -9223,7 +9837,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 71 }, "hiddenSeries": false, "id": 125, @@ -9319,7 +9933,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 79 }, "hiddenSeries": false, "id": 126, @@ -9415,7 +10029,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 79 }, "hiddenSeries": false, "id": 127, @@ -9513,7 +10127,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 15 }, "id": 115, "panels": [ @@ -9539,7 +10153,7 @@ "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 31 }, "hiddenSeries": false, "id": 209, @@ -9638,7 +10252,7 @@ "h": 8, "w": 12, "x": 12, - "y": 15 + "y": 31 }, "hiddenSeries": false, "id": 178, @@ -9737,7 +10351,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 39 }, "hiddenSeries": false, "id": 210, @@ -9836,7 +10450,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 39 }, "hiddenSeries": false, "id": 217, @@ -9933,7 +10547,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 47 }, "hiddenSeries": false, "id": 211, @@ -10032,7 +10646,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 47 }, "hiddenSeries": false, "id": 213, @@ -10157,7 +10771,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 55 }, "hiddenSeries": false, "id": 212, @@ -10256,7 +10870,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 55 }, "hiddenSeries": false, "id": 215, @@ -10355,7 +10969,7 @@ "h": 8, "w": 12, "x": 0, - "y": 47 + "y": 63 }, "hiddenSeries": false, "id": 214, @@ -10454,7 +11068,7 @@ "h": 8, "w": 12, "x": 12, - "y": 47 + "y": 63 }, "hiddenSeries": false, "id": 216, @@ -10550,7 +11164,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 218, "panels": [ @@ -10576,7 +11190,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 32 }, "hiddenSeries": false, "id": 219, @@ -10675,7 +11289,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 32 }, "hiddenSeries": false, "id": 220, @@ -10800,7 +11414,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 40 }, "hiddenSeries": false, "id": 221, @@ -10900,7 +11514,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 40 }, "hiddenSeries": false, "id": 222, @@ -11000,7 +11614,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 48 }, "hiddenSeries": false, "id": 223, @@ -11099,7 +11713,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 48 }, "hiddenSeries": false, "id": 226, @@ -11198,7 +11812,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 56 }, "hiddenSeries": false, "id": 225, @@ -11298,7 +11912,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 56 }, "hiddenSeries": false, "id": 227, @@ -11398,7 +12012,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 64 }, "hiddenSeries": false, "id": 224, @@ -11497,7 +12111,7 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 64 }, "hiddenSeries": false, "id": 228, @@ -11584,7 +12198,7 @@ "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 17 }, "id": 229, "panels": [ @@ -11610,7 +12224,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 33 }, "hiddenSeries": false, "id": 232, @@ -11710,7 +12324,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 33 }, "hiddenSeries": false, "id": 231, @@ -11809,7 +12423,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 41 }, "hiddenSeries": false, "id": 234, @@ -11908,7 +12522,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 41 }, "hiddenSeries": false, "id": 233, @@ -12008,7 +12622,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 49 }, "hiddenSeries": false, "id": 230, @@ -12107,7 +12721,7 @@ "h": 8, "w": 12, "x": 12, - "y": 33 + "y": 49 }, "hiddenSeries": false, "id": 235, @@ -12198,7 +12812,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 18 }, "id": 32, "panels": [ @@ -12223,7 +12837,7 @@ "h": 6, "w": 12, "x": 0, - "y": 364 + "y": 138 }, "hiddenSeries": false, "id": 36, @@ -12245,7 +12859,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -12319,7 +12933,7 @@ "h": 6, "w": 12, "x": 12, - "y": 364 + "y": 138 }, "hiddenSeries": false, "id": 128, @@ -12341,7 +12955,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -12415,7 +13029,7 @@ "h": 6, "w": 12, "x": 0, - "y": 370 + "y": 144 }, "hiddenSeries": false, "id": 129, @@ -12437,7 +13051,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -12511,7 +13125,7 @@ "h": 6, "w": 12, "x": 12, - "y": 370 + "y": 144 }, "hiddenSeries": false, "id": 130, @@ -12533,7 +13147,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -12607,7 +13221,7 @@ "h": 6, "w": 12, "x": 0, - "y": 376 + "y": 150 }, "hiddenSeries": false, "id": 131, @@ -12629,7 +13243,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -12703,7 +13317,7 @@ "h": 6, "w": 12, "x": 12, - "y": 376 + "y": 150 }, "hiddenSeries": false, "id": 132, @@ -12725,7 +13339,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.0.6", + "pluginVersion": "10.1.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -12799,7 +13413,7 @@ "h": 6, "w": 12, "x": 0, - "y": 382 + "y": 156 }, "hiddenSeries": false, "id": 133, @@ -12895,7 +13509,7 @@ "h": 6, "w": 12, "x": 12, - "y": 382 + "y": 156 }, "hiddenSeries": false, "id": 134, @@ -12991,7 +13605,7 @@ "h": 6, "w": 12, "x": 0, - "y": 388 + "y": 162 }, "hiddenSeries": false, "id": 135, @@ -13087,7 +13701,7 @@ "h": 6, "w": 12, "x": 12, - "y": 388 + "y": 162 }, "hiddenSeries": false, "id": 136, @@ -13183,7 +13797,7 @@ "h": 6, "w": 12, "x": 0, - "y": 394 + "y": 168 }, "hiddenSeries": false, "id": 137, @@ -13279,7 +13893,7 @@ "h": 6, "w": 12, "x": 12, - "y": 394 + "y": 168 }, "hiddenSeries": false, "id": 138, @@ -13375,7 +13989,7 @@ "h": 6, "w": 12, "x": 0, - "y": 400 + "y": 174 }, "hiddenSeries": false, "id": 139, @@ -13471,7 +14085,7 @@ "h": 6, "w": 12, "x": 12, - "y": 400 + "y": 174 }, "hiddenSeries": false, "id": 140, @@ -13566,7 +14180,7 @@ "h": 6, "w": 12, "x": 0, - "y": 406 + "y": 180 }, "hiddenSeries": false, "id": 141, @@ -13661,7 +14275,7 @@ "h": 6, "w": 12, "x": 12, - "y": 406 + "y": 180 }, "hiddenSeries": false, "id": 142, @@ -13757,7 +14371,7 @@ "h": 6, "w": 12, "x": 0, - "y": 412 + "y": 186 }, "hiddenSeries": false, "id": 143, @@ -13853,7 +14467,7 @@ "h": 6, "w": 12, "x": 12, - "y": 412 + "y": 186 }, "hiddenSeries": false, "id": 144, @@ -13949,7 +14563,7 @@ "h": 6, "w": 12, "x": 0, - "y": 418 + "y": 192 }, "hiddenSeries": false, "id": 145, @@ -14045,7 +14659,7 @@ "h": 6, "w": 12, "x": 12, - "y": 418 + "y": 192 }, "hiddenSeries": false, "id": 146, @@ -14141,7 +14755,7 @@ "h": 6, "w": 12, "x": 0, - "y": 424 + "y": 198 }, "hiddenSeries": false, "id": 147, @@ -14237,7 +14851,7 @@ "h": 6, "w": 12, "x": 12, - "y": 424 + "y": 198 }, "hiddenSeries": false, "id": 148, @@ -14333,7 +14947,7 @@ "h": 6, "w": 12, "x": 0, - "y": 430 + "y": 204 }, "hiddenSeries": false, "id": 149, @@ -14429,7 +15043,7 @@ "h": 6, "w": 12, "x": 12, - "y": 430 + "y": 204 }, "hiddenSeries": false, "id": 150, @@ -14527,7 +15141,7 @@ "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 19 }, "id": 34, "panels": [ @@ -14552,7 +15166,7 @@ "h": 6, "w": 12, "x": 0, - "y": 50 + "y": 66 }, "hiddenSeries": false, "id": 155, @@ -14648,7 +15262,7 @@ "h": 6, "w": 12, "x": 12, - "y": 50 + "y": 66 }, "hiddenSeries": false, "id": 152, @@ -14744,7 +15358,7 @@ "h": 6, "w": 12, "x": 0, - "y": 56 + "y": 72 }, "hiddenSeries": false, "id": 153, @@ -14840,7 +15454,7 @@ "h": 6, "w": 12, "x": 12, - "y": 56 + "y": 72 }, "hiddenSeries": false, "id": 156, @@ -14936,7 +15550,7 @@ "h": 6, "w": 12, "x": 0, - "y": 62 + "y": 78 }, "hiddenSeries": false, "id": 157, @@ -15032,7 +15646,7 @@ "h": 6, "w": 12, "x": 12, - "y": 62 + "y": 78 }, "hiddenSeries": false, "id": 154, @@ -15128,7 +15742,7 @@ "h": 6, "w": 12, "x": 0, - "y": 68 + "y": 84 }, "hiddenSeries": false, "id": 151, @@ -15224,7 +15838,7 @@ "h": 6, "w": 12, "x": 12, - "y": 68 + "y": 84 }, "hiddenSeries": false, "id": 158, @@ -15320,7 +15934,7 @@ "h": 6, "w": 12, "x": 0, - "y": 74 + "y": 90 }, "hiddenSeries": false, "id": 159, @@ -15416,7 +16030,7 @@ "h": 6, "w": 12, "x": 12, - "y": 74 + "y": 90 }, "hiddenSeries": false, "id": 160, @@ -15512,7 +16126,7 @@ "h": 6, "w": 12, "x": 0, - "y": 80 + "y": 96 }, "hiddenSeries": false, "id": 161, @@ -15608,7 +16222,7 @@ "h": 6, "w": 12, "x": 12, - "y": 80 + "y": 96 }, "hiddenSeries": false, "id": 162, @@ -15704,7 +16318,7 @@ "h": 6, "w": 12, "x": 0, - "y": 86 + "y": 102 }, "hiddenSeries": false, "id": 163, @@ -15800,7 +16414,7 @@ "h": 6, "w": 12, "x": 12, - "y": 86 + "y": 102 }, "hiddenSeries": false, "id": 164, @@ -15896,7 +16510,7 @@ "h": 6, "w": 12, "x": 0, - "y": 92 + "y": 108 }, "hiddenSeries": false, "id": 165, @@ -15992,7 +16606,7 @@ "h": 6, "w": 12, "x": 12, - "y": 92 + "y": 108 }, "hiddenSeries": false, "id": 166, @@ -16088,7 +16702,7 @@ "h": 6, "w": 12, "x": 0, - "y": 98 + "y": 114 }, "hiddenSeries": false, "id": 167, @@ -16184,7 +16798,7 @@ "h": 6, "w": 12, "x": 12, - "y": 98 + "y": 114 }, "hiddenSeries": false, "id": 168, @@ -16280,7 +16894,7 @@ "h": 6, "w": 12, "x": 0, - "y": 104 + "y": 120 }, "hiddenSeries": false, "id": 169, @@ -16376,7 +16990,7 @@ "h": 6, "w": 12, "x": 12, - "y": 104 + "y": 120 }, "hiddenSeries": false, "id": 170, @@ -16472,7 +17086,7 @@ "h": 6, "w": 12, "x": 0, - "y": 110 + "y": 126 }, "hiddenSeries": false, "id": 171, @@ -16568,7 +17182,7 @@ "h": 6, "w": 12, "x": 12, - "y": 110 + "y": 126 }, "hiddenSeries": false, "id": 172, @@ -16664,7 +17278,7 @@ "h": 6, "w": 12, "x": 0, - "y": 116 + "y": 132 }, "hiddenSeries": false, "id": 173, @@ -16760,7 +17374,7 @@ "h": 6, "w": 12, "x": 12, - "y": 116 + "y": 132 }, "hiddenSeries": false, "id": 174, @@ -16854,7 +17468,7 @@ "h": 1, "w": 24, "x": 0, - "y": 19 + "y": 20 }, "id": 180, "panels": [ @@ -16919,7 +17533,7 @@ "h": 8, "w": 12, "x": 0, - "y": 206 + "y": 222 }, "id": 182, "options": { @@ -17014,7 +17628,7 @@ "h": 8, "w": 12, "x": 12, - "y": 206 + "y": 222 }, "id": 184, "options": { @@ -17110,7 +17724,7 @@ "h": 8, "w": 12, "x": 0, - "y": 214 + "y": 230 }, "id": 186, "options": { @@ -17155,13 +17769,9 @@ "list": [ { "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "selected": false, + "text": "All", + "value": "$__all" }, "datasource": { "type": "prometheus", @@ -17186,7 +17796,7 @@ }, { "current": { - "selected": false, + "selected": true, "text": [ "All" ], @@ -17249,7 +17859,7 @@ }, { "current": { - "selected": false, + "selected": true, "text": [ "All" ], diff --git a/curvefs/src/client/fuse_s3_client.cpp b/curvefs/src/client/fuse_s3_client.cpp index 829f00e1d3..5534857bab 100644 --- a/curvefs/src/client/fuse_s3_client.cpp +++ b/curvefs/src/client/fuse_s3_client.cpp @@ -20,11 +20,12 @@ * Author: xuchaojie */ +#include "curvefs/src/client/fuse_s3_client.h" +#include #include #include -#include "curvefs/src/client/fuse_s3_client.h" #include "curvefs/src/client/kvclient/memcache_client.h" namespace curvefs { @@ -100,6 +101,8 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) { inodeManager_, mdsClient_, fsCacheManager, nullptr, kvClientManager_, true); } + ioLatencyMetric_ = absl::make_unique( + fsInfo_->fsname()); return ret; } @@ -169,14 +172,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, LOG(ERROR) << "s3Adaptor_ write failed, ret = " << wRet; return CURVEFS_ERROR::INTERNAL; } - - if (fsMetric_.get() != nullptr) { - fsMetric_->userWrite.bps.count << wRet; - fsMetric_->userWrite.qps.count << 1; - uint64_t duration = butil::cpuwide_time_us() - start; - fsMetric_->userWrite.latency << duration; - fsMetric_->userWriteIoSize.set_value(wRet); - } + uint64_t mid = butil::cpuwide_time_us(); + ioLatencyMetric_->writeDataLatency << mid - start; std::shared_ptr inodeWrapper; CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper); @@ -220,6 +217,15 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, } inodeWrapper->GetInodeAttrLocked(&fileOut->attr); + + if (fsMetric_.get() != nullptr) { + fsMetric_->userWrite.bps.count << wRet; + fsMetric_->userWrite.qps.count << 1; + uint64_t duration = butil::cpuwide_time_us() - start; + fsMetric_->userWrite.latency << duration; + fsMetric_->userWriteIoSize.set_value(wRet); + } + ioLatencyMetric_->writeAttrLatency << butil::cpuwide_time_us() - mid; return ret; } @@ -243,6 +249,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino, << ", inodeid = " << ino; return ret; } + uint64_t mid = butil::cpuwide_time_us(); + ioLatencyMetric_->readAttrLatency << mid - start; uint64_t fileSize = inodeWrapper->GetLength(); size_t len = 0; @@ -262,6 +270,11 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino, return CURVEFS_ERROR::INTERNAL; } *rSize = rRet; + ioLatencyMetric_->readDataLatency << butil::cpuwide_time_us() - mid; + + ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock(); + inodeWrapper->UpdateTimestampLocked(kAccessTime); + inodeManager_->ShipToFlush(inodeWrapper); if (fsMetric_.get() != nullptr) { fsMetric_->userRead.bps.count << rRet; @@ -271,10 +284,6 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino, fsMetric_->userReadIoSize.set_value(rRet); } - ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock(); - inodeWrapper->UpdateTimestampLocked(kAccessTime); - inodeManager_->ShipToFlush(inodeWrapper); - VLOG(9) << "read end, read size = " << *rSize; return ret; } diff --git a/curvefs/src/client/fuse_s3_client.h b/curvefs/src/client/fuse_s3_client.h index cd1154beb8..1b4b4a70b4 100644 --- a/curvefs/src/client/fuse_s3_client.h +++ b/curvefs/src/client/fuse_s3_client.h @@ -24,13 +24,14 @@ #ifndef CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_ #define CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_ +#include #include #include -#include -#include #include +#include #include "curvefs/src/client/fuse_client.h" +#include "curvefs/src/client/metric/client_metric.h" #include "curvefs/src/client/s3/client_s3_cache_manager.h" #include "curvefs/src/client/warmup/warmup_manager.h" #include "curvefs/src/volume/common.h" @@ -133,6 +134,7 @@ class FuseS3Client : public FuseClient { // s3 adaptor std::shared_ptr s3Adaptor_; std::shared_ptr kvClientManager_; + std::unique_ptr ioLatencyMetric_; static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB; }; diff --git a/curvefs/src/client/metric/client_metric.cpp b/curvefs/src/client/metric/client_metric.cpp index f97e7ef87a..d6c2592ebb 100644 --- a/curvefs/src/client/metric/client_metric.cpp +++ b/curvefs/src/client/metric/client_metric.cpp @@ -23,6 +23,7 @@ #include "curvefs/src/client/metric/client_metric.h" #include +#include #include "src/client/client_metric.h" @@ -45,6 +46,8 @@ const std::string MemcacheClientMetric::prefix = // NOLINT "curvefs_memcache_client"; // NOLINT const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info"; // NOLINT const std::string WarmupManagerS3Metric::prefix = "curvefs_warmup"; // NOLINT +const std::string FuseS3ClientIOLatencyMetric::prefix = // NOLINT + "curvefs_fuse_s3_client"; // NOLINT void AsyncContextCollectMetrics( std::shared_ptr s3Metric, diff --git a/curvefs/src/client/metric/client_metric.h b/curvefs/src/client/metric/client_metric.h index e5819cfa8a..58a23fa84b 100644 --- a/curvefs/src/client/metric/client_metric.h +++ b/curvefs/src/client/metric/client_metric.h @@ -368,6 +368,25 @@ void AsyncContextCollectMetrics( std::shared_ptr s3Metric, const std::shared_ptr& context); +struct FuseS3ClientIOLatencyMetric { + static const std::string prefix; + + std::string fsName; + + bvar::LatencyRecorder readAttrLatency; + bvar::LatencyRecorder readDataLatency; + bvar::LatencyRecorder writeAttrLatency; + bvar::LatencyRecorder writeDataLatency; + + explicit FuseS3ClientIOLatencyMetric(const std::string& name = "") + : fsName(!name.empty() ? name + : prefix + curve::common::ToHexString(this)), + readAttrLatency(prefix, fsName + "_read_attr_latency"), + readDataLatency(prefix, fsName + "_read_data_latency"), + writeAttrLatency(prefix, fsName + "_write_attr_latency"), + writeDataLatency(prefix, fsName + "_write_data_latency") {} +}; + } // namespace metric } // namespace client } // namespace curvefs From 7819a31ed2bbc151aebf19216fd5961599ba7063 Mon Sep 17 00:00:00 2001 From: Cyber-SiKu Date: Mon, 23 Oct 2023 17:15:08 +0800 Subject: [PATCH 08/27] [feat]curvefs/monitor: support plugin Signed-off-by: Cyber-SiKu --- curvefs/monitor/target_json.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/curvefs/monitor/target_json.py b/curvefs/monitor/target_json.py index 3b36e2e617..ebb726229d 100644 --- a/curvefs/monitor/target_json.py +++ b/curvefs/monitor/target_json.py @@ -11,6 +11,7 @@ CURVEFS_TOOL = "curvefs_tool" JSON_PATH = "/tmp/topology.json" +PLUGIN_PATH = "plugin" HOSTNAME_PORT_REGEX = r"[^\"\ ]\S*:\d+" IP_PORT_REGEX = r"[0-9]+(?:\.[0-9]+){3}:\d+" @@ -97,6 +98,16 @@ def unitValue(lables, targets): unit["targets"] = targets return unit +def loadPlugin(): + # load *.json file in plugin dir + # merge to one json + data = [] + for filename in os.listdir(PLUGIN_PATH): + if filename.endswith('.json'): + with open(os.path.join(PLUGIN_PATH, filename)) as f: + plugin_data = json.load(f) + data.append(unitValue(plugin_data["labels"], plugin_data["targets"]) + return data def refresh(): targets = [] @@ -113,6 +124,8 @@ def refresh(): # load client client = loadClient() targets.append(client) + plugin = loadPlugin() + targets += plugin with open(targetPath+'.new', 'w', 0o777) as fd: json.dump(targets, fd, indent=4) From 0059b9bde018359bc729f64a80fdab11a0a9b88a Mon Sep 17 00:00:00 2001 From: quasdo <211250106@smail.nju.edu.cn> Date: Fri, 20 Oct 2023 17:44:18 +0800 Subject: [PATCH 09/27] Build: the completion of specified cc_library Signed-off-by: quas-modo <690164387@qq.com> Signed-off-by: quasdo <211250106@smail.nju.edu.cn> --- .obm.cfg | 2 +- Makefile | 30 +++- thirdparties/etcdclient/Makefile | 6 +- util/basic.sh | 6 + util/build.sh | 92 +++--------- util/build_functions.sh | 241 ++++++++++++++++++++++++++++++ util/build_in_image.sh | 243 +------------------------------ util/playground.sh | 27 +++- 8 files changed, 324 insertions(+), 323 deletions(-) mode change 100755 => 100644 util/build.sh create mode 100644 util/build_functions.sh mode change 100755 => 100644 util/build_in_image.sh diff --git a/.obm.cfg b/.obm.cfg index 93bee71c49..012870a5fd 100644 --- a/.obm.cfg +++ b/.obm.cfg @@ -1,2 +1,2 @@ container_name: curve-build-playground-master -container_image: opencurvedocker/curve-base:build-debian9 +container_image: opencurvedocker/curve-base:build-debian11 diff --git a/Makefile b/Makefile index 807b0a78c7..c71b5ab045 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. -.PHONY: list build dep install image playground check test docker +.PHONY: list build dep ci-list ci-build ci-dep install image playground check test docker format stor?="" prefix?= "$(PWD)/projects" @@ -13,6 +13,14 @@ os?= "debian11" ci?=0 commit_id="HEAD^" define help_msg +## build curvebs + make build stor=bs dep=1 + make dep stor=bs && make build stor=bs + +## build curvefs + make build stor=fs dep=1 + make dep stor=fs && make build stor=fs + ## list Usage: make list stor=bs/fs @@ -25,17 +33,21 @@ Usage: Examples: make build stor=bs only=//src/chunkserver:chunkserver make build stor=bs only=src/*,test/* dep=0 - make build stor=fs only=test/* os=debian9 + make build stor=fs only=test/* os=debian11 + make build stor=fs release=1 Note: Extra build options can be specified using BUILD_OPTS environment variable, which will be passed to bazel build command. ## dep +## configure dependency(before build) Usage: make dep stor=bs/fs Examples: make dep stor=bs +## ci-list/build/dep +## use the same way above, but in the container ## install Usage: @@ -50,7 +62,7 @@ Examples: Usage: make image stor=bs/fs tag=TAG os=OS Examples: - make image stor=bs tag=opencurvedocker/curvebs:v1.2 os=debian9 + make image stor=bs tag=opencurvedocker/curvebs:v1.2 os=debian11 ## package @@ -59,6 +71,12 @@ Usage: Examples: make deb make tar release=1 dep=1 os=debian11 + + +## playground +## create/run a container, changes outside will be mapped into the container +Usage/Example: + make playground endef export help_msg @@ -74,6 +92,9 @@ build: dep: @bash util/build.sh --stor=$(stor) --only="" --dep=1 +ci-list: + @bash util/build_in_image.sh --stor=$(stor) --list + ci-build: @bash util/build_in_image.sh --stor=$(stor) --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os) --sanitizer=$(sanitizer) @@ -90,7 +111,7 @@ tar deb: @RELEASE=$(release) DEP=$(dep) OS=$(os) bash util/package.sh $@ playground: - @bash util/playground.sh + @bash util/playground.sh --version=$(version) check: @bash util/check.sh $(stor) @@ -100,5 +121,6 @@ test: docker: @bash util/docker.sh --os=$(os) --ci=$(ci) + format: @bash util/format.sh $(commit_id) diff --git a/thirdparties/etcdclient/Makefile b/thirdparties/etcdclient/Makefile index 18f694b1e3..6590918ee3 100644 --- a/thirdparties/etcdclient/Makefile +++ b/thirdparties/etcdclient/Makefile @@ -18,6 +18,7 @@ SHELL=/bin/bash pwd := $(shell pwd) + # Download and build libetcdclient only when necessary, i.e., # when dependent code is changed. # Typically, this target should always be used to avoid unnecessary @@ -29,9 +30,10 @@ libetcdclient.so libetcdclient.h: # do not use `clean' and `all' targets here as # Download and build libetcdclient, regardless of whether the code # is changed or not. -all: intall-go install-etcdclient libetcdclient +all: install-go install-etcdclient libetcdclient + -intall-go: +install-go: mkdir -p $(pwd)/tmp cd $(pwd)/tmp && wget -N https://golang.google.cn/dl/go1.12.8.linux-amd64.tar.gz cd $(pwd)/tmp && tar zxvf go1.12.8.linux-amd64.tar.gz diff --git a/util/basic.sh b/util/basic.sh index 272b25f43d..5435d586c8 100755 --- a/util/basic.sh +++ b/util/basic.sh @@ -13,3 +13,9 @@ die() { msg "\33[31m[✘]\33[0m ${1}${2}" exit 1 } + +print_title() { + local delimiter=$(printf '=%.0s' {1..20}) + msg "$delimiter [$1] $delimiter\n" +} + diff --git a/util/build.sh b/util/build.sh old mode 100755 new mode 100644 index a62eaac6ce..6a66d0f5f3 --- a/util/build.sh +++ b/util/build.sh @@ -4,86 +4,32 @@ set -x # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. ############################ GLOBAL VARIABLES +g_stor="" +g_list=0 +g_depend=0 +g_target="" +g_release=0 +g_ci=0 +g_build_rocksdb=0 +g_build_opts=( + "--define=with_glog=true" + "--define=libunwind=true" + "--copt -DHAVE_ZLIB=1" + "--copt -DGFLAGS_NS=google" + "--copt -DUSE_BTHREAD_MUTEX" +) + g_os="debian11" source "$(dirname "${BASH_SOURCE}")/docker_opts.sh" -############################ BASIC FUNCTIONS -msg() { - printf '%b' "$1" >&2 -} - -success() { - msg "\33[32m[✔]\33[0m ${1}${2}" -} - -die() { - msg "\33[31m[✘]\33[0m ${1}${2}" - exit 1 -} - -print_title() { - local delimiter=$(printf '=%.0s' {1..20}) - msg "$delimiter [$1] $delimiter\n" -} - -############################ FUNCTIONS -get_options() { - local args=`getopt -o ldorh --long stor:,list,dep:,only:,os:,release:,ci:,build_rocksdb: -n "$0" -- "$@"` - eval set -- "${args}" - while true - do - case "$1" in - -s|--stor) - g_stor=$2 - shift 2 - ;; - -l|--list) - g_list=1 - shift 1 - ;; - -d|--dep) - g_depend=$2 - shift 2 - ;; - -o|--only) - g_target=$2 - shift 2 - ;; - -r|--release) - g_release=$2 - shift 2 - ;; - -c|--ci) - g_ci=$2 - shift 2 - ;; - --os) - g_os=$2 - shift 2 - ;; - --build_rocksdb) - g_build_rocksdb=$2 - shift 2 - ;; - -h) - usage - exit 1 - ;; - --) - shift - break - ;; - *) - exit 1 - ;; - esac - done -} main() { - get_options "$@" + source "util/basic.sh" + source "util/build_functions.sh" + get_options "$@" + get_version sudo docker run \ --rm \ -w /curve \ diff --git a/util/build_functions.sh b/util/build_functions.sh new file mode 100644 index 0000000000..cbdb48719b --- /dev/null +++ b/util/build_functions.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash + +############################ BUILD BASIC FUNCTIONS +get_version() { + #get tag version + tag_version=`git status | grep -w "HEAD detached at" | awk '{print $NF}' | awk -F"v" '{print $2}'` + + # get git commit version + commit_id=`git show --abbrev-commit HEAD|head -n 1|awk '{print $2}'` + if [ $g_release -eq 1 ] + then + debug="+release" + else + debug="+debug" + fi + if [ -z ${tag_version} ] + then + echo "not found version info" + curve_version=${commit_id}${debug} + else + curve_version=${tag_version}+${commit_id}${debug} + fi + echo "version: ${curve_version}" +} + +usage () { + cat << _EOC_ +Usage: + build.sh --stor=bs/fs --list + build.sh --stor=bs/fs --only=target +Examples: + build.sh --stor=bs --only=//src/chunkserver:chunkserver + build.sh --stor=bs --only=//src/chunkserver/datastore:chunkserver_datastore + build.sh --stor=fs --only=src/* + build.sh --stor=fs --only=//curvefs/src/metaserver:curvefs_metaserver + build.sh --stor=fs --only=test/* + build.sh --stor=bs --only=test/chunkserver +_EOC_ +} + +############################ BUILD FUNCTIONS + +list_target() { + if [ "$g_stor" == "bs" ]; then + git submodule update --init -- nbd + if [ $? -ne 0 ] + then + echo "submodule init failed" + exit + fi + print_title " SOURCE TARGETS " + bazel query 'kind("cc_binary", //src/...)' + bazel query 'kind("cc_binary", //tools/...)' + bazel query 'kind("cc_binary", //nebd/src/...)' + bazel query 'kind("cc_binary", //nbd/src/...)' + + bazel query 'kind("cc_library", //include/...)' + bazel query 'kind("cc_library", //src/...)' + bazel query 'kind("cc_library", //nebd/src/...)' + bazel query 'kind("cc_library", //nbd/src/...)' + + print_title " TEST TARGETS " + bazel query 'kind("cc_(test|binary)", //test/...)' + bazel query 'kind("cc_(test|binary)", //nebd/test/...)' + bazel query 'kind("cc_(test|binary)", //nbd/test/...)' + + bazel query 'kind("cc_library", //test/...)' + bazel query 'kind("cc_library", //nebd/test/...)' + bazel query 'kind("cc_library", //nbd/test/...)' + elif [ "$g_stor" == "fs" ]; then + print_title "SOURCE TARGETS" + bazel query 'kind("cc_binary", //curvefs/src/...)' + bazel query 'kind("cc_library", //curvefs/src/...)' + + print_title " TEST TARGETS " + bazel query 'kind("cc_(test|binary)", //curvefs/test/...)' + bazel query 'kind("cc_library", //curvefs/test/...)' + fi +} + + +get_options() { + local args=`getopt -o ldorhS --long sanitizer:,stor:,list,dep:,only:,os:,release:,ci:,build_rocksdb: -n "$0" -- "$@"` + eval set -- "${args}" + while true + do + case "$1" in + -S|--sanitizer) + g_san=$2 + shift 2 + ;; + -s|--stor) + g_stor=$2 + shift 2 + ;; + -l|--list) + g_list=1 + shift 1 + ;; + -d|--dep) + g_depend=$2 + shift 2 + ;; + -o|--only) + g_target=$2 + shift 2 + ;; + -r|--release) + g_release=$2 + shift 2 + ;; + -c|--ci) + g_ci=$2 + shift 2 + ;; + --os) + g_os=$2 + shift 2 + ;; + --build_rocksdb) + g_build_rocksdb=$2 + shift 2 + ;; + -h) + usage + exit 1 + ;; + --) + shift + break + ;; + *) + exit 1 + ;; + esac + done +} + +build_target() { + if [ "$g_stor" == "bs" ]; then + git submodule update --init -- nbd + if [ $? -ne 0 ] + then + echo "submodule init failed" + exit + fi + fi + local targets + declare -A result + if [ $g_release -eq 1 ]; then + g_build_opts+=("--compilation_mode=opt --copt -g") + echo "release" > .BUILD_MODE + else + g_build_opts+=("--compilation_mode=dbg") + echo "debug" > .BUILD_MODE + fi + g_build_opts+=("--copt -DCURVEVERSION=${curve_version}") + + if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -gt 6 ]; then + g_build_opts+=("--config=gcc7-later") + fi + + local target_array + + # Allows specifying multiple targets in `g_target`, like "src/*,tools/*" + IFS=',' read -ra target_splitted <<<"$g_target" + target_splitted=("${target_splitted[@]/#/-e}") + + if [ "$g_stor" == "bs" ]; then + if [[ "$g_target" = "*" ]]; then + target_array=("-- //... -//curvefs/...") + else + target_array=($(bazel query 'kind("cc_(test|binary|library)", //... -//curvefs/...)' | grep -E ${target_splitted[@]})) + fi + elif [ "$g_stor" == "fs" ]; then + if [[ "$g_target" = "*" ]]; then + target_array=("-- //curvefs/...") + else + target_array=($(bazel query 'kind("cc_(test|binary|library)", //curvefs/...)' | grep -E ${target_splitted[@]})) + fi + fi + + if [ $g_ci -eq 1 ]; then + g_build_opts+=("--collect_code_coverage") + target_array=("...") + fi + + if [ $g_san == 1 ]; then + g_build_opts+=("--config=asan") + elif [ $g_san == 2 ]; then + g_build_opts+=("--config=msan") + fi + + for target in "${target_array[@]}" + do + bazel build ${g_build_opts[@]} $target + local ret="$?" + targets+=("$target") + result["$target"]=$ret + if [ "$ret" -ne 0 ]; then + break + fi + done + + echo "" + print_title " BUILD SUMMARY " + for target in "${targets[@]}" + do + if [ "${result[$target]}" -eq 0 ]; then + success "$target ${version}\n" + else + die "$target ${version}\n" + fi + done + + # build tools-v2 + g_toolsv2_root="tools-v2" + if [ $g_release -eq 1 ] + then + (cd ${g_toolsv2_root} && make build version=${curve_version}) + else + (cd ${g_toolsv2_root} && make debug version=${curve_version}) + fi +} + + +build_requirements() { + if [[ "$g_stor" == "fs" || $g_ci -eq 1 ]]; then + kernel_version=`uname -r | awk -F . '{print $1 * 1000 + $2}'` + if [ $kernel_version -gt 5001 ]; then + g_build_opts+=("--define IO_URING_SUPPORT=1") + fi + g_rocksdb_root="${PWD}/thirdparties/rocksdb" + (cd ${g_rocksdb_root} && make build from_source=${g_build_rocksdb} && make install prefix=${g_rocksdb_root}) + g_memcache_root="${PWD}/thirdparties/memcache" + (cd ${g_memcache_root} && make build) + fi + g_etcdclient_root="thirdparties/etcdclient" + (cd ${g_etcdclient_root} && make) # Use cache when necessary. No need to do full-rebuild every time. +} + diff --git a/util/build_in_image.sh b/util/build_in_image.sh old mode 100755 new mode 100644 index acf47f041a..7cd39811c4 --- a/util/build_in_image.sh +++ b/util/build_in_image.sh @@ -27,249 +27,12 @@ if [ -n "$BUILD_OPTS" ]; then g_build_opts+=("$BUILD_OPTS") fi -g_os="debian9" +g_os="debian11" -############################ BASIC FUNCTIONS -get_version() { - #get tag version - tag_version=`git status | grep -w "HEAD detached at" | awk '{print $NF}' | awk -F"v" '{print $2}'` - - # get git commit version - commit_id=`git show --abbrev-commit HEAD|head -n 1|awk '{print $2}'` - if [ $g_release -eq 1 ] - then - debug="+release" - else - debug="+debug" - fi - if [ -z ${tag_version} ] - then - echo "not found version info" - curve_version=${commit_id}${debug} - else - curve_version=${tag_version}+${commit_id}${debug} - fi - echo "version: ${curve_version}" -} - -msg() { - printf '%b' "$1" >&2 -} - -success() { - msg "\33[32m[✔]\33[0m ${1}${2}" -} - -die() { - msg "\33[31m[✘]\33[0m ${1}${2}" - exit 1 -} - -print_title() { - local delimiter=$(printf '=%.0s' {1..20}) - msg "$delimiter [$1] $delimiter\n" -} - -############################ FUNCTIONS -usage () { - cat << _EOC_ -Usage: - build.sh --stor=bs/fs --list - build.sh --stor=bs/fs --only=target -Examples: - build.sh --stor=bs --only=//src/chunkserver:chunkserver - build.sh --stor=fs --only=src/* - build.sh --stor=fs --only=test/* - build.sh --stor=bs --only=test/chunkserver -_EOC_ -} - -get_options() { - local args=`getopt -o ldorhS --long sanitizer:,stor:,list,dep:,only:,os:,release:,ci:,build_rocksdb: -n "$0" -- "$@"` - eval set -- "${args}" - while true - do - case "$1" in - -S|--sanitizer) - g_san=$2 - shift 2 - ;; - -s|--stor) - g_stor=$2 - shift 2 - ;; - -l|--list) - g_list=1 - shift 1 - ;; - -d|--dep) - g_depend=$2 - shift 2 - ;; - -o|--only) - g_target=$2 - shift 2 - ;; - -r|--release) - g_release=$2 - shift 2 - ;; - -c|--ci) - g_ci=$2 - shift 2 - ;; - --os) - g_os=$2 - shift 2 - ;; - --build_rocksdb) - g_build_rocksdb=$2 - shift 2 - ;; - -h) - usage - exit 1 - ;; - --) - shift - break - ;; - *) - exit 1 - ;; - esac - done -} - -list_target() { - if [ "$g_stor" == "bs" ]; then - git submodule update --init -- nbd - if [ $? -ne 0 ] - then - echo "submodule init failed" - exit - fi - print_title " SOURCE TARGETS " - bazel query 'kind("cc_binary", //src/...)' - bazel query 'kind("cc_binary", //tools/...)' - bazel query 'kind("cc_binary", //nebd/src/...)' - bazel query 'kind("cc_binary", //nbd/src/...)' - print_title " TEST TARGETS " - bazel query 'kind("cc_(test|binary)", //test/...)' - bazel query 'kind("cc_(test|binary)", //nebd/test/...)' - bazel query 'kind("cc_(test|binary)", //nbd/test/...)' - elif [ "$g_stor" == "fs" ]; then - print_title "SOURCE TARGETS" - bazel query 'kind("cc_binary", //curvefs/src/...)' - print_title " TEST TARGETS " - bazel query 'kind("cc_(test|binary)", //curvefs/test/...)' - fi -} - -build_target() { - if [ "$g_stor" == "bs" ]; then - git submodule update --init -- nbd - if [ $? -ne 0 ] - then - echo "submodule init failed" - exit - fi - fi - local targets - declare -A result - if [ $g_release -eq 1 ]; then - g_build_opts+=("--compilation_mode=opt --copt -g") - echo "release" > .BUILD_MODE - else - g_build_opts+=("--compilation_mode=dbg") - echo "debug" > .BUILD_MODE - fi - g_build_opts+=("--copt -DCURVEVERSION=${curve_version}") - - if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -gt 6 ]; then - g_build_opts+=("--config=gcc7-later") - fi - - local target_array - - # Allows specifying multiple targets in `g_target`, like "src/*,tools/*" - IFS=',' read -ra target_splitted <<<"$g_target" - target_splitted=("${target_splitted[@]/#/-e}") - - if [ "$g_stor" == "bs" ]; then - if [[ "$g_target" = "*" ]]; then - target_array=("-- //... -//curvefs/...") - else - target_array=($(bazel query 'kind("cc_(test|binary)", //... -//curvefs/...)' | grep -E ${target_splitted[@]})) - fi - elif [ "$g_stor" == "fs" ]; then - if [[ "$g_target" = "*" ]]; then - target_array=("-- //curvefs/...") - else - target_array=($(bazel query 'kind("cc_(test|binary)", //curvefs/...)' | grep -E ${target_splitted[@]})) - fi - fi - - if [ $g_ci -eq 1 ]; then - g_build_opts+=("--collect_code_coverage") - target_array=("...") - fi - - if [ $g_san == 1 ]; then - g_build_opts+=("--config=asan") - elif [ $g_san == 2 ]; then - g_build_opts+=("--config=msan") - fi - - for target in "${target_array[@]}" - do - bazel build ${g_build_opts[@]} $target - local ret="$?" - targets+=("$target") - result["$target"]=$ret - if [ "$ret" -ne 0 ]; then - break - fi - done - - echo "" - print_title " BUILD SUMMARY " - for target in "${targets[@]}" - do - if [ "${result[$target]}" -eq 0 ]; then - success "$target ${version}\n" - else - die "$target ${version}\n" - fi - done - - # build tools-v2 - g_toolsv2_root="tools-v2" - if [ $g_release -eq 1 ] - then - (cd ${g_toolsv2_root} && make build version=${curve_version}) - else - (cd ${g_toolsv2_root} && make debug version=${curve_version}) - fi -} - - -build_requirements() { - if [[ "$g_stor" == "fs" || $g_ci -eq 1 ]]; then - kernel_version=`uname -r | awk -F . '{print $1 * 1000 + $2}'` - if [ $kernel_version -gt 5001 ]; then - g_build_opts+=("--define IO_URING_SUPPORT=1") - fi - g_rocksdb_root="${PWD}/thirdparties/rocksdb" - (cd ${g_rocksdb_root} && make build from_source=${g_build_rocksdb} && make install prefix=${g_rocksdb_root}) - g_memcache_root="${PWD}/thirdparties/memcache" - (cd ${g_memcache_root} && make build) - fi - g_etcdclient_root="thirdparties/etcdclient" - (cd ${g_etcdclient_root} && make) # Use cache when necessary. No need to do full-rebuild every time. -} main() { + source "util/basic.sh" + source "util/build_functions.sh" get_options "$@" get_version diff --git a/util/playground.sh b/util/playground.sh index e964bb0be2..e762d31ebf 100755 --- a/util/playground.sh +++ b/util/playground.sh @@ -8,7 +8,7 @@ g_obm_cfg=".obm.cfg" g_worker_dir="/curve" g_container_name="curve-build-playground.master" -g_container_image="opencurvedocker/curve-base:build-debian9" +g_container_image="opencurvedocker/curve-base:build-debian11" g_init_script=$(cat << EOF useradd -m -s /bin/bash -N -u $UID $USER echo "${USER} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers @@ -26,11 +26,31 @@ EOF ############################ BASIC FUNCTIONS parse_cfg() { + local args=`getopt -o v: --long version: -n "playground.sh" -- "$@"` + eval set -- "${args}" if [ ! -f "${g_obm_cfg}" ]; then die "${g_obm_cfg} not found\n" fi g_container_name=$(cat < "${g_obm_cfg}" | grep -oP '(?<=container_name: ).*') g_container_image=$(cat < "${g_obm_cfg}" | grep -oP '(?<=container_image: ).*') + + while true + do + case "$1" in + -v|--version) + g_container_image="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + exit 1 + ;; + esac + done + } create_container() { @@ -61,12 +81,13 @@ enter_container() { "${g_container_name}" /bin/bash } -############################ MAIN() + main() { source "util/basic.sh" - parse_cfg + parse_cfg "$@" create_container enter_container } +############################ MAIN() main "$@" From 86677ef954a0a55788086e5390622ffe41ae3b6c Mon Sep 17 00:00:00 2001 From: wanghai01 Date: Tue, 24 Oct 2023 11:16:11 +0800 Subject: [PATCH 10/27] fix some unnecessary error logs Signed-off-by: wanghai01 --- curvefs/src/metaserver/s3/metaserver_s3.cpp | 3 ++- src/common/s3_adapter.cpp | 20 ++++++-------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/curvefs/src/metaserver/s3/metaserver_s3.cpp b/curvefs/src/metaserver/s3/metaserver_s3.cpp index 5a4d487fde..ee22366255 100644 --- a/curvefs/src/metaserver/s3/metaserver_s3.cpp +++ b/curvefs/src/metaserver/s3/metaserver_s3.cpp @@ -50,11 +50,12 @@ int S3ClientImpl::Delete(const std::string& name) { ret = s3Adapter_->DeleteObject(aws_key); if (ret < 0) { // -1 - LOG(ERROR) << "delete object: " << aws_key << " get error:" << ret; if (!s3Adapter_->ObjectExist(aws_key)) { // the aws_key is not exist // may delete by others ret = 1; + } else { + LOG(ERROR) << "delete object: " << aws_key << " get error:" << ret; } } else { // 0 diff --git a/src/common/s3_adapter.cpp b/src/common/s3_adapter.cpp index 96c7c87e6b..e3b3d917a0 100644 --- a/src/common/s3_adapter.cpp +++ b/src/common/s3_adapter.cpp @@ -489,13 +489,9 @@ bool S3Adapter::ObjectExist(const Aws::String &key) { if (response.IsSuccess()) { return true; } else { - LOG(ERROR) << "HeadObject error:" - << bucketName_ - << "--" - << key - << "--" - << response.GetError().GetExceptionName() - << response.GetError().GetMessage(); + LOG(WARNING) << "HeadObject error:" << bucketName_ << "--" << key + << "--" << response.GetError().GetExceptionName() + << response.GetError().GetMessage(); return false; } } @@ -508,13 +504,9 @@ int S3Adapter::DeleteObject(const Aws::String &key) { if (response.IsSuccess()) { return 0; } else { - LOG(ERROR) << "DeleteObject error:" - << bucketName_ - << "--" - << key - << "--" - << response.GetError().GetExceptionName() - << response.GetError().GetMessage(); + LOG(WARNING) << "DeleteObject error:" << bucketName_ << "--" << key + << "--" << response.GetError().GetExceptionName() + << response.GetError().GetMessage(); return -1; } } From 0e5f305da924f4a01fbecffe0b93d6354dd88343 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 22:31:57 +0000 Subject: [PATCH 11/27] build(deps): bump golang.org/x/net from 0.10.0 to 0.17.0 in /tools-v2 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.10.0 to 0.17.0. - [Commits](https://github.com/golang/net/compare/v0.10.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] --- tools-v2/go.mod | 10 +++++----- tools-v2/go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tools-v2/go.mod b/tools-v2/go.mod index c51325de20..f48f77e0dc 100644 --- a/tools-v2/go.mod +++ b/tools-v2/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.16.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/sys v0.8.0 + golang.org/x/sys v0.13.0 google.golang.org/grpc v1.55.0 google.golang.org/protobuf v1.30.0 ) @@ -71,12 +71,12 @@ require ( github.com/subosito/gotenv v1.4.2 // indirect github.com/theupdateframework/notary v0.7.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/crypto v0.9.0 // indirect + golang.org/x/crypto v0.14.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/term v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/tools-v2/go.sum b/tools-v2/go.sum index 518e3d6aa4..a87ca30cf0 100644 --- a/tools-v2/go.sum +++ b/tools-v2/go.sum @@ -431,8 +431,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -505,8 +505,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -576,13 +576,13 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -591,8 +591,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 704169d7faa16379db972b1db92ca56037cff2f8 Mon Sep 17 00:00:00 2001 From: h0hmj Date: Thu, 19 Oct 2023 15:18:09 +0800 Subject: [PATCH 12/27] curvefs/mds: add update fsinfo capacity Signed-off-by: h0hmj --- curvefs/proto/mds.proto | 8 ++-- curvefs/src/mds/fs_manager.cpp | 33 ++++++++++++++++ curvefs/src/mds/fs_manager.h | 3 ++ curvefs/src/mds/mds_service.cpp | 34 +++++++++++++++++ curvefs/src/mds/mds_service.h | 5 +++ curvefs/test/mds/mds_service_test.cpp | 54 +++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 5 deletions(-) diff --git a/curvefs/proto/mds.proto b/curvefs/proto/mds.proto index eebec8ee34..13c67d5871 100644 --- a/curvefs/proto/mds.proto +++ b/curvefs/proto/mds.proto @@ -145,16 +145,14 @@ message UmountFsResponse { required FSStatusCode statusCode = 1; } -/* UpdateFsInfoRequest comment now message UpdateFsInfoRequest { required string fsName = 1; - // todo + optional uint64 capacity = 2; } + message UpdateFsInfoResponse { required FSStatusCode statusCode = 1; - optional FsInfo fsInfo = 2; } -*/ message DeleteFsRequest { required string fsName = 1; @@ -232,7 +230,7 @@ service MdsService { rpc UmountFs(UmountFsRequest) returns (UmountFsResponse); // TODO(chengyi01): move to GetFssInfo rpc GetFsInfo(GetFsInfoRequest) returns (GetFsInfoResponse); - // rpc UpdateFsInfo(UpdateFsInfoRequest) returns (UpdateFsInfoResponse); + rpc UpdateFsInfo(UpdateFsInfoRequest) returns (UpdateFsInfoResponse); rpc DeleteFs(DeleteFsRequest) returns (DeleteFsResponse); rpc AllocateS3Chunk(AllocateS3ChunkRequest) returns (AllocateS3ChunkResponse); rpc ListClusterFsInfo (ListClusterFsInfoRequest) returns (ListClusterFsInfoResponse); diff --git a/curvefs/src/mds/fs_manager.cpp b/curvefs/src/mds/fs_manager.cpp index 3af5b118d3..e8ec55834c 100644 --- a/curvefs/src/mds/fs_manager.cpp +++ b/curvefs/src/mds/fs_manager.cpp @@ -790,6 +790,39 @@ FSStatusCode FsManager::GetFsInfo(const std::string& fsName, uint32_t fsId, return FSStatusCode::OK; } +FSStatusCode FsManager::UpdateFsInfo( + const ::curvefs::mds::UpdateFsInfoRequest* req) { + const auto& fsName = req->fsname(); + NameLockGuard lock(nameLock_, fsName); + + // 1. query fs + FsInfoWrapper wrapper; + FSStatusCode ret = fsStorage_->Get(fsName, &wrapper); + if (ret != FSStatusCode::OK) { + LOG(WARNING) << "UpdateFsInfo fail, get fs fail, fsName = " << fsName + << ", errCode = " << FSStatusCode_Name(ret); + return ret; + } + + // 2. check fs status + FsStatus status = wrapper.GetStatus(); + if (status == FsStatus::NEW) { + LOG(WARNING) << "UpdateFsInfo fs is not inited, fsName = " << fsName; + return FSStatusCode::NOT_INITED; + } else if (status == FsStatus::DELETING) { + LOG(WARNING) << "UpdateFsInfo fs is in deleting, fsName = " << fsName; + return FSStatusCode::UNDER_DELETING; + } + + // 3. update fs info + // set capacity + if (req->has_capacity()) { + wrapper.SetCapacity(req->capacity()); + } + + return fsStorage_->Update(wrapper); +} + int FsManager::IsExactlySameOrCreateUnComplete(const std::string& fsName, FSType fsType, uint64_t blocksize, diff --git a/curvefs/src/mds/fs_manager.h b/curvefs/src/mds/fs_manager.h index 2183786017..21bdcd0203 100644 --- a/curvefs/src/mds/fs_manager.h +++ b/curvefs/src/mds/fs_manager.h @@ -187,6 +187,9 @@ class FsManager { FSStatusCode GetFsInfo(const std::string& fsName, uint32_t fsId, FsInfo* fsInfo); + FSStatusCode UpdateFsInfo( + const ::curvefs::mds::UpdateFsInfoRequest* request); + void GetAllFsInfo(::google::protobuf::RepeatedPtrField< ::curvefs::mds::FsInfo>* fsInfoVec); diff --git a/curvefs/src/mds/mds_service.cpp b/curvefs/src/mds/mds_service.cpp index 76ea58fd5a..0b1c32c76a 100644 --- a/curvefs/src/mds/mds_service.cpp +++ b/curvefs/src/mds/mds_service.cpp @@ -227,6 +227,40 @@ void MdsServiceImpl::GetFsInfo(::google::protobuf::RpcController *controller, << response->ShortDebugString(); } +void MdsServiceImpl::UpdateFsInfo( + ::google::protobuf::RpcController* controller, + const ::curvefs::mds::UpdateFsInfoRequest* request, + ::curvefs::mds::UpdateFsInfoResponse* response, + ::google::protobuf::Closure* done) { + (void)controller; + brpc::ClosureGuard doneGuard(done); + + LOG(INFO) << "UpdateFsInfo request: " << request->ShortDebugString(); + + // request parameter check, we want at least one field that need to change + if (!request->has_capacity()) { + response->set_statuscode(FSStatusCode::PARAM_ERROR); + LOG(WARNING) << "UpdateFsInfo fail, request: " + << request->ShortDebugString() << ", errCode = " + << FSStatusCode_Name(FSStatusCode::PARAM_ERROR); + return; + } + + FSStatusCode status = fsManager_->UpdateFsInfo(request); + + if (status != FSStatusCode::OK) { + response->set_statuscode(status); + LOG(ERROR) << "UpdateFsInfo fail, request: " + << request->ShortDebugString() + << ", errCode = " << FSStatusCode_Name(status); + return; + } + + response->set_statuscode(FSStatusCode::OK); + LOG(INFO) << "UpdateFsInfo success, response: " + << response->ShortDebugString(); +} + void MdsServiceImpl::DeleteFs(::google::protobuf::RpcController *controller, const ::curvefs::mds::DeleteFsRequest *request, ::curvefs::mds::DeleteFsResponse *response, diff --git a/curvefs/src/mds/mds_service.h b/curvefs/src/mds/mds_service.h index 7efa417779..343b946427 100644 --- a/curvefs/src/mds/mds_service.h +++ b/curvefs/src/mds/mds_service.h @@ -65,6 +65,11 @@ class MdsServiceImpl : public MdsService { GetFsInfoResponse* response, ::google::protobuf::Closure* done); + void UpdateFsInfo(::google::protobuf::RpcController* controller, + const UpdateFsInfoRequest* request, + UpdateFsInfoResponse* response, + ::google::protobuf::Closure* done); + void DeleteFs(::google::protobuf::RpcController* controller, const DeleteFsRequest* request, DeleteFsResponse* response, diff --git a/curvefs/test/mds/mds_service_test.cpp b/curvefs/test/mds/mds_service_test.cpp index 4c14a6a161..bdae7f7691 100644 --- a/curvefs/test/mds/mds_service_test.cpp +++ b/curvefs/test/mds/mds_service_test.cpp @@ -898,5 +898,59 @@ TEST_F(MdsServiceTest, test_success_delete_fs_after_umounting_all_paths) { } } +TEST_F(MdsServiceTest, test_update_fsinfo_success) { + FsInfo fs; + + auto get = [&]() { + cntl.Reset(); + GetFsInfoRequest getRequest; + GetFsInfoResponse getResponse; + getRequest.set_fsid(fsinfo1.fsid()); + stub_->GetFsInfo(&cntl, &getRequest, &getResponse, nullptr); + if (!cntl.Failed()) { + ASSERT_EQ(getResponse.statuscode(), FSStatusCode::OK); + fs = getResponse.fsinfo(); + } else { + LOG(ERROR) << "error = " << cntl.ErrorText(); + ASSERT_TRUE(false); + } + }; + + // 1. check current fsinfo + get(); + ASSERT_EQ(fs.capacity(), fsinfo1.capacity()); + + // 2. update fsinfo + cntl.Reset(); + UpdateFsInfoRequest updateRequest; + UpdateFsInfoResponse updateResponse; + updateRequest.set_fsname(fsinfo1.fsname()); + updateRequest.set_capacity(0); + stub_->UpdateFsInfo(&cntl, &updateRequest, &updateResponse, nullptr); + if (!cntl.Failed()) { + ASSERT_EQ(updateResponse.statuscode(), FSStatusCode::OK); + } else { + LOG(ERROR) << "error = " << cntl.ErrorText(); + ASSERT_TRUE(false); + } + + // 3. check updated fsinfo + get(); + ASSERT_EQ(fs.capacity(), 0); +} + +TEST_F(MdsServiceTest, test_update_fsinfo_parameter_error) { + UpdateFsInfoRequest updateRequest; + UpdateFsInfoResponse updateResponse; + updateRequest.set_fsname(fsinfo1.fsname()); + stub_->UpdateFsInfo(&cntl, &updateRequest, &updateResponse, nullptr); + if (!cntl.Failed()) { + ASSERT_EQ(updateResponse.statuscode(), FSStatusCode::PARAM_ERROR); + } else { + LOG(ERROR) << "error = " << cntl.ErrorText(); + ASSERT_TRUE(false); + } +} + } // namespace mds } // namespace curvefs From ba010be946990b96c42719c39f2d456f9e225aa3 Mon Sep 17 00:00:00 2001 From: h0hmj Date: Thu, 19 Oct 2023 17:17:50 +0800 Subject: [PATCH 13/27] tools-v2: add update fs command to change capacity Signed-off-by: h0hmj --- tools-v2/internal/error/error.go | 3 + tools-v2/pkg/cli/command/curvefs/fs.go | 2 + .../pkg/cli/command/curvefs/update/fs/fs.go | 157 ++++++++++++++++++ .../pkg/cli/command/curvefs/update/update.go | 45 +++++ tools-v2/pkg/config/config.go | 4 + 5 files changed, 211 insertions(+) create mode 100644 tools-v2/pkg/cli/command/curvefs/update/fs/fs.go create mode 100644 tools-v2/pkg/cli/command/curvefs/update/update.go diff --git a/tools-v2/internal/error/error.go b/tools-v2/internal/error/error.go index d9676184db..72f8de384d 100644 --- a/tools-v2/internal/error/error.go +++ b/tools-v2/internal/error/error.go @@ -265,6 +265,9 @@ var ( ErrGetClusterFsInfo = func() *CmdError { return NewInternalCmdError(8, "get cluster fs info failed, the error is: \n%s") } + ErrUpdateClusterFsInfo = func() *CmdError { + return NewInternalCmdError(9, "update cluster fs info failed, the error is: \n%s") + } ErrGetAddr = func() *CmdError { return NewInternalCmdError(9, "invalid %s addr is: %s") } diff --git a/tools-v2/pkg/cli/command/curvefs/fs.go b/tools-v2/pkg/cli/command/curvefs/fs.go index 4404c2ca06..da0cdaa19f 100644 --- a/tools-v2/pkg/cli/command/curvefs/fs.go +++ b/tools-v2/pkg/cli/command/curvefs/fs.go @@ -31,6 +31,7 @@ import ( "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/query" status "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/status" umount "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/umount" + update "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/update" usage "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/usage" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/warmup" "github.com/spf13/cobra" @@ -53,6 +54,7 @@ func (fsCmd *CurveFsCommand) AddSubCommands() { create.NewCreateCommand(), check.NewCheckCommand(), warmup.NewWarmupCommand(), + update.NewUpdateCommand(), ) } diff --git a/tools-v2/pkg/cli/command/curvefs/update/fs/fs.go b/tools-v2/pkg/cli/command/curvefs/update/fs/fs.go new file mode 100644 index 0000000000..f4b5ad2c2d --- /dev/null +++ b/tools-v2/pkg/cli/command/curvefs/update/fs/fs.go @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs + +import ( + "context" + "fmt" + + "github.com/dustin/go-humanize" + cmderror "github.com/opencurve/curve/tools-v2/internal/error" + cobrautil "github.com/opencurve/curve/tools-v2/internal/utils" + basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command" + "github.com/opencurve/curve/tools-v2/pkg/config" + "github.com/opencurve/curve/tools-v2/pkg/output" + mds "github.com/opencurve/curve/tools-v2/proto/curvefs/proto/mds" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "google.golang.org/grpc" +) + +const ( + fsExample = `$ curve fs update fs --fsname test1` +) + +type UpdateFsRpc struct { + Info *basecmd.Rpc + Request *mds.UpdateFsInfoRequest + mdsClient mds.MdsServiceClient +} + +var _ basecmd.RpcFunc = (*UpdateFsRpc)(nil) // check interface + +type FsCommand struct { + basecmd.FinalCurveCmd + Rpc *UpdateFsRpc +} + +var _ basecmd.FinalCurveCmdFunc = (*FsCommand)(nil) // check interface + +func (ufRp *UpdateFsRpc) NewRpcClient(cc grpc.ClientConnInterface) { + ufRp.mdsClient = mds.NewMdsServiceClient(cc) +} + +func (ufRp *UpdateFsRpc) Stub_Func(ctx context.Context) (interface{}, error) { + return ufRp.mdsClient.UpdateFsInfo(ctx, ufRp.Request) +} + +func NewFsCommand() *cobra.Command { + fsCmd := &FsCommand{ + FinalCurveCmd: basecmd.FinalCurveCmd{ + Use: "fs", + Short: "update fsinfo", + Long: "update fsinfo", + Example: fsExample, + }, + } + basecmd.NewFinalCurveCli(&fsCmd.FinalCurveCmd, fsCmd) + return fsCmd.Cmd +} + +func (fCmd *FsCommand) AddFlags() { + config.AddRpcRetryTimesFlag(fCmd.Cmd) + config.AddRpcTimeoutFlag(fCmd.Cmd) + config.AddFsMdsAddrFlag(fCmd.Cmd) + config.AddFsNameRequiredFlag(fCmd.Cmd) + // things can be changed + config.AddCapacityOptionFlag(fCmd.Cmd) +} + +func (fCmd *FsCommand) Init(cmd *cobra.Command, args []string) error { + // args check + fsName, _ := cmd.Flags().GetString("fsName") + request := &mds.UpdateFsInfoRequest{ + FsName: &fsName, + } + + nothingToChange := true + var capacity string + if config.GetFlagChanged(cmd, config.CURVEFS_CAPACITY) { + nothingToChange = false + capacity = config.GetFlagString(cmd, config.CURVEFS_CAPACITY) + cap_bytes, _ := humanize.ParseBytes(capacity) + request.Capacity = &cap_bytes + } + + if nothingToChange { + return fmt.Errorf("please specify something to change") + } + + addrs, addrErr := config.GetFsMdsAddrSlice(fCmd.Cmd) + if addrErr.TypeCode() != cmderror.CODE_SUCCESS { + return fmt.Errorf(addrErr.Message) + } + timeout := viper.GetDuration(config.VIPER_GLOBALE_RPCTIMEOUT) + retrytimes := viper.GetInt32(config.VIPER_GLOBALE_RPCRETRYTIMES) + + // output format + header := []string{cobrautil.ROW_FS_NAME, cobrautil.ROW_RESULT} + fCmd.SetHeader(header) + + // set rpc + fCmd.Rpc = &UpdateFsRpc{ + Request: request, + } + fCmd.Rpc.Info = basecmd.NewRpc(addrs, timeout, retrytimes, "UpdateFsInfo") + return nil +} + +func (fCmd *FsCommand) Print(cmd *cobra.Command, args []string) error { + return output.FinalCmdOutput(&fCmd.FinalCurveCmd, fCmd) +} + +func (fCmd *FsCommand) RunCommand(cmd *cobra.Command, args []string) error { + result, errCmd := basecmd.GetRpcResponse(fCmd.Rpc.Info, fCmd.Rpc) + if errCmd.TypeCode() != cmderror.CODE_SUCCESS { + return fmt.Errorf(errCmd.Message) + } + + response := result.(*mds.UpdateFsInfoResponse) + errCreate := cmderror.ErrCreateFs(int(response.GetStatusCode())) + row := map[string]string{ + cobrautil.ROW_FS_NAME: fCmd.Rpc.Request.GetFsName(), + cobrautil.ROW_RESULT: errCreate.Message, + } + + fCmd.TableNew.Append(cobrautil.Map2List(row, fCmd.Header)) + + var errs []*cmderror.CmdError + res, errTranslate := output.MarshalProtoJson(response) + if errTranslate != nil { + errMar := cmderror.ErrMarShalProtoJson() + errMar.Format(errTranslate.Error()) + errs = append(errs, errMar) + } + + fCmd.Result = res + fCmd.Error = cmderror.MostImportantCmdError(errs) + return nil +} + +func (fCmd *FsCommand) ResultPlainOutput() error { + return output.FinalCmdOutputPlain(&fCmd.FinalCurveCmd) +} diff --git a/tools-v2/pkg/cli/command/curvefs/update/update.go b/tools-v2/pkg/cli/command/curvefs/update/update.go new file mode 100644 index 0000000000..2b7cf4513f --- /dev/null +++ b/tools-v2/pkg/cli/command/curvefs/update/update.go @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package update + +import ( + basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command" + "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvefs/update/fs" + "github.com/spf13/cobra" +) + +type UpdateCommand struct { + basecmd.MidCurveCmd +} + +var _ basecmd.MidCurveCmdFunc = (*UpdateCommand)(nil) // check interface + +func (updateCmd *UpdateCommand) AddSubCommands() { + updateCmd.Cmd.AddCommand( + fs.NewFsCommand(), + ) +} + +func NewUpdateCommand() *cobra.Command { + updateCmd := &UpdateCommand{ + basecmd.MidCurveCmd{ + Use: "update", + Short: "update resources in the curvefs", + }, + } + return basecmd.NewMidCurveCli(&updateCmd.MidCurveCmd, updateCmd) +} diff --git a/tools-v2/pkg/config/config.go b/tools-v2/pkg/config/config.go index 6a30e6d364..dd37904218 100644 --- a/tools-v2/pkg/config/config.go +++ b/tools-v2/pkg/config/config.go @@ -134,6 +134,10 @@ func AddShowErrorPFlag(cmd *cobra.Command) { } } +func AddFsCapacityFlag(cmd *cobra.Command) { + AddStringOptionFlag(cmd, "capacity", "capacity of the filesystem") +} + // Align the flag (changed) in the caller with the callee func AlignFlagsValue(caller *cobra.Command, callee *cobra.Command, flagNames []string) { callee.Flags().VisitAll(func(flag *pflag.Flag) { From 2009d2e49051e19876a608e5d7bc53190b3b1736 Mon Sep 17 00:00:00 2001 From: yfeng Date: Sun, 16 Jul 2023 18:06:58 +0800 Subject: [PATCH 14/27] Chunk pool format asyn Signed-off-by: yyyyufeng --- conf/chunkserver.conf | 18 +- conf/chunkserver.conf.example | 18 +- proto/heartbeat.proto | 2 + proto/topology.proto | 14 + src/chunkserver/chunkserver.cpp | 116 +++++++-- src/chunkserver/datastore/file_pool.cpp | 245 ++++++++++++++++-- src/chunkserver/datastore/file_pool.h | 88 ++++++- src/chunkserver/heartbeat.cpp | 14 +- src/chunkserver/trash.h | 42 ++- src/common/bytes_convert.h | 86 ++++++ src/mds/heartbeat/heartbeat_manager.cpp | 5 + src/mds/topology/topology_service.cpp | 13 + src/mds/topology/topology_service.h | 5 + src/mds/topology/topology_service_manager.cpp | 19 ++ src/mds/topology/topology_service_manager.h | 4 + src/mds/topology/topology_stat.h | 17 +- src/tools/curve_tool_define.h | 1 + src/tools/mds_client.cpp | 19 ++ src/tools/mds_client.h | 33 ++- src/tools/status_tool.cpp | 37 ++- src/tools/status_tool.h | 1 + .../datastore/filepool_mock_unittest.cpp | 5 +- .../datastore/filepool_unittest.cpp | 60 ++++- .../chunkserver/chunkserver_basic_test.cpp | 1 + .../chunkserver_concurrent_test.cpp | 1 + .../client/chunkserver_exception_test.cpp | 114 ++++---- .../integration/client/mds_exception_test.cpp | 120 ++++----- .../unstable_chunkserver_exception_test.cpp | 32 +-- .../snapshotcloneserver_common_test.cpp | 1 + .../snapshotcloneserver_concurrent_test.cpp | 1 + .../snapshotcloneserver_recover_test.cpp | 1 + tools-v2/internal/error/error.go | 3 + tools-v2/internal/utils/row.go | 3 +- .../curvebs/list/formatstatus/formatstatus.go | 137 ++++++++++ tools-v2/pkg/cli/command/curvebs/list/list.go | 2 + 35 files changed, 1036 insertions(+), 242 deletions(-) create mode 100644 src/common/bytes_convert.h create mode 100644 tools-v2/pkg/cli/command/curvebs/list/formatstatus/formatstatus.go diff --git a/conf/chunkserver.conf b/conf/chunkserver.conf index 0cfc27b544..19457b3c18 100644 --- a/conf/chunkserver.conf +++ b/conf/chunkserver.conf @@ -194,7 +194,7 @@ rconcurrentapply.queuedepth=1 # 是否开启从chunkfilepool获取chunk，一般是true chunkfilepool.enable_get_chunk_from_pool=true # chunkfilepool目录 -chunkfilepool.chunk_file_pool_dir=./0/ # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__ +chunkfilepool.chunk_file_pool_dir=./0/chunks # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__ # chunkfilepool meta文件路径 chunkfilepool.meta_path=./chunkfilepool.meta # __CURVEADM_TEMPLATE__ ${prefix}/data/chunkfilepool.meta __CURVEADM_TEMPLATE__ # chunkfilepool meta文件大小 @@ -207,6 +207,14 @@ chunkfilepool.clean.enable=true chunkfilepool.clean.bytes_per_write=4096 # The throttle iops for cleaning chunk (4KB/IO) chunkfilepool.clean.throttle_iops=500 +# Whether allocate filePool by percent of disk size. +chunkfilepool.allocated_by_percent=true +# Preallocate storage percent of total disk +chunkfilepool.allocate_percent=80 +# Preallocate storage size of chunkfilepool (None/KB/MB/GB/TB) +chunkfilepool.chunk_file_pool_size=1GB +# The thread num for format chunks +chunkfilepool.thread_num=1 # # WAL file pool @@ -229,6 +237,14 @@ walfilepool.metapage_size=4096 walfilepool.meta_file_size=4096 # WAL filepool get chunk最大重试次数 walfilepool.retry_times=5 +# Whether allocate filePool by percent of disk size. +walfilepool.allocated_by_percent=true +# Preallocate storage percent of total disk +walfilepool.allocate_percent=90 +# Preallocate storage size size of walfilepool (None/KB/MB/GB/TB) +walfilepool.wal_file_pool_size=0 +# The thread num for format chunks +walfilepool.thread_num=1 # # trash settings diff --git a/conf/chunkserver.conf.example b/conf/chunkserver.conf.example index eb664c2fd6..443412215b 100644 --- a/conf/chunkserver.conf.example +++ b/conf/chunkserver.conf.example @@ -186,7 +186,7 @@ rconcurrentapply.queuedepth=1 # 是否开启从chunkfilepool获取chunk，一般是true chunkfilepool.enable_get_chunk_from_pool=true # chunkfilepool目录 -chunkfilepool.chunk_file_pool_dir=./0/ +chunkfilepool.chunk_file_pool_dir=./0/chunks # chunkfilepool meta文件路径 #chunkfilepool.meta_path=./chunkfilepool.meta # chunkfilepool meta文件大小 @@ -199,6 +199,14 @@ chunkfilepool.clean.enable=true chunkfilepool.clean.bytes_per_write=4096 # The throttle iops for cleaning chunk (4KB/IO) chunkfilepool.clean.throttle_iops=500 +# Whether allocate filePool by percent of disk size. +chunkfilepool.allocated_by_percent=true +# Preallocate storage percent of total disk +chunkfilepool.allocate_percent=80 +# Preallocate storage size of chunkfilepool (None/KB/MB/GB/TB) +chunkfilepool.chunk_file_pool_size=1GB +# The thread num for format chunks +chunkfilepool.thread_num=1 # # WAL file pool @@ -221,6 +229,14 @@ walfilepool.metapage_size=4096 walfilepool.meta_file_size=4096 # WAL filepool get chunk最大重试次数 walfilepool.retry_times=5 +# Whether allocate filePool by percent of disk size. +walfilepool.allocated_by_percent=true +# Preallocate storage percent of total disk +walfilepool.allocate_percent=10 +# Preallocate storage size size of walfilepool (None/KB/MB/GB/TB) +walfilepool.wal_file_pool_size=0 +# The thread num for format chunks +walfilepool.thread_num=1 # # trash settings diff --git a/proto/heartbeat.proto b/proto/heartbeat.proto index d54723dfb8..6b51d40277 100644 --- a/proto/heartbeat.proto +++ b/proto/heartbeat.proto @@ -89,6 +89,8 @@ message ChunkServerStatisticInfo { required uint64 chunkSizeTrashedBytes = 7; // chunkfilepool的大小 optional uint64 chunkFilepoolSize = 8; + // percentage of chunkfilepool formatting + optional uint32 chunkFilepoolFormatPercent = 9; }; message ChunkServerHeartbeatRequest { diff --git a/proto/topology.proto b/proto/topology.proto index 2057cafe2a..6e88d4e102 100644 --- a/proto/topology.proto +++ b/proto/topology.proto @@ -501,6 +501,19 @@ message GetCopySetsInChunkServerRequest { optional uint32 port = 3; } +message ChunkFormatStatus { + required string ip = 1; + required uint32 port = 2; + required uint32 chunkServerID = 3; + required uint32 formatPercent = 4; +} + +message ListChunkFormatStatusRequest {} + +message ListChunkFormatStatusResponse { + repeated ChunkFormatStatus chunkFormatStatus = 1; +} + message GetCopySetsInChunkServerResponse { required sint32 statusCode = 1; repeated common.CopysetInfo copysetInfos = 2; @@ -595,4 +608,5 @@ service TopologyService { rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse); rpc SetCopysetsAvailFlag(SetCopysetsAvailFlagRequest) returns (SetCopysetsAvailFlagResponse); rpc ListUnAvailCopySets(ListUnAvailCopySetsRequest) returns (ListUnAvailCopySetsResponse); + rpc ListChunkFormatStatus(ListChunkFormatStatusRequest) returns (ListChunkFormatStatusResponse); } diff --git a/src/chunkserver/chunkserver.cpp b/src/chunkserver/chunkserver.cpp index 07f1f48d5f..784628447f 100644 --- a/src/chunkserver/chunkserver.cpp +++ b/src/chunkserver/chunkserver.cpp @@ -20,30 +20,31 @@ * Author: lixiaocui */ -#include +#include "src/chunkserver/chunkserver.h" -#include #include #include #include +#include +#include #include -#include "src/chunkserver/chunkserver.h" -#include "src/chunkserver/chunkserver_metrics.h" -#include "src/chunkserver/chunkserver_service.h" -#include "src/chunkserver/copyset_service.h" -#include "src/chunkserver/chunk_service.h" #include "src/chunkserver/braft_cli_service.h" #include "src/chunkserver/braft_cli_service2.h" +#include "src/chunkserver/chunk_service.h" #include "src/chunkserver/chunkserver_helper.h" -#include "src/common/concurrent/task_thread_pool.h" -#include "src/common/uri_parser.h" -#include "src/chunkserver/raftsnapshot/curve_snapshot_attachment.h" +#include "src/chunkserver/chunkserver_metrics.h" +#include "src/chunkserver/chunkserver_service.h" +#include "src/chunkserver/copyset_service.h" +#include "src/chunkserver/raftlog/curve_segment_log_storage.h" #include "src/chunkserver/raftsnapshot/curve_file_service.h" +#include "src/chunkserver/raftsnapshot/curve_snapshot_attachment.h" #include "src/chunkserver/raftsnapshot/curve_snapshot_storage.h" -#include "src/chunkserver/raftlog/curve_segment_log_storage.h" +#include "src/common/bytes_convert.h" +#include "src/common/concurrent/task_thread_pool.h" #include "src/common/curve_version.h" +#include "src/common/uri_parser.h" using ::curve::fs::LocalFileSystem; using ::curve::fs::LocalFileSystemOption; @@ -65,6 +66,10 @@ DEFINE_string(raftSnapshotUri, "curve://./0/copysets", "raft snapshot uri"); DEFINE_string(raftLogUri, "curve://./0/copysets", "raft log uri"); DEFINE_string(recycleUri, "local://./0/recycler" , "recycle uri"); DEFINE_string(chunkFilePoolDir, "./0/", "chunk file pool location"); +DEFINE_int32(chunkFilePoolAllocatedPercent, 80, + "format percent for chunkfillpool."); +DEFINE_uint32(chunkFormatThreadNum, 1, + "number of threads while file pool formatting"); DEFINE_string(chunkFilePoolMetaPath, "./chunkfilepool.meta", "chunk file pool meta path"); DEFINE_string(logPath, "./0/chunkserver.log-", "log file path"); @@ -479,8 +484,6 @@ void ChunkServer::Stop() { brpc::AskToQuit(); } - - void ChunkServer::InitChunkFilePoolOptions( common::Configuration *conf, FilePoolOptions *chunkFilePoolOptions) { LOG_IF(FATAL, !conf->GetUInt32Value("global.chunk_size", @@ -513,13 +516,56 @@ void ChunkServer::InitChunkFilePoolOptions( "chunkfilepool.meta_path", &metaUri)); ::memcpy( chunkFilePoolOptions->metaPath, metaUri.c_str(), metaUri.size()); + + std::string chunkFilePoolUri; + LOG_IF(FATAL, !conf->GetStringValue("chunkfilepool.chunk_file_pool_dir", + &chunkFilePoolUri)); + + ::memcpy(chunkFilePoolOptions->filePoolDir, chunkFilePoolUri.c_str(), + chunkFilePoolUri.size()); + std::string pool_size; + LOG_IF(FATAL, !conf->GetStringValue( + "chunkfilepool.chunk_file_pool_size", &pool_size)); + LOG_IF(FATAL, !curve::common::ToNumbericByte( + pool_size, &chunkFilePoolOptions->filePoolSize)); + LOG_IF(FATAL, + !conf->GetBoolValue("chunkfilepool.allocated_by_percent", + &chunkFilePoolOptions->allocatedByPercent)); + LOG_IF(FATAL, + !conf->GetUInt32Value("chunkfilepool.allocate_percent", + &chunkFilePoolOptions->allocatedPercent)); + LOG_IF(FATAL, !conf->GetUInt32Value( + "chunkfilepool.chunk_file_pool_format_thread_num", + &chunkFilePoolOptions->formatThreadNum)); LOG_IF(FATAL, !conf->GetBoolValue("chunkfilepool.clean.enable", &chunkFilePoolOptions->needClean)); - LOG_IF(FATAL, !conf->GetUInt32Value("chunkfilepool.clean.bytes_per_write", // NOLINT - &chunkFilePoolOptions->bytesPerWrite)); + LOG_IF(FATAL, + !conf->GetUInt32Value("chunkfilepool.clean.bytes_per_write", + &chunkFilePoolOptions->bytesPerWrite)); LOG_IF(FATAL, !conf->GetUInt32Value("chunkfilepool.clean.throttle_iops", &chunkFilePoolOptions->iops4clean)); + std::string copysetUri; + LOG_IF(FATAL, + !conf->GetStringValue("copyset.raft_snapshot_uri", ©setUri)); + curve::common::UriParser::ParseUri(copysetUri, + &chunkFilePoolOptions->copysetDir); + + std::string recycleUri; + LOG_IF(FATAL, + !conf->GetStringValue("copyset.recycler_uri", &recycleUri)); + curve::common::UriParser::ParseUri(recycleUri, + &chunkFilePoolOptions->recycleDir); + + bool useChunkFilePoolAsWalPool; + LOG_IF(FATAL, !conf->GetBoolValue("walfilepool.use_chunk_file_pool", + &useChunkFilePoolAsWalPool)); + + chunkFilePoolOptions->isAllocated = [=](const std::string& filename) { + return Trash::IsChunkOrSnapShotFile(filename) || + (useChunkFilePoolAsWalPool && Trash::IsWALFile(filename)); + }; + if (0 == chunkFilePoolOptions->bytesPerWrite || chunkFilePoolOptions->bytesPerWrite > 1 * 1024 * 1024 || 0 != chunkFilePoolOptions->bytesPerWrite % 4096) { @@ -565,6 +611,36 @@ void ChunkServer::InitWalFilePoolOptions( std::string metaUri; LOG_IF(FATAL, !conf->GetStringValue( "walfilepool.meta_path", &metaUri)); + + std::string pool_size; + LOG_IF(FATAL, !conf->GetStringValue("walfilepool.chunk_file_pool_size", + &pool_size)); + LOG_IF(FATAL, !curve::common::ToNumbericByte( + pool_size, &walPoolOptions->filePoolSize)); + LOG_IF(FATAL, !conf->GetUInt64Value("walfilepool.wal_file_pool_size", + &walPoolOptions->filePoolSize)); + LOG_IF(FATAL, !conf->GetBoolValue("walfilepool.allocated_by_percent", + &walPoolOptions->allocatedByPercent)); + LOG_IF(FATAL, !conf->GetUInt32Value("walfilepool.allocated_percent", + &walPoolOptions->allocatedPercent)); + LOG_IF(FATAL, !conf->GetUInt32Value("walfilepool.thread_num", + &walPoolOptions->formatThreadNum)); + + std::string copysetUri; + LOG_IF(FATAL, + !conf->GetStringValue("copyset.raft_log_uri", ©setUri)); + curve::common::UriParser::ParseUri(copysetUri, + &walPoolOptions->copysetDir); + + std::string recycleUri; + LOG_IF(FATAL, + !conf->GetStringValue("copyset.recycler_uri", &recycleUri)); + curve::common::UriParser::ParseUri(recycleUri, + &walPoolOptions->recycleDir); + + walPoolOptions->isAllocated = [](const string& filename) { + return Trash::IsWALFile(filename); + }; ::memcpy( walPoolOptions->metaPath, metaUri.c_str(), metaUri.size()); } @@ -833,6 +909,16 @@ void ChunkServer::LoadConfigFromCmdline(common::Configuration *conf) { << "chunkFilePoolDir must be set when run chunkserver in command."; } + if (GetCommandLineFlagInfo("chunkFilePoolAllocatedPercent", &info)) { + conf->SetUInt32Value("chunkfilepool.allocate_percent", + FLAGS_chunkFilePoolAllocatedPercent); + } + + if (GetCommandLineFlagInfo("chunkFormatThreadNum", &info)) { + conf->SetUInt64Value("chunkfilepool.chunk_file_pool_format_thread_num", + FLAGS_chunkFormatThreadNum); + } + if (GetCommandLineFlagInfo("chunkFilePoolMetaPath", &info) && !info.is_default) { conf->SetStringValue( diff --git a/src/chunkserver/datastore/file_pool.cpp b/src/chunkserver/datastore/file_pool.cpp index d3f20feb6a..0c6489ed80 100644 --- a/src/chunkserver/datastore/file_pool.cpp +++ b/src/chunkserver/datastore/file_pool.cpp @@ -22,8 +22,10 @@ #include "src/chunkserver/datastore/file_pool.h" +#include #include #include +#include #include #include #include @@ -33,18 +35,19 @@ #include #include #include -#include #include +#include -#include "src/common/string_util.h" -#include "src/common/throttle.h" +#include "absl/utility/utility.h" #include "src/common/configuration.h" #include "src/common/crc32.h" #include "src/common/curve_define.h" - -#include "absl/utility/utility.h" +#include "src/common/string_util.h" +#include "src/common/throttle.h" using curve::common::kFilePoolMagic; +DEFINE_int64(formatInterval, 100, "Sets a interval between formatting."); +DEFINE_validator(formatInterval, brpc::PositiveInteger); namespace curve { namespace chunkserver { @@ -68,7 +71,6 @@ std::ostream& operator<<(std::ostream& os, const FilePoolMeta& meta) { << ", hasblocksize: " << meta.hasBlockSize << ", blocksize: " << meta.blockSize << ", filepoolpath: " << meta.filePoolPath; - return os; } @@ -220,20 +222,31 @@ FilePool::FilePool(std::shared_ptr fsptr) memset(writeBuffer_.get(), 0, poolOpt_.bytesPerWrite); } +FilePool::~FilePool() { UnInitialize(); } + bool FilePool::Initialize(const FilePoolOptions &cfopt) { poolOpt_ = cfopt; if (poolOpt_.getFileFromPool) { + currentdir_ = poolOpt_.filePoolDir; + currentState_.chunkSize = poolOpt_.fileSize; + currentState_.metaPageSize = poolOpt_.metaPageSize; if (!CheckValid()) { - LOG(ERROR) << "check valid failed!"; + LOG(ERROR) << "Check vaild failed!"; return false; } - if (fsptr_->DirExists(currentdir_)) { - return ScanInternal(); - } else { - LOG(ERROR) << "chunkfile pool not exists, inited failed!" - << " chunkfile pool path = " << currentdir_.c_str(); + + if (!ScanInternal()) { + LOG(ERROR) << "Scan pool files failed!"; + return false; + } + + if (!PrepareFormat()) { + LOG(ERROR) << "Prepare format failed!"; return false; } + + formatAlived_.store(true); + formatThread_ = Thread(&FilePool::FormatWorker, this); } else { currentdir_ = poolOpt_.filePoolDir; if (!fsptr_->DirExists(currentdir_.c_str())) { @@ -245,6 +258,15 @@ bool FilePool::Initialize(const FilePoolOptions &cfopt) { bool FilePool::CheckValid() { FilePoolMeta meta; + if (!fsptr_->FileExists(poolOpt_.metaPath)) { + LOG(INFO) << "Metafile in path '" << poolOpt_.metaPath + << "'not found, it's the first initialized."; + currentState_.chunkSize = poolOpt_.fileSize; + currentState_.metaPageSize = poolOpt_.metaPageSize; + currentState_.blockSize = poolOpt_.blockSize; + currentdir_ = poolOpt_.filePoolDir; + return true; + } int ret = FilePoolHelper::DecodeMetaInfoFromMetaFile( fsptr_, poolOpt_.metaPath, poolOpt_.metaFileSize, &meta); if (ret == -1) { @@ -375,6 +397,129 @@ void FilePool::CleanWorker() { } } +bool FilePool::PrepareFormat() { + curve::fs::FileSystemInfo finfo; + int r = fsptr_->Statfs(currentdir_, &finfo); + if (r != 0) { + LOG(ERROR) << "get disk usage info failed!"; + return false; + } + + if (poolOpt_.allocatedByPercent) { + poolOpt_.filePoolSize = finfo.total * poolOpt_.allocatedPercent / 100; + } + + uint64_t bytesPerPage = poolOpt_.fileSize + poolOpt_.metaFileSize; + uint64_t needSpace = + poolOpt_.filePoolSize - currentState_.chunkNum * bytesPerPage; + uint64_t vaildSpace = 0; + + if (poolOpt_.filePoolSize / bytesPerPage < currentState_.chunkNum) { + LOG(INFO) << "It is no need to format chunks."; + formatStat_.preAllocateNum = 0; + formatStat_.allocateChunkNum = 0; + return true; + } + + vaildSpace = finfo.available; + LOG(INFO) << "free space = " << finfo.available + << ", total space = " << finfo.total + << ", need space = " << needSpace; + if (vaildSpace < needSpace) { + LOG(ERROR) << "disk free space not enough."; + return false; + } + + formatStat_.preAllocateNum = needSpace / bytesPerPage; + formatStat_.allocateChunkNum = 0; + LOG(INFO) << "preAllocateNum = " << formatStat_.preAllocateNum; + return true; +} + +bool FilePool::WaitoFormatDoneForTesting() { + std::unique_lock lk(mtx_); + if (formatStat_.allocateChunkNum.load() != formatStat_.preAllocateNum) { + cond_.wait(lk, [&]() { + return formatStat_.allocateChunkNum.load() == + formatStat_.preAllocateNum; + }); + } + lk.unlock(); + if (formatThread_.joinable()) { + formatThread_.join(); + } + return true; +} + +bool FilePool::StopFormatting() { + if (formatAlived_.exchange(false)) { + LOG(INFO) << "Stop formatting..."; + if (formatThread_.joinable()) { + formatThread_.join(); + } + LOG(INFO) << "Stop format thread ok."; + } + return true; +} +int FilePool::FormatTask(uint64_t indexOffset, + std::atomic* allocatIndex) { + LOG(INFO) << "format thread has been work!"; + while (!this->formatStat_.isWrong.load() && this->formatAlived_.load()) { + uint32_t chunkIndex = 0; + if ((chunkIndex = allocatIndex->fetch_add(1)) >= + formatStat_.preAllocateNum) { + allocatIndex->fetch_sub(1); + break; + } + std::string chunkPath = this->currentdir_ + "/" + + std::to_string(chunkIndex + indexOffset) + + kCleanChunkSuffix_; + this->formatSleeper_.wait_for( + std::chrono::milliseconds{FLAGS_formatInterval}); + int res = this->AllocateChunk(chunkPath); + if (res != 0) { + this->formatStat_.isWrong.store(true); + LOG(ERROR) << "Format ERROR!"; + break; + } + this->mtx_.lock(); + cleanChunks_.push_back(chunkIndex + indexOffset); + this->currentState_.cleanChunksLeft++; + this->currentState_.preallocatedChunksLeft++; + this->currentState_.chunkNum++; + this->formatStat_.allocateChunkNum++; + this->mtx_.unlock(); + this->cond_.notify_all(); + } + LOG(INFO) << "format thread has done!"; + return 0; +} + +int FilePool::FormatWorker() { + std::vector threads; + uint64_t offset = + this->currentmaxfilenum_.fetch_add(formatStat_.preAllocateNum); + std::atomic allocatIndex{0}; + + for (uint32_t i = 0; i < poolOpt_.formatThreadNum; i++) { + threads.emplace_back( + Thread(&FilePool::FormatTask, this, offset, &allocatIndex)); + } + + for (auto& thread : threads) { + if (thread.joinable()) { + thread.join(); + } + } + LOG(INFO) << "format worker done"; + + if (formatStat_.isWrong.load()) { + LOG(ERROR) << "Chunk format failed!"; + return -1; + } + return 0; +} + bool FilePool::StartCleaning() { if (poolOpt_.needClean && !cleanAlived_.exchange(true)) { ReadWriteThrottleParams params; @@ -400,9 +545,8 @@ bool FilePool::StopCleaning() { } bool FilePool::GetChunk(bool needClean, uint64_t *chunkid, bool *isCleaned) { - auto pop = [&](std::vector *chunks, uint64_t *chunksLeft, + auto pop = [&](std::vector* chunks, uint64_t* chunksLeft, bool isCleanChunks) -> bool { - std::unique_lock lk(mtx_); if (chunks->empty()) { return false; } @@ -415,16 +559,28 @@ bool FilePool::GetChunk(bool needClean, uint64_t *chunkid, bool *isCleaned) { return true; }; - if (!needClean) { - return pop(&dirtyChunks_, ¤tState_.dirtyChunksLeft, false) || - pop(&cleanChunks_, ¤tState_.cleanChunksLeft, true); - } + auto wake_up = [&]() { + return (formatStat_.allocateChunkNum.load() == + formatStat_.preAllocateNum) // NOLINT + || !(dirtyChunks_.empty() && cleanChunks_.empty()); + }; - // Need clean chunk - *isCleaned = false; - bool ret = pop(&cleanChunks_, ¤tState_.cleanChunksLeft, true) || - pop(&dirtyChunks_, ¤tState_.dirtyChunksLeft, false); + bool ret = true; + { + std::unique_lock lk(mtx_); + if (formatStat_.allocateChunkNum.load() != formatStat_.preAllocateNum) { + cond_.wait(lk, wake_up); + } + if (!needClean) { + return pop(&dirtyChunks_, ¤tState_.dirtyChunksLeft, false) || + pop(&cleanChunks_, ¤tState_.cleanChunksLeft, true); + } + // Need clean chunk + *isCleaned = false; + ret = pop(&cleanChunks_, ¤tState_.cleanChunksLeft, true) || + pop(&dirtyChunks_, ¤tState_.dirtyChunksLeft, false); + } if (true == ret && false == *isCleaned && CleanChunk(*chunkid, true)) { *isCleaned = true; } @@ -638,7 +794,7 @@ int FilePool::RecycleFile(const std::string &chunkpath) { void FilePool::UnInitialize() { currentdir_ = ""; - + StopFormatting(); std::unique_lock lk(mtx_); dirtyChunks_.clear(); cleanChunks_.clear(); @@ -647,8 +803,17 @@ void FilePool::UnInitialize() { bool FilePool::ScanInternal() { uint64_t maxnum = 0; std::vector tmpvec; + int ret = 0; LOG(INFO) << "scan dir" << currentdir_; - int ret = fsptr_->List(currentdir_.c_str(), &tmpvec); + if (!fsptr_->DirExists(currentdir_.c_str())) { + ret = fsptr_->Mkdir(currentdir_.c_str()); + if (ret != 0) { + LOG(ERROR) << "Mkdir [" << currentdir_ << "]" + << " failed!"; + return false; + } + } + ret = fsptr_->List(currentdir_.c_str(), &tmpvec); if (ret < 0) { LOG(ERROR) << "list file pool dir failed!"; return false; @@ -709,6 +874,10 @@ bool FilePool::ScanInternal() { } } + currentState_.chunkNum = tmpvec.size(); + currentState_.chunkNum += CountAllocatedNum(poolOpt_.copysetDir); + currentState_.chunkNum += CountAllocatedNum(poolOpt_.recycleDir); + std::unique_lock lk(mtx_); currentmaxfilenum_.store(maxnum + 1); currentState_.dirtyChunksLeft = dirtyChunks_.size(); @@ -721,6 +890,30 @@ bool FilePool::ScanInternal() { return true; } +uint64_t FilePool::CountAllocatedNum(const std::string& path) { + std::vector files; + if ("" == path || 0 != fsptr_->List(path, &files)) { + LOG(ERROR) << "FilePool failed to list files in " << path; + return 0; + } + + // Traverse subdirectories + uint32_t chunkNum = 0; + for (auto& file : files) { + std::string filePath = path + "/" + file; + bool isDir = fsptr_->DirExists(filePath); + if (!isDir) { + LOG(INFO) << "path = " << filePath; + if (poolOpt_.isAllocated(file)) { + ++chunkNum; + } + } else { + chunkNum += CountAllocatedNum(filePath); + } + } + return chunkNum; +} + size_t FilePool::Size() { std::unique_lock lk(mtx_); return currentState_.preallocatedChunksLeft; @@ -730,6 +923,10 @@ FilePoolState FilePool::GetState() const { return currentState_; } +const ChunkFormatStat& FilePool::GetChunkFormatStat() const { + return formatStat_; +} + uint32_t FilePoolMeta::Crc32() const { const size_t size = sizeof(kFilePoolMagic) + sizeof(chunkSize) + sizeof(metaPageSize) + filePoolPath.size() + diff --git a/src/chunkserver/datastore/file_pool.h b/src/chunkserver/datastore/file_pool.h index f3417e160a..16b2311982 100644 --- a/src/chunkserver/datastore/file_pool.h +++ b/src/chunkserver/datastore/file_pool.h @@ -25,19 +25,20 @@ #include -#include +#include +#include +#include +#include #include // NOLINT -#include +#include #include -#include -#include -#include +#include +#include "include/curve_compiler_specific.h" #include "src/common/concurrent/concurrent.h" #include "src/common/interruptible_sleeper.h" #include "src/common/throttle.h" #include "src/fs/local_filesystem.h" -#include "include/curve_compiler_specific.h" using curve::fs::LocalFileSystem; using curve::common::Thread; @@ -65,6 +66,17 @@ struct FilePoolOptions { // retry times for get file uint16_t retryTimes; + bool allocatedByPercent; + uint32_t allocatedPercent; + uint32_t preAllocateNum; + uint64_t filePoolSize; + uint32_t formatThreadNum; + + std::string copysetDir; + std::string recycleDir; + + std::function isAllocated; + FilePoolOptions() { getFileFromPool = true; needClean = false; @@ -75,6 +87,11 @@ struct FilePoolOptions { metaPageSize = 0; retryTimes = 5; blockSize = 0; + allocatedByPercent = false; + allocatedPercent = 0; + preAllocateNum = 0; + filePoolSize = 0; + formatThreadNum = 1; ::memset(metaPath, 0, 256); ::memset(filePoolDir, 0, 256); } @@ -87,7 +104,8 @@ struct FilePoolState { uint64_t cleanChunksLeft = 0; // How many pre-allocated chunks are not used by the datastore uint64_t preallocatedChunksLeft = 0; - + // Total num of chunks in the datastore + uint64_t chunkNum; // chunksize uint32_t chunkSize = 0; // metapage size @@ -126,6 +144,12 @@ struct FilePoolMeta { uint32_t Crc32() const; }; +struct ChunkFormatStat { + std::atomic isWrong{false}; + std::atomic allocateChunkNum{0}; + uint32_t preAllocateNum = 0; +}; + class FilePoolHelper { public: static const char* kFileSize; @@ -168,7 +192,7 @@ class FilePoolHelper { class CURVE_CACHELINE_ALIGNMENT FilePool { public: explicit FilePool(std::shared_ptr fsptr); - virtual ~FilePool() = default; + virtual ~FilePool(); /** * Initialization function @@ -237,12 +261,30 @@ class CURVE_CACHELINE_ALIGNMENT FilePool { */ bool StopCleaning(); + /** + * @brief: Wait for format thread done. + * @return: Return true if success. + */ + bool WaitoFormatDoneForTesting(); + + /** + * @brief: Get the format status of FilePool + * @return: Return the format status. + */ + virtual const ChunkFormatStat& GetChunkFormatStat() const; + private: // Traverse the pre-allocated chunk information from the // chunkfile pool directory bool ScanInternal(); - // Check whether the chunkfile pool pre-allocation is legal + // Prepare for format. + bool PrepareFormat(); + // bool CheckValid(); + // Count the num of files that has been allocated. + uint64_t CountAllocatedNum(const std::string& path); + // Check whether pool file is legal. + bool CheckPoolFile(const std::string& file); /** * Perform metapage assignment for the new chunkfile * @param: sourcepath is the file path to be written @@ -282,6 +324,19 @@ class CURVE_CACHELINE_ALIGNMENT FilePool { */ bool CleaningChunk(); + int FormatTask(uint64_t indexOffset, std::atomic* allocatIndex); + + /** + * @brief: The function of thread for formatting chunk + */ + int FormatWorker(); + + /** + * @brief: Stop thread for formatting chunk + * @return: Return true if success, otherwise return false + */ + bool StopFormatting(); + /** * @brief: The function of thread for cleaning chunk */ @@ -300,6 +355,9 @@ class CURVE_CACHELINE_ALIGNMENT FilePool { // Protect dirtyChunks_, cleanChunks_ std::mutex mtx_; + // Wait for GetChunk + std::condition_variable cond_; + // Current FilePool pre-allocated files, folder path std::string currentdir_; @@ -331,9 +389,21 @@ class CURVE_CACHELINE_ALIGNMENT FilePool { // The throttle iops for cleaning chunk (4KB/IO) Throttle cleanThrottle_; + // Whether the format thread is alive + Atomic formatAlived_{true}; + + // Thread for format chunks. + Thread formatThread_; + + // Stat for format chunks. + ChunkFormatStat formatStat_; + // Sleeper for cleaning chunk thread InterruptibleSleeper cleanSleeper_; + // Sleeper for formatting chunk thread + InterruptibleSleeper formatSleeper_; + // The buffer for write chunk file std::unique_ptr writeBuffer_; }; diff --git a/src/chunkserver/heartbeat.cpp b/src/chunkserver/heartbeat.cpp index 0e756b29c6..b81fe6bdb3 100644 --- a/src/chunkserver/heartbeat.cpp +++ b/src/chunkserver/heartbeat.cpp @@ -268,10 +268,21 @@ int Heartbeat::BuildRequest(HeartbeatRequest* req) { * walSegmentFileSize; uint64_t chunkPoolSize = options_.chunkFilePool->Size() * options_.chunkFilePool->GetFilePoolOpt().fileSize; + + // compute format progress rate. + const ChunkFormatStat& formatStat = + options_.chunkFilePool->GetChunkFormatStat(); // NOLINT + stats->set_chunkfilepoolsize(chunkPoolSize); stats->set_chunksizeusedbytes(usedChunkSize+usedWalSegmentSize); stats->set_chunksizeleftbytes(leftChunkSize+leftWalSegmentSize); stats->set_chunksizetrashedbytes(trashedChunkSize); + if (formatStat.preAllocateNum != 0) { + stats->set_chunkfilepoolformatpercent( + 100 * formatStat.allocateChunkNum / formatStat.preAllocateNum); + } else { + stats->set_chunkfilepoolformatpercent(100); + } req->set_allocated_stats(stats); size_t cap, avail; @@ -391,7 +402,8 @@ int Heartbeat::SendHeartbeat(const HeartbeatRequest& request, cntl.ErrorCode() == brpc::ELOGOFF || cntl.ErrorCode() == brpc::ERPCTIMEDOUT) { LOG(WARNING) << "current mds: " << mdsEps_[inServiceIndex_] - << " is shutdown or going to quit"; + << " is shutdown or going to quit," + << cntl.ErrorText(); inServiceIndex_ = (inServiceIndex_ + 1) % mdsEps_.size(); LOG(INFO) << "next heartbeat switch to " << mdsEps_[inServiceIndex_]; diff --git a/src/chunkserver/trash.h b/src/chunkserver/trash.h index ff037db8a4..a3a3c89d53 100644 --- a/src/chunkserver/trash.h +++ b/src/chunkserver/trash.h @@ -73,6 +73,25 @@ class Trash { */ uint32_t GetChunkNum() {return chunkNum_.load();} + /** + * @brief is WAL or not ? + * + * @param fileName file name + * + * @retval true yes + * @retval false no + */ + static bool IsWALFile(const std::string& fileName); + + /* + * @brief IsChunkOrSnapShotFile 是否为chunk或snapshot文件 + * + * @param[in] chunkName 文件名 + * + * @return true-符合chunk或snapshot文件命名规则 + */ + static bool IsChunkOrSnapShotFile(const std::string& chunkName); + private: /* * @brief DeleteEligibleFileInTrashInterval 每隔一段时间进行trash物理空间回收 @@ -98,15 +117,6 @@ class Trash { */ bool IsCopysetInTrash(const std::string &dirName); - /* - * @brief IsChunkOrSnapShotFile 是否为chunk或snapshot文件 - * - * @param[in] chunkName 文件名 - * - * @return true-符合chunk或snapshot文件命名规则 - */ - bool IsChunkOrSnapShotFile(const std::string &chunkName); - /* * @brief Recycle Chunkfile and wal file in Copyset * @@ -134,19 +144,7 @@ class Trash { * @retval true success * @retval false failure */ - bool RecycleWAL( - const std::string &filepath, const std::string &filename); - - - /** - * @brief is WAL or not ? - * - * @param fileName file name - * - * @retval true yes - * @retval false no - */ - bool IsWALFile(const std::string &fileName); + bool RecycleWAL(const std::string& filepath, const std::string& filename); /* * @brief 统计copyset目录中的chunk个数 diff --git a/src/common/bytes_convert.h b/src/common/bytes_convert.h new file mode 100644 index 0000000000..3093567d90 --- /dev/null +++ b/src/common/bytes_convert.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2023-10-06 + * Author: yyyyufeng + */ + +#ifndef SRC_COMMON_BYTES_CONVERT_H_ +#define SRC_COMMON_BYTES_CONVERT_H_ + +#include +#include + +namespace curve { +namespace common { + +constexpr uint64_t kKiB = 1024ULL; +constexpr uint64_t kMiB = 1024ULL * kKiB; +constexpr uint64_t kGiB = 1024ULL * kMiB; +constexpr uint64_t kTiB = 1024ULL * kGiB; + +/** + * @brief convert string to bytes + * @return true if success + */ +inline bool ToNumbericByte(const std::string& source, uint64_t* target) { + int len = source.size(); + if (source[len - 1] >= '0' && source[len - 1] <= '9') { + *target = std::stoul(source); + return true; + } + if (len < 3 || (source[len - 1] != 'b' && source[len - 1] != 'B')) { + return false; + } + *target = 0; + for (int i = 0; i < len - 2; i++) { + char ch = source[i]; + if (ch < '0' || ch > '9') { + return false; + } + *target *= 10; + *target += (ch - '0'); + } + bool ret = true; + switch (source[len - 2]) { + case 'k': + case 'K': + *target *= kKiB; + break; + case 'M': + case 'm': + *target *= kMiB; + break; + case 'G': + case 'g': + *target *= kGiB; + break; + case 'T': + case 't': + *target *= kTiB; + break; + default: + ret = false; + break; + } + return ret; +} + +} // namespace common +} // namespace curve +#endif // SRC_COMMON_BYTES_CONVERT_H_ diff --git a/src/mds/heartbeat/heartbeat_manager.cpp b/src/mds/heartbeat/heartbeat_manager.cpp index b5100eaa73..188ee03112 100644 --- a/src/mds/heartbeat/heartbeat_manager.cpp +++ b/src/mds/heartbeat/heartbeat_manager.cpp @@ -137,6 +137,11 @@ void HeartbeatManager::UpdateChunkServerStatistics( stat.chunkFilepoolSize = request.stats().chunkfilepoolsize(); } + if (request.stats().has_chunkfilepoolformatpercent()) { + stat.chunkFilepoolFormatPercent = + request.stats().chunkfilepoolformatpercent(); // NOLINT + } + for (int i = 0; i < request.copysetinfos_size(); i++) { CopysetStat cstat; cstat.logicalPoolId = request.copysetinfos(i).logicalpoolid(); diff --git a/src/mds/topology/topology_service.cpp b/src/mds/topology/topology_service.cpp index 2671b5d8d5..95ad76fcbc 100644 --- a/src/mds/topology/topology_service.cpp +++ b/src/mds/topology/topology_service.cpp @@ -1154,6 +1154,19 @@ void TopologyServiceImpl::ListUnAvailCopySets( } } +void TopologyServiceImpl::ListChunkFormatStatus( + google::protobuf::RpcController* cntl_base, + const ListChunkFormatStatusRequest* request, + ListChunkFormatStatusResponse* response, google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received request[log_id=" << cntl->log_id() << "] from " + << cntl->remote_side() << " to " << cntl->local_side() + << ". [ListChunkFormatStatus] " << request->DebugString(); + topology_->ListChunkFormatStatus(request, response); +} + } // namespace topology } // namespace mds } // namespace curve diff --git a/src/mds/topology/topology_service.h b/src/mds/topology/topology_service.h index a3eeaed257..ff56fb6dbd 100644 --- a/src/mds/topology/topology_service.h +++ b/src/mds/topology/topology_service.h @@ -232,6 +232,11 @@ class TopologyServiceImpl : public TopologyService { const ListUnAvailCopySetsRequest* request, ListUnAvailCopySetsResponse* response, google::protobuf::Closure* done); + virtual void ListChunkFormatStatus( + google::protobuf::RpcController* cntl_base, + const ListChunkFormatStatusRequest* request, + ListChunkFormatStatusResponse* response, + google::protobuf::Closure* done); private: std::shared_ptr topology_; diff --git a/src/mds/topology/topology_service_manager.cpp b/src/mds/topology/topology_service_manager.cpp index ca0e87e4a8..ce0b6822a4 100644 --- a/src/mds/topology/topology_service_manager.cpp +++ b/src/mds/topology/topology_service_manager.cpp @@ -1795,6 +1795,25 @@ void TopologyServiceManager::ConvertCopyset(const CopySetInfo& in, out->set_lastscanconsistent(in.GetLastScanConsistent()); } +void TopologyServiceManager::ListChunkFormatStatus( + const ListChunkFormatStatusRequest* request, + ListChunkFormatStatusResponse* response) { + std::vector chunkserverlist = + topology_->GetChunkServerInCluster(); + for (auto& id : chunkserverlist) { + ChunkServer cs; + ChunkServerStat stat; + if (topology_->GetChunkServer(id, &cs) && + topoStat_->GetChunkServerStat(id, &stat)) { + ChunkFormatStatus* status = response->add_chunkformatstatus(); + status->set_chunkserverid(id); + status->set_ip(cs.GetHostIp()); + status->set_port(cs.GetPort()); + status->set_formatpercent(stat.chunkFilepoolFormatPercent); + } + } +} + } // namespace topology } // namespace mds } // namespace curve diff --git a/src/mds/topology/topology_service_manager.h b/src/mds/topology/topology_service_manager.h index 2de2ab5c1d..bcbfc98f20 100644 --- a/src/mds/topology/topology_service_manager.h +++ b/src/mds/topology/topology_service_manager.h @@ -198,6 +198,10 @@ class TopologyServiceManager { ChunkServerIdType id, const std::vector ©setInfos); + virtual void ListChunkFormatStatus( + const ListChunkFormatStatusRequest* request, + ListChunkFormatStatusResponse* response); + private: /** * @brief create copyset for logical pool diff --git a/src/mds/topology/topology_stat.h b/src/mds/topology/topology_stat.h index 1dfba17e00..f2d2b89943 100644 --- a/src/mds/topology/topology_stat.h +++ b/src/mds/topology/topology_stat.h @@ -85,17 +85,20 @@ struct ChunkServerStat { uint64_t chunkSizeTrashedBytes; // Size of chunkfilepool uint64_t chunkFilepoolSize; + // Rate of chunkfilepool format + uint32_t chunkFilepoolFormatPercent; // Copyset statistic std::vector copysetStats; - ChunkServerStat() : - leaderCount(0), - copysetCount(0), - readRate(0), - writeRate(0), - readIOPS(0), - writeIOPS(0) {} + ChunkServerStat() + : leaderCount(0), + copysetCount(0), + readRate(0), + writeRate(0), + readIOPS(0), + writeIOPS(0), + chunkFilepoolFormatPercent(0) {} }; /** diff --git a/src/tools/curve_tool_define.h b/src/tools/curve_tool_define.h index a392b807bd..8800bf847c 100644 --- a/src/tools/curve_tool_define.h +++ b/src/tools/curve_tool_define.h @@ -57,6 +57,7 @@ const char kClientListCmd[] = "client-list"; const char kSnapshotCloneStatusCmd[] = "snapshot-clone-status"; const char kClusterStatusCmd[] = "cluster-status"; const char kScanStatusCmd[] = "scan-status"; +const char kFormatStatusCmd[] = "format-status"; // NameSpaceTool相关命令 const char kGetCmd[] = "get"; diff --git a/src/tools/mds_client.cpp b/src/tools/mds_client.cpp index 7a119c77bc..4db8bb81f0 100644 --- a/src/tools/mds_client.cpp +++ b/src/tools/mds_client.cpp @@ -1176,5 +1176,24 @@ int MDSClient::ListPoolset(std::vector* poolsets) { return -1; } +int MDSClient::ListChunkFormatStatus( + std::vector* formatStatuses) { + assert(formatStatuses != nullptr); + curve::mds::topology::ListChunkFormatStatusRequest request; + curve::mds::topology::ListChunkFormatStatusResponse response; + curve::mds::topology::TopologyService_Stub stub(&channel_); + + auto fp = &curve::mds::topology::TopologyService_Stub:: + ListChunkFormatStatus; // NOLINT + if (0 != SendRpcToMds(&request, &response, &stub, fp)) { + std::cout << "ListChunkFormatStatus fail" << std::endl; + return -1; + } + for (auto stat : response.chunkformatstatus()) { + formatStatuses->push_back(stat); + } + return 0; +} + } // namespace tool } // namespace curve diff --git a/src/tools/mds_client.h b/src/tools/mds_client.h index 05bac69cd5..fbbc94ffab 100644 --- a/src/tools/mds_client.h +++ b/src/tools/mds_client.h @@ -47,28 +47,31 @@ #include "src/tools/common.h" #include "src/tools/curve_tool_define.h" +using curve::common::ChunkServerLocation; +using curve::common::CopysetInfo; using curve::mds::FileInfo; +using curve::mds::PageFileChunkInfo; using curve::mds::PageFileSegment; using curve::mds::StatusCode; -using curve::mds::PageFileChunkInfo; -using curve::mds::topology::kTopoErrCodeSuccess; +using curve::mds::topology::ChunkFormatStatus; +using curve::mds::topology::ChunkServerIdType; using curve::mds::topology::ChunkServerInfo; -using curve::common::ChunkServerLocation; +using curve::mds::topology::ChunkServerStatus; +using curve::mds::topology::CopySetIdType; using curve::mds::topology::CopySetServerInfo; -using curve::mds::topology::ServerInfo; -using curve::mds::topology::ZoneInfo; -using curve::mds::topology::PhysicalPoolInfo; +using curve::mds::topology::GetChunkServerInfoRequest; +using curve::mds::topology::GetCopySetsInChunkServerRequest; +using curve::mds::topology::kTopoErrCodeSuccess; +using curve::mds::topology::ListChunkFormatStatusRequest; +using curve::mds::topology::ListChunkServerRequest; using curve::mds::topology::LogicalPoolInfo; -using curve::common::CopysetInfo; +using curve::mds::topology::PhysicalPoolInfo; +using curve::mds::topology::PoolIdType; using curve::mds::topology::ServerIdType; +using curve::mds::topology::ServerInfo; using curve::mds::topology::ZoneIdType; -using curve::mds::topology::PoolIdType; -using curve::mds::topology::CopySetIdType; -using curve::mds::topology::ChunkServerIdType; -using curve::mds::topology::ChunkServerStatus; -using curve::mds::topology::ListChunkServerRequest; -using curve::mds::topology::GetChunkServerInfoRequest; -using curve::mds::topology::GetCopySetsInChunkServerRequest; +using curve::mds::topology::ZoneInfo; + using curve::mds::schedule::RapidLeaderScheduleRequst; using curve::mds::schedule::RapidLeaderScheduleResponse; using curve::common::Authenticator; @@ -480,6 +483,8 @@ class MDSClient { int ListPoolset(std::vector* poolsets); + int ListChunkFormatStatus(std::vector* formatStatuses); + private: /** * @brief 切换mds diff --git a/src/tools/status_tool.cpp b/src/tools/status_tool.cpp index bb9e8f97b9..4444f51fd2 100644 --- a/src/tools/status_tool.cpp +++ b/src/tools/status_tool.cpp @@ -114,18 +114,14 @@ bool StatusTool::CommandNeedSnapshotClone(const std::string& command) { } bool StatusTool::SupportCommand(const std::string& command) { - return (command == kSpaceCmd || command == kStatusCmd - || command == kChunkserverListCmd - || command == kChunkserverStatusCmd - || command == kMdsStatusCmd - || command == kEtcdStatusCmd - || command == kClientStatusCmd - || command == kClientListCmd - || command == kSnapshotCloneStatusCmd - || command == kClusterStatusCmd - || command == kServerListCmd - || command == kLogicalPoolList - || command == kScanStatusCmd); + return (command == kSpaceCmd || command == kStatusCmd || + command == kChunkserverListCmd || + command == kChunkserverStatusCmd || command == kMdsStatusCmd || + command == kEtcdStatusCmd || command == kClientStatusCmd || + command == kClientListCmd || command == kSnapshotCloneStatusCmd || + command == kClusterStatusCmd || command == kServerListCmd || + command == kLogicalPoolList || command == kScanStatusCmd || + command == kFormatStatusCmd); } void StatusTool::PrintHelp(const std::string& cmd) { @@ -225,6 +221,21 @@ int StatusTool::SpaceCmd() { return 0; } +int StatusTool::FormatStatusCmd() { + std::vector formatStatus; + int res = mdsClient_->ListChunkFormatStatus(&formatStatus); + if (res != 0) { + std::cout << "ListChunkserversInCluster fail!" << std::endl; + return -1; + } + for (auto stat : formatStatus) { + std::cout << "ip:" << stat.ip() << " port:" << stat.port() + << " id:" << stat.chunkserverid() + << " format percent:" << stat.formatpercent() << std::endl; + } + return 0; +} + int StatusTool::ChunkServerListCmd() { std::vector chunkservers; int res = mdsClient_->ListChunkServersInCluster(&chunkservers); @@ -1138,6 +1149,8 @@ int StatusTool::RunCommand(const std::string &cmd) { return ClientListCmd(); } else if (cmd == kScanStatusCmd) { return ScanStatusCmd(); + } else if (cmd == kFormatStatusCmd) { + return FormatStatusCmd(); } else { std::cout << "Command not supported!" << std::endl; return -1; diff --git a/src/tools/status_tool.h b/src/tools/status_tool.h index 2b54d70943..82b776fa73 100644 --- a/src/tools/status_tool.h +++ b/src/tools/status_tool.h @@ -150,6 +150,7 @@ class StatusTool : public CurveTool { int PrintClientStatus(); int ClientListCmd(); int ScanStatusCmd(); + int FormatStatusCmd(); void PrintCsLeftSizeStatistics( const std::string &name, const std::map> &poolLeftSize); diff --git a/test/chunkserver/datastore/filepool_mock_unittest.cpp b/test/chunkserver/datastore/filepool_mock_unittest.cpp index f9fc0502e1..5a70d46551 100644 --- a/test/chunkserver/datastore/filepool_mock_unittest.cpp +++ b/test/chunkserver/datastore/filepool_mock_unittest.cpp @@ -102,6 +102,7 @@ class CSChunkfilePoolMockTest : public testing::Test { } void FakeMetaFile() { + EXPECT_CALL(*lfs_, FileExists(poolMetaPath)).WillOnce(Return(true)); EXPECT_CALL(*lfs_, Open(poolMetaPath, _)) .WillOnce(Return(100)); EXPECT_CALL(*lfs_, Read(100, NotNull(), 0, metaFileSize)) @@ -391,6 +392,7 @@ TEST_F(CSChunkfilePoolMockTest, InitializeTest) { char buf[metaFileSize] = {0}; EXPECT_CALL(*lfs_, Open(poolMetaPath, _)) .WillOnce(Return(1)); + EXPECT_CALL(*lfs_, FileExists(poolMetaPath)).WillOnce(Return(true)); EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, metaFileSize)) .WillOnce(DoAll(SetArrayArgument<1>(buf, buf + metaFileSize), Return(metaFileSize))); @@ -404,7 +406,8 @@ TEST_F(CSChunkfilePoolMockTest, InitializeTest) { FakeMetaFile(); EXPECT_CALL(*lfs_, DirExists(_)) .WillOnce(Return(false)); - ASSERT_EQ(false, pool.Initialize(options)); + ASSERT_EQ(true, pool.Initialize(options)); + pool.WaitoFormatDoneForTesting(); } // 当前目录存在，list目录失败 { diff --git a/test/chunkserver/datastore/filepool_unittest.cpp b/test/chunkserver/datastore/filepool_unittest.cpp index 480f6da72a..ea1592f62b 100644 --- a/test/chunkserver/datastore/filepool_unittest.cpp +++ b/test/chunkserver/datastore/filepool_unittest.cpp @@ -169,7 +169,7 @@ bool CheckFileOpenOrNot(const std::string& filename) { return out.find("No such file or directory") != out.npos; } -TEST_P(CSFilePool_test, InitializeTest) { +TEST_P(CSFilePool_test, InitializeNomalTest) { std::string filePool = "./cspooltest/filePool.meta"; const std::string filePoolPath = FILEPOOL_DIR; @@ -216,6 +216,64 @@ TEST_P(CSFilePool_test, InitializeTest) { fsptr->Delete("./cspooltest/filePool.meta3"); } +TEST_P(CSFilePool_test, InitializeFormatTest) { + std::string filePool = "./cspooltest/filePool1.meta"; + const std::string filePoolPath = POOL1_DIR; + std::string filePath = "./cspooltest/file"; + + FilePoolOptions cfop; + cfop.fileSize = 4096; + cfop.metaPageSize = 4096; + cfop.blockSize = 4096; + + cfop.formatThreadNum = 2; + memcpy(cfop.metaPath, filePool.c_str(), filePool.size()); + memcpy(cfop.filePoolDir, filePoolPath.c_str(), filePoolPath.size()); + + char metaPage[cfop.metaPageSize]; // NOLINT + + { + // meta file not exit. + FilePool pool(fsptr); + cfop.filePoolSize = 8192 * 10; + ASSERT_TRUE(pool.Initialize(cfop)); + ASSERT_EQ(0, pool.GetFile(filePath, (const char*)&metaPage)); + ASSERT_EQ(0, pool.RecycleFile(filePath)); + pool.WaitoFormatDoneForTesting(); + ASSERT_EQ(pool.Size(), 10); + } + + { + // filepool is not empty. + FilePool pool(fsptr); + cfop.filePoolSize = 8192 * 20; + ASSERT_TRUE(pool.Initialize(cfop)); + pool.WaitoFormatDoneForTesting(); + ASSERT_EQ(pool.Size(), 20); + } + + { + // the chunk num of filepool is less than preAllcateNum. + FilePool pool(fsptr); + cfop.filePoolSize = 8192 * 10; + ASSERT_TRUE(pool.Initialize(cfop)); + pool.WaitoFormatDoneForTesting(); + ASSERT_EQ(20, pool.Size()); + } + + { + // get file while fil epool is formatting. + FilePool pool(fsptr); + std::string filePath = "./cspooltest/file"; + cfop.filePoolSize = 8192 * 100; + ASSERT_TRUE(pool.Initialize(cfop)); + ASSERT_EQ(0, pool.GetFile(filePath, (const char*)&metaPage)); + ASSERT_EQ(0, pool.RecycleFile(filePath)); + sleep(3); + ASSERT_GT(pool.Size(), 20); + } +} + TEST_P(CSFilePool_test, GetFileTest) { std::string filePool = "./cspooltest/filePool.meta"; FilePoolOptions cfop; diff --git a/test/integration/chunkserver/chunkserver_basic_test.cpp b/test/integration/chunkserver/chunkserver_basic_test.cpp index bc922d19e2..a36bfedcee 100644 --- a/test/integration/chunkserver/chunkserver_basic_test.cpp +++ b/test/integration/chunkserver/chunkserver_basic_test.cpp @@ -95,6 +95,7 @@ class ChunkServerIoTest : public testing::Test { cg1_.SetKV("copyset.election_timeout_ms", "3000"); cg1_.SetKV("copyset.snapshot_interval_s", "60"); cg1_.SetKV("chunkfilepool.enable_get_chunk_from_pool", "true"); + cg1_.SetKV("chunkfilepool.allocated_by_percent", "false"); externalIp_ = butil::my_ip_cstr(); cg1_.SetKV("global.external_ip", externalIp_); cg1_.SetKV("global.enable_external_server", "true"); diff --git a/test/integration/chunkserver/chunkserver_concurrent_test.cpp b/test/integration/chunkserver/chunkserver_concurrent_test.cpp index b38f819da7..a5ac75a823 100644 --- a/test/integration/chunkserver/chunkserver_concurrent_test.cpp +++ b/test/integration/chunkserver/chunkserver_concurrent_test.cpp @@ -187,6 +187,7 @@ class ChunkServerConcurrentFromFilePoolTest : public testing::Test { cg1.SetKV("copyset.election_timeout_ms", "3000"); cg1.SetKV("copyset.snapshot_interval_s", "60"); cg1.SetKV("chunkfilepool.enable_get_chunk_from_pool", "true"); + cg1.SetKV("chunkfilepool.allocated_by_percent", "false"); cg1.SetKV("mds.listen.addr", kFakeMdsAddr); ASSERT_TRUE(cg1.Generate()); diff --git a/test/integration/client/chunkserver_exception_test.cpp b/test/integration/client/chunkserver_exception_test.cpp index dca71bdaf3..aa676fc718 100644 --- a/test/integration/client/chunkserver_exception_test.cpp +++ b/test/integration/client/chunkserver_exception_test.cpp @@ -57,67 +57,67 @@ const std::vector mdsConf{ }; const std::vector chunkserverConf4{ - { "-chunkServerStoreUri=local://./moduleException4/" }, - { "-chunkServerMetaUri=local://./moduleException4/chunkserver.dat" }, - { "-copySetUri=local://./moduleException4/copysets" }, - { "-raftSnapshotUri=curve://./moduleException4/copysets" }, - { "-raftLogUri=curve://./moduleException4/copysets" }, - { "-recycleUri=local://./moduleException4/recycler" }, - { "-chunkFilePoolDir=./moduleException4/chunkfilepool/" }, - { "-chunkFilePoolMetaPath=./moduleException4/chunkfilepool.meta" }, - { "-conf=./conf/chunkserver.conf.example" }, - { "-raft_sync_segments=true" }, - { "--log_dir=./runlog/ChunkserverException" }, - { "-chunkServerIp=127.0.0.1" }, - { "-chunkServerPort=22125" }, - { "-enableChunkfilepool=false" }, - { "-mdsListenAddr=127.0.0.1:22122,127.0.0.1:22123,127.0.0.1:22124" }, - { "-enableWalfilepool=false" }, - { "-walFilePoolDir=./moduleException4/walfilepool/" }, - { "-walFilePoolMetaPath=./moduleException4/walfilepool.meta" } -}; + {"-chunkServerStoreUri=local://./moduleException4/"}, + {"-chunkServerMetaUri=local://./moduleException4/chunkserver.dat"}, + {"-copySetUri=local://./moduleException4/copysets"}, + {"-raftSnapshotUri=curve://./moduleException4/copysets"}, + {"-raftLogUri=curve://./moduleException4/copysets"}, + {"-recycleUri=local://./moduleException4/recycler"}, + {"-chunkFilePoolDir=./moduleException4/chunkfilepool/"}, + {"-chunkFilePoolAllocatedPercent=0"}, + {"-chunkFilePoolMetaPath=./moduleException4/chunkfilepool.meta"}, + {"-conf=./conf/chunkserver.conf.example"}, + {"-raft_sync_segments=true"}, + {"--log_dir=./runlog/ChunkserverException"}, + {"-chunkServerIp=127.0.0.1"}, + {"-chunkServerPort=22125"}, + {"-enableChunkfilepool=false"}, + {"-mdsListenAddr=127.0.0.1:22122,127.0.0.1:22123,127.0.0.1:22124"}, + {"-enableWalfilepool=false"}, + {"-walFilePoolDir=./moduleException4/walfilepool/"}, + {"-walFilePoolMetaPath=./moduleException4/walfilepool.meta"}}; const std::vector chunkserverConf5{ - { "-chunkServerStoreUri=local://./moduleException5/" }, - { "-chunkServerMetaUri=local://./moduleException5/chunkserver.dat" }, - { "-copySetUri=local://./moduleException5/copysets" }, - { "-raftSnapshotUri=curve://./moduleException5/copysets" }, - { "-raftLogUri=curve://./moduleException5/copysets" }, - { "-recycleUri=local://./moduleException5/recycler" }, - { "-chunkFilePoolDir=./moduleException5/chunkfilepool/" }, - { "-chunkFilePoolMetaPath=./moduleException5/chunkfilepool.meta" }, - { "-conf=./conf/chunkserver.conf.example" }, - { "-raft_sync_segments=true" }, - { "--log_dir=./runlog/ChunkserverException" }, - { "-chunkServerIp=127.0.0.1" }, - { "-chunkServerPort=22126" }, - { "-enableChunkfilepool=false" }, - { "-mdsListenAddr=127.0.0.1:22122,127.0.0.1:22123,127.0.0.1:22124" }, - { "-enableWalfilepool=false" }, - { "-walFilePoolDir=./moduleException5/walfilepool/" }, - { "-walFilePoolMetaPath=./moduleException5/walfilepool.meta" } -}; + {"-chunkServerStoreUri=local://./moduleException5/"}, + {"-chunkServerMetaUri=local://./moduleException5/chunkserver.dat"}, + {"-copySetUri=local://./moduleException5/copysets"}, + {"-raftSnapshotUri=curve://./moduleException5/copysets"}, + {"-raftLogUri=curve://./moduleException5/copysets"}, + {"-recycleUri=local://./moduleException5/recycler"}, + {"-chunkFilePoolDir=./moduleException5/chunkfilepool/"}, + {"-chunkFilePoolAllocatedPercent=0"}, + {"-chunkFilePoolMetaPath=./moduleException5/chunkfilepool.meta"}, + {"-conf=./conf/chunkserver.conf.example"}, + {"-raft_sync_segments=true"}, + {"--log_dir=./runlog/ChunkserverException"}, + {"-chunkServerIp=127.0.0.1"}, + {"-chunkServerPort=22126"}, + {"-enableChunkfilepool=false"}, + {"-mdsListenAddr=127.0.0.1:22122,127.0.0.1:22123,127.0.0.1:22124"}, + {"-enableWalfilepool=false"}, + {"-walFilePoolDir=./moduleException5/walfilepool/"}, + {"-walFilePoolMetaPath=./moduleException5/walfilepool.meta"}}; const std::vector chunkserverConf6{ - { "-chunkServerStoreUri=local://./moduleException6/" }, - { "-chunkServerMetaUri=local://./moduleException6/chunkserver.dat" }, - { "-copySetUri=local://./moduleException6/copysets" }, - { "-raftSnapshotUri=curve://./moduleException6/copysets" }, - { "-raftLogUri=curve://./moduleException6/copysets" }, - { "-recycleUri=local://./moduleException6/recycler" }, - { "-chunkFilePoolDir=./moduleException6/chunkfilepool/" }, - { "-chunkFilePoolMetaPath=./moduleException6/chunkfilepool.meta" }, - { "-conf=./conf/chunkserver.conf.example" }, - { "-raft_sync_segments=true" }, - { "--log_dir=./runlog/ChunkserverException" }, - { "-chunkServerIp=127.0.0.1" }, - { "-chunkServerPort=22127" }, - { "-enableChunkfilepool=false" }, - { "-mdsListenAddr=127.0.0.1:22122,127.0.0.1:22123,127.0.0.1:22124" }, - { "-enableWalfilepool=false" }, - { "-walFilePoolDir=./moduleException6/walfilepool/" }, - { "-walFilePoolMetaPath=./moduleException6/walfilepool.meta" } -}; + {"-chunkServerStoreUri=local://./moduleException6/"}, + {"-chunkServerMetaUri=local://./moduleException6/chunkserver.dat"}, + {"-copySetUri=local://./moduleException6/copysets"}, + {"-raftSnapshotUri=curve://./moduleException6/copysets"}, + {"-raftLogUri=curve://./moduleException6/copysets"}, + {"-recycleUri=local://./moduleException6/recycler"}, + {"-chunkFilePoolDir=./moduleException6/chunkfilepool/"}, + {"-chunkFilePoolAllocatedPercent=0"}, + {"-chunkFilePoolMetaPath=./moduleException6/chunkfilepool.meta"}, + {"-conf=./conf/chunkserver.conf.example"}, + {"-raft_sync_segments=true"}, + {"--log_dir=./runlog/ChunkserverException"}, + {"-chunkServerIp=127.0.0.1"}, + {"-chunkServerPort=22127"}, + {"-enableChunkfilepool=false"}, + {"-mdsListenAddr=127.0.0.1:22122,127.0.0.1:22123,127.0.0.1:22124"}, + {"-enableWalfilepool=false"}, + {"-walFilePoolDir=./moduleException6/walfilepool/"}, + {"-walFilePoolMetaPath=./moduleException6/walfilepool.meta"}}; std::string mdsaddr = // NOLINT "127.0.0.1:22122,127.0.0.1:22123,127.0.0.1:22124"; diff --git a/test/integration/client/mds_exception_test.cpp b/test/integration/client/mds_exception_test.cpp index 4cf9f8ede3..ad0d82b093 100644 --- a/test/integration/client/mds_exception_test.cpp +++ b/test/integration/client/mds_exception_test.cpp @@ -59,70 +59,70 @@ const std::vector mdsConf{ }; const std::vector chunkserverConf1{ - { "-chunkServerStoreUri=local://./moduleException1/" }, - { "-chunkServerMetaUri=local://./moduleException1/chunkserver.dat" }, - { "-copySetUri=local://./moduleException1/copysets" }, - { "-raftSnapshotUri=curve://./moduleException1/copysets" }, - { "-raftLogUri=curve://./moduleException1/copysets" }, - { "-recycleUri=local://./moduleException1/recycler" }, - { "-chunkFilePoolDir=./moduleException1/chunkfilepool/" }, - { "-chunkFilePoolMetaPath=./moduleException1/chunkfilepool.meta" }, - { "-conf=./conf/chunkserver.conf.example" }, - { "-raft_sync_segments=true" }, - { "--log_dir=./runlog/MDSExceptionTest" }, - { "--graceful_quit_on_sigterm" }, - { "-chunkServerIp=127.0.0.1" }, - { "-chunkServerPort=22225" }, - { "-enableChunkfilepool=false" }, - { "-mdsListenAddr=127.0.0.1:22222,127.0.0.1:22223,127.0.0.1:22224" }, - { "-enableWalfilepool=false" }, - { "-walFilePoolDir=./moduleException1/walfilepool/" }, - { "-walFilePoolMetaPath=./moduleException1/walfilepool.meta" } -}; + {"-chunkServerStoreUri=local://./moduleException1/"}, + {"-chunkServerMetaUri=local://./moduleException1/chunkserver.dat"}, + {"-copySetUri=local://./moduleException1/copysets"}, + {"-raftSnapshotUri=curve://./moduleException1/copysets"}, + {"-raftLogUri=curve://./moduleException1/copysets"}, + {"-recycleUri=local://./moduleException1/recycler"}, + {"-chunkFilePoolDir=./moduleException1/chunkfilepool/"}, + {"-chunkFilePoolAllocatedPercent=0"}, + {"-chunkFilePoolMetaPath=./moduleException1/chunkfilepool.meta"}, + {"-conf=./conf/chunkserver.conf.example"}, + {"-raft_sync_segments=true"}, + {"--log_dir=./runlog/MDSExceptionTest"}, + {"--graceful_quit_on_sigterm"}, + {"-chunkServerIp=127.0.0.1"}, + {"-chunkServerPort=22225"}, + {"-enableChunkfilepool=false"}, + {"-mdsListenAddr=127.0.0.1:22222,127.0.0.1:22223,127.0.0.1:22224"}, + {"-enableWalfilepool=false"}, + {"-walFilePoolDir=./moduleException1/walfilepool/"}, + {"-walFilePoolMetaPath=./moduleException1/walfilepool.meta"}}; const std::vector chunkserverConf2{ - { "-chunkServerStoreUri=local://./moduleException2/" }, - { "-chunkServerMetaUri=local://./moduleException2/chunkserver.dat" }, - { "-copySetUri=local://./moduleException2/copysets" }, - { "-raftSnapshotUri=curve://./moduleException2/copysets" }, - { "-raftLogUri=curve://./moduleException2/copysets" }, - { "-recycleUri=local://./moduleException2/recycler" }, - { "-chunkFilePoolDir=./moduleException2/chunkfilepool/" }, - { "-chunkFilePoolMetaPath=./moduleException2/chunkfilepool.meta" }, - { "-conf=./conf/chunkserver.conf.example" }, - { "-raft_sync_segments=true" }, - { "--log_dir=./runlog/MDSExceptionTest" }, - { "--graceful_quit_on_sigterm" }, - { "-chunkServerIp=127.0.0.1" }, - { "-chunkServerPort=22226" }, - { "-enableChunkfilepool=false" }, - { "-mdsListenAddr=127.0.0.1:22222,127.0.0.1:22223,127.0.0.1:22224" }, - { "-enableWalfilepool=false" }, - { "-walFilePoolDir=./moduleException2/walfilepool/" }, - { "-walFilePoolMetaPath=./moduleException2/walfilepool.meta" } -}; + {"-chunkServerStoreUri=local://./moduleException2/"}, + {"-chunkServerMetaUri=local://./moduleException2/chunkserver.dat"}, + {"-copySetUri=local://./moduleException2/copysets"}, + {"-raftSnapshotUri=curve://./moduleException2/copysets"}, + {"-raftLogUri=curve://./moduleException2/copysets"}, + {"-recycleUri=local://./moduleException2/recycler"}, + {"-chunkFilePoolDir=./moduleException2/chunkfilepool/"}, + {"-chunkFilePoolAllocatedPercent=0"}, + {"-chunkFilePoolMetaPath=./moduleException2/chunkfilepool.meta"}, + {"-conf=./conf/chunkserver.conf.example"}, + {"-raft_sync_segments=true"}, + {"--log_dir=./runlog/MDSExceptionTest"}, + {"--graceful_quit_on_sigterm"}, + {"-chunkServerIp=127.0.0.1"}, + {"-chunkServerPort=22226"}, + {"-enableChunkfilepool=false"}, + {"-mdsListenAddr=127.0.0.1:22222,127.0.0.1:22223,127.0.0.1:22224"}, + {"-enableWalfilepool=false"}, + {"-walFilePoolDir=./moduleException2/walfilepool/"}, + {"-walFilePoolMetaPath=./moduleException2/walfilepool.meta"}}; const std::vector chunkserverConf3{ - { "-chunkServerStoreUri=local://./moduleException3/" }, - { "-chunkServerMetaUri=local://./moduleException3/chunkserver.dat" }, - { "-copySetUri=local://./moduleException3/copysets" }, - { "-raftSnapshotUri=curve://./moduleException3/copysets" }, - { "-raftLogUri=curve://./moduleException3/copysets" }, - { "-recycleUri=local://./moduleException3/recycler" }, - { "-chunkFilePoolDir=./moduleException3/chunkfilepool/" }, - { "-chunkFilePoolMetaPath=./moduleException3/chunkfilepool.meta" }, - { "-conf=./conf/chunkserver.conf.example" }, - { "-raft_sync_segments=true" }, - { "--log_dir=./runlog/MDSExceptionTest" }, - { "--graceful_quit_on_sigterm" }, - { "-chunkServerIp=127.0.0.1" }, - { "-chunkServerPort=22227" }, - { "-enableChunkfilepool=false" }, - { "-mdsListenAddr=127.0.0.1:22222,127.0.0.1:22223,127.0.0.1:22224" }, - { "-enableWalfilepool=false" }, - { "-walFilePoolDir=./moduleException3/walfilepool/" }, - { "-walFilePoolMetaPath=./moduleException3/walfilepool.meta" } -}; + {"-chunkServerStoreUri=local://./moduleException3/"}, + {"-chunkServerMetaUri=local://./moduleException3/chunkserver.dat"}, + {"-copySetUri=local://./moduleException3/copysets"}, + {"-raftSnapshotUri=curve://./moduleException3/copysets"}, + {"-raftLogUri=curve://./moduleException3/copysets"}, + {"-recycleUri=local://./moduleException3/recycler"}, + {"-chunkFilePoolDir=./moduleException3/chunkfilepool/"}, + {"-chunkFilePoolAllocatedPercent=0"}, + {"-chunkFilePoolMetaPath=./moduleException3/chunkfilepool.meta"}, + {"-conf=./conf/chunkserver.conf.example"}, + {"-raft_sync_segments=true"}, + {"--log_dir=./runlog/MDSExceptionTest"}, + {"--graceful_quit_on_sigterm"}, + {"-chunkServerIp=127.0.0.1"}, + {"-chunkServerPort=22227"}, + {"-enableChunkfilepool=false"}, + {"-mdsListenAddr=127.0.0.1:22222,127.0.0.1:22223,127.0.0.1:22224"}, + {"-enableWalfilepool=false"}, + {"-walFilePoolDir=./moduleException3/walfilepool/"}, + {"-walFilePoolMetaPath=./moduleException3/walfilepool.meta"}}; std::string mdsaddr = // NOLINT "127.0.0.1:22222,127.0.0.1:22223,127.0.0.1:22224"; // NOLINT diff --git a/test/integration/client/unstable_chunkserver_exception_test.cpp b/test/integration/client/unstable_chunkserver_exception_test.cpp index cf1753ff2c..ea5c7e4c37 100644 --- a/test/integration/client/unstable_chunkserver_exception_test.cpp +++ b/test/integration/client/unstable_chunkserver_exception_test.cpp @@ -86,23 +86,23 @@ const std::vector mdsConf{ }; const std::vector chunkserverConfTemplate{ - { "-raft_sync_segments=true" }, + {"-raft_sync_segments=true"}, std::string("-conf=") + kCSConfPath, - { "-chunkServerPort=%d" }, - { "-chunkServerStoreUri=local://./ttt/%d/" }, - { "-chunkServerMetaUri=local://./ttt/%d/chunkserver.dat" }, - { "-copySetUri=local://./ttt/%d/copysets" }, - { "-raftSnapshotUri=curve://./ttt/%d/copysets" }, - { "-raftLogUri=curve://./ttt/%d/copysets" }, - { "-recycleUri=local://./ttt/%d/recycler" }, - { "-chunkFilePoolDir=./ttt/%d/chunkfilepool/" }, - { "-chunkFilePoolMetaPath=./ttt/%d/chunkfilepool.meta" }, - { "-walFilePoolDir=./ttt/%d/walfilepool/" }, - { "-walFilePoolMetaPath=./ttt/%d/walfilepool.meta" }, - { "-mdsListenAddr=127.0.0.1:30010,127.0.0.1:30011,127.0.0.1:30012" }, - { "-log_dir=./runlog/cs_%d" }, - { "--stderrthreshold=3" } -}; + {"-chunkServerPort=%d"}, + {"-chunkServerStoreUri=local://./ttt/%d/"}, + {"-chunkServerMetaUri=local://./ttt/%d/chunkserver.dat"}, + {"-copySetUri=local://./ttt/%d/copysets"}, + {"-raftSnapshotUri=curve://./ttt/%d/copysets"}, + {"-raftLogUri=curve://./ttt/%d/copysets"}, + {"-recycleUri=local://./ttt/%d/recycler"}, + {"-chunkFilePoolDir=./ttt/%d/chunkfilepool/"}, + {"-chunkFilePoolAllocatedPercent=0"}, + {"-chunkFilePoolMetaPath=./ttt/%d/chunkfilepool.meta"}, + {"-walFilePoolDir=./ttt/%d/walfilepool/"}, + {"-walFilePoolMetaPath=./ttt/%d/walfilepool.meta"}, + {"-mdsListenAddr=127.0.0.1:30010,127.0.0.1:30011,127.0.0.1:30012"}, + {"-log_dir=./runlog/cs_%d"}, + {"--stderrthreshold=3"}}; const std::vector chunkserverPorts{ 31000, 31001, 31010, 31011, 31020, 31021, diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp index 8eff45065c..8f312b9a88 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_common_test.cpp @@ -110,6 +110,7 @@ const std::vector chunkserverConfigOptions{ std::string("mds.listen.addr=") + kMdsIpPort, std::string("curve.config_path=") + kCsClientConfigPath, std::string("s3.config_path=") + kS3ConfigPath, + "chunkfilepool.allocated_by_percent=false", "walfilepool.use_chunk_file_pool=false", "walfilepool.enable_get_segment_from_pool=false", "global.block_size=4096", diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_concurrent_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_concurrent_test.cpp index 55484d7ec3..a725cbe12f 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_concurrent_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_concurrent_test.cpp @@ -107,6 +107,7 @@ const std::vector chunkserverConfigOptions{ std::string("mds.listen.addr=") + kMdsIpPort, std::string("curve.config_path=") + kCsClientConfigPath, std::string("s3.config_path=") + kS3ConfigPath, + "chunkfilepool.allocated_by_percent=false", "walfilepool.use_chunk_file_pool=false", "walfilepool.enable_get_segment_from_pool=false", "global.block_size=4096", diff --git a/test/integration/snapshotcloneserver/snapshotcloneserver_recover_test.cpp b/test/integration/snapshotcloneserver/snapshotcloneserver_recover_test.cpp index 18a113ef0f..d4ccb66c65 100644 --- a/test/integration/snapshotcloneserver/snapshotcloneserver_recover_test.cpp +++ b/test/integration/snapshotcloneserver/snapshotcloneserver_recover_test.cpp @@ -133,6 +133,7 @@ const std::vector chunkserverConfigOptions{ std::string("s3.config_path=") + kS3ConfigPath, std::string("curve.root_username") + mdsRootUser_, std::string("curve.root_password") + mdsRootPassword_, + "chunkfilepool.allocated_by_percent=false", "walfilepool.use_chunk_file_pool=false", "walfilepool.enable_get_segment_from_pool=false", "global.block_size=4096", diff --git a/tools-v2/internal/error/error.go b/tools-v2/internal/error/error.go index 72f8de384d..a4737ba33d 100644 --- a/tools-v2/internal/error/error.go +++ b/tools-v2/internal/error/error.go @@ -484,6 +484,9 @@ var ( ErrListWarmup = func() *CmdError { return NewInternalCmdError(74, "list warmup progress fail, err: %s") } + ErrBsGetFormatStatus = func() *CmdError { + return NewInternalCmdError(75, "get format status fail, err: %s") + } // http error ErrHttpUnreadableResult = func() *CmdError { diff --git a/tools-v2/internal/utils/row.go b/tools-v2/internal/utils/row.go index 4154203619..33590c75f8 100644 --- a/tools-v2/internal/utils/row.go +++ b/tools-v2/internal/utils/row.go @@ -129,7 +129,8 @@ const ( ROW_HEALTHY_COUNT = "healthyCount" ROW_UNHEALTHY_COUNT = "unhealthyCount" ROW_HEALTHY_RATIO = "ratio" - ROW_UNHEALTHY_RATIO = "unhealthy-Ratio" + ROW_UNHEALTHY_RATIO = "unhealthyRatio" + ROW_FORMAT_PERCENT = "formatPercent" ROW_RW_STATUS = "rwStatus" ROW_DISK_STATE = "diskState" diff --git a/tools-v2/pkg/cli/command/curvebs/list/formatstatus/formatstatus.go b/tools-v2/pkg/cli/command/curvebs/list/formatstatus/formatstatus.go new file mode 100644 index 0000000000..30b2dd2470 --- /dev/null +++ b/tools-v2/pkg/cli/command/curvebs/list/formatstatus/formatstatus.go @@ -0,0 +1,137 @@ +package formatstatus + +import ( + "context" + "fmt" + + cmderror "github.com/opencurve/curve/tools-v2/internal/error" + cobrautil "github.com/opencurve/curve/tools-v2/internal/utils" + basecmd "github.com/opencurve/curve/tools-v2/pkg/cli/command" + "github.com/opencurve/curve/tools-v2/pkg/config" + "github.com/opencurve/curve/tools-v2/pkg/output" + "github.com/opencurve/curve/tools-v2/proto/proto/topology" + "github.com/spf13/cobra" + "google.golang.org/grpc" +) + +const ( + formatExample = `$ curve bs list format-status` +) + +type GetFormatStatusRpc struct { + Info *basecmd.Rpc + Request *topology.ListChunkFormatStatusRequest + mdsClient topology.TopologyServiceClient +} + +var _ basecmd.RpcFunc = (*GetFormatStatusRpc)(nil) // check interface + +func (gRpc *GetFormatStatusRpc) NewRpcClient(cc grpc.ClientConnInterface) { + gRpc.mdsClient = topology.NewTopologyServiceClient(cc) +} + +func (gRpc *GetFormatStatusRpc) Stub_Func(ctx context.Context) (interface{}, error) { + return gRpc.mdsClient.ListChunkFormatStatus(ctx, gRpc.Request) +} + +type FormatStatusCommand struct { + Rpc *GetFormatStatusRpc + FormatStatusInfoList []*topology.ChunkFormatStatus + basecmd.FinalCurveCmd +} + +var _ basecmd.FinalCurveCmdFunc = (*FormatStatusCommand)(nil) // check interface + +func NewListFormatStatusCommand() *FormatStatusCommand { + formatCmd := &FormatStatusCommand{ + FinalCurveCmd: basecmd.FinalCurveCmd{ + Use: "format-status", + Short: "list all format status in cluster", + Example: formatExample, + }, + } + + basecmd.NewFinalCurveCli(&formatCmd.FinalCurveCmd, formatCmd) + return formatCmd +} + +func NewFormatStatusCommand() *cobra.Command { + return NewListFormatStatusCommand().Cmd +} + +func (fCmd *FormatStatusCommand) AddFlags() { + config.AddRpcTimeoutFlag(fCmd.Cmd) + config.AddRpcRetryTimesFlag(fCmd.Cmd) + config.AddBsMdsFlagOption(fCmd.Cmd) + config.AddBsFilterOptionFlag(fCmd.Cmd) +} + +func (fCmd *FormatStatusCommand) Init(cmd *cobra.Command, args []string) error { + mdsAddrs, err := config.GetBsMdsAddrSlice(fCmd.Cmd) + if err.TypeCode() != cmderror.CODE_SUCCESS { + return err.ToError() + } + timeout := config.GetFlagDuration(fCmd.Cmd, config.RPCTIMEOUT) + retrytimes := config.GetFlagInt32(fCmd.Cmd, config.RPCRETRYTIMES) + + fCmd.Rpc = &GetFormatStatusRpc{ + Info: basecmd.NewRpc(mdsAddrs, timeout, retrytimes, "ListChunkFormatStatus"), + Request: &topology.ListChunkFormatStatusRequest{}, + mdsClient: nil, + } + fCmd.SetHeader([]string{ + cobrautil.ROW_CHUNKSERVER, cobrautil.ROW_IP, cobrautil.ROW_PORT, cobrautil.ROW_FORMAT_PERCENT, + }) + return nil +} + +func (fCmd *FormatStatusCommand) Print(cmd *cobra.Command, args []string) error { + return output.FinalCmdOutput(&fCmd.FinalCurveCmd, fCmd) +} + +func (fCmd *FormatStatusCommand) RunCommand(cmd *cobra.Command, args []string) error { + result, err := basecmd.GetRpcResponse(fCmd.Rpc.Info, fCmd.Rpc) + if err.TypeCode() != cmderror.CODE_SUCCESS { + fCmd.Error = err + fCmd.Result = nil + return err.ToError() + } + res := result.(*topology.ListChunkFormatStatusResponse) + fCmd.FormatStatusInfoList = res.GetChunkFormatStatus() + + rows := make([]map[string]string, 0) + for _, info := range fCmd.FormatStatusInfoList { + fmt.Println("sssss: ", info.GetIp()) + row := make(map[string]string) + row[cobrautil.ROW_CHUNKSERVER] = fmt.Sprint(info.GetChunkServerID()) + row[cobrautil.ROW_IP] = info.GetIp() + row[cobrautil.ROW_PORT] = fmt.Sprint(uint64(info.GetPort())) + row[cobrautil.ROW_FORMAT_PERCENT] = fmt.Sprint(info.GetFormatPercent()) + rows = append(rows, row) + } + list := cobrautil.ListMap2ListSortByKeys(rows, fCmd.Header, []string{}) + fCmd.TableNew.AppendBulk(list) + fCmd.Result, fCmd.Error = rows, cmderror.Success() + return nil +} + +func (fCmd *FormatStatusCommand) ResultPlainOutput() error { + return output.FinalCmdOutputPlain(&fCmd.FinalCurveCmd) +} + +func GetFormatStatusInfoList(caller *cobra.Command) ([]*topology.ChunkFormatStatus, *cmderror.CmdError) { + getCmd := NewListFormatStatusCommand() + config.AlignFlagsValue(caller, getCmd.Cmd, []string{ + config.CURVEBS_MDSADDR, config.RPCRETRYTIMES, config.RPCTIMEOUT, config.CURVEBS_FIlTER, + }) + getCmd.Cmd.SilenceErrors = true + getCmd.Cmd.SilenceUsage = true + getCmd.Cmd.SetArgs([]string{fmt.Sprintf("--%s", config.FORMAT), config.FORMAT_NOOUT}) + err := getCmd.Cmd.Execute() + if err != nil { + retErr := cmderror.ErrBsGetFormatStatus() + retErr.Format(err.Error()) + return getCmd.FormatStatusInfoList, retErr + } + return getCmd.FormatStatusInfoList, cmderror.Success() +} diff --git a/tools-v2/pkg/cli/command/curvebs/list/list.go b/tools-v2/pkg/cli/command/curvebs/list/list.go index 58e9b433fd..0b2e4266e3 100644 --- a/tools-v2/pkg/cli/command/curvebs/list/list.go +++ b/tools-v2/pkg/cli/command/curvebs/list/list.go @@ -27,6 +27,7 @@ import ( "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/chunkserver" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/client" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/dir" + "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/formatstatus" logicalpool "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/logicalPool" may_broken_vol "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/may-broken-vol" "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/scanstatus" @@ -51,6 +52,7 @@ func (listCmd *ListCommand) AddSubCommands() { chunkserver.NewChunkServerCommand(), scanstatus.NewScanStatusCommand(), may_broken_vol.NewMayBrokenVolCommand(), + formatstatus.NewFormatStatusCommand(), ) } From 08934eed5dad1a00e0cc7cfb71b6f8b7eabc8feb Mon Sep 17 00:00:00 2001 From: Wine93 Date: Thu, 6 Jul 2023 11:46:36 +0800 Subject: [PATCH 15/27] curvefs/client: fix modify access logging switch on the fly not works. Signed-off-by: Wine93 --- curvefs/src/client/common/config.cpp | 2 ++ curvefs/src/client/filesystem/access_log.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/curvefs/src/client/common/config.cpp b/curvefs/src/client/common/config.cpp index 7f42202c2b..5437fde2b5 100644 --- a/curvefs/src/client/common/config.cpp +++ b/curvefs/src/client/common/config.cpp @@ -48,11 +48,13 @@ DEFINE_validator(fs_disableXattr, [](const char*, bool value) { return true; }); namespace curvefs { namespace client { namespace common { +static bool pass_bool(const char *, bool) { return true; } DEFINE_bool(enableCto, true, "acheieve cto consistency"); DEFINE_bool(useFakeS3, false, "Use fake s3 to inject more metadata for testing metaserver"); DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing"); DEFINE_bool(access_logging, true, "enable access log"); +DEFINE_validator(access_logging, &pass_bool); /** * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true diff --git a/curvefs/src/client/filesystem/access_log.h b/curvefs/src/client/filesystem/access_log.h index 1e2067c9b1..608b564f65 100644 --- a/curvefs/src/client/filesystem/access_log.h +++ b/curvefs/src/client/filesystem/access_log.h @@ -51,7 +51,7 @@ using MessageHandler = std::function; static std::shared_ptr Logger; bool InitAccessLog(const std::string& prefix) { - std::string filename = StrFormat("%s/access.%d.log", prefix, getpid()); + std::string filename = StrFormat("%s/access_%d.log", prefix, getpid()); Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0); spdlog::flush_every(std::chrono::seconds(1)); return true; From 251f8a1d657c7f47669172c7211bee0ca33eb519 Mon Sep 17 00:00:00 2001 From: Wine93 Date: Sun, 30 Jul 2023 20:26:40 +0800 Subject: [PATCH 16/27] curvefs/client: fix memory usage is so high. Signed-off-by: Wine93 --- curvefs/src/client/filesystem/dir_cache.cpp | 43 +++++++----- curvefs/src/client/filesystem/dir_cache.h | 8 ++- curvefs/src/client/filesystem/metric.h | 72 ++++++++++++++++++++ curvefs/src/client/filesystem/openfile.cpp | 7 +- curvefs/src/client/filesystem/openfile.h | 2 + curvefs/src/client/filesystem/rpc_client.cpp | 4 ++ 6 files changed, 116 insertions(+), 20 deletions(-) create mode 100644 curvefs/src/client/filesystem/metric.h diff --git a/curvefs/src/client/filesystem/dir_cache.cpp b/curvefs/src/client/filesystem/dir_cache.cpp index 3beed645fe..64133d33ff 100644 --- a/curvefs/src/client/filesystem/dir_cache.cpp +++ b/curvefs/src/client/filesystem/dir_cache.cpp @@ -33,7 +33,7 @@ DirEntryList::DirEntryList() : rwlock_(), mtime_(), entries_(), - attrs_() {} + index_() {} size_t DirEntryList::Size() { ReadLockGuard lk(rwlock_); @@ -43,39 +43,41 @@ size_t DirEntryList::Size() { void DirEntryList::Add(const DirEntry& dirEntry) { WriteLockGuard lk(rwlock_); entries_.push_back(std::move(dirEntry)); - attrs_[dirEntry.ino] = &entries_.back(); + index_[dirEntry.ino] = entries_.size() - 1; } bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) { ReadLockGuard lk(rwlock_); - auto iter = attrs_.find(ino); - if (iter == attrs_.end()) { + auto iter = index_.find(ino); + if (iter == index_.end()) { return false; } - *dirEntry = *iter->second; + + *dirEntry = entries_[iter->second]; return true; } bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) { WriteLockGuard lk(rwlock_); - auto iter = attrs_.find(ino); - if (iter == attrs_.end()) { + auto iter = index_.find(ino); + if (iter == index_.end()) { return false; } - DirEntry* dirEntry = iter->second; + DirEntry* dirEntry = &entries_[iter->second]; dirEntry->attr = std::move(attr); return true; } bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) { WriteLockGuard lk(rwlock_); - auto iter = attrs_.find(ino); - if (iter == attrs_.end()) { + auto iter = index_.find(ino); + if (iter == index_.end()) { return false; } - InodeAttr* attr = &iter->second->attr; + DirEntry* dirEntry = &entries_[iter->second]; + InodeAttr* attr = &(dirEntry->attr); attr->set_length(open.length()); attr->set_mtime(open.mtime()); attr->set_mtime_ns(open.mtime_ns()); @@ -96,7 +98,7 @@ void DirEntryList::Iterate(IterateHandler handler) { void DirEntryList::Clear() { WriteLockGuard lk(rwlock_); entries_.clear(); - attrs_.clear(); + index_.clear(); } void DirEntryList::SetMtime(TimeSpec mtime) { @@ -118,6 +120,7 @@ DirCache::DirCache(DirCacheOption option) mq_->Subscribe([&](const std::shared_ptr& entries){ entries->Clear(); }); + metric_ = std::make_shared(); LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize; } @@ -135,14 +138,16 @@ void DirCache::Stop() { void DirCache::Delete(Ino parent, std::shared_ptr entries, bool evit) { - nentries_ -= entries->Size(); + size_t ndelete = entries->Size(); + nentries_ -= ndelete; + metric_->AddEntries(-static_cast(ndelete)); mq_->Publish(entries); // clear entries in background lru_->Remove(parent); VLOG(1) << "Delete directory cache (evit=" << evit << "): " << "parent = " << parent << ", mtime = " << entries->GetMtime() - << ", size = " << entries->Size() + << ", delete size = " << ndelete << ", nentries = " << nentries_; } @@ -160,13 +165,19 @@ void DirCache::Evit(size_t size) { void DirCache::Put(Ino parent, std::shared_ptr entries) { WriteLockGuard lk(rwlock_); + if (entries->Size() == 0) { // TODO(Wine93): cache it! + return; + } + Evit(entries->Size()); // it guarantee put entries success lru_->Put(parent, entries); - nentries_ += entries->Size(); + int64_t ninsert = entries->Size(); + nentries_ += ninsert; + metric_->AddEntries(static_cast(ninsert)); VLOG(1) << "Insert directory cache: parent = " << parent << ", mtime = " << entries->GetMtime() - << ", size = " << entries->Size() + << ", insert size = " << ninsert << ", nentries = " << nentries_; } diff --git a/curvefs/src/client/filesystem/dir_cache.h b/curvefs/src/client/filesystem/dir_cache.h index 6aa57fe5d5..0fd0e6a7dd 100644 --- a/curvefs/src/client/filesystem/dir_cache.h +++ b/curvefs/src/client/filesystem/dir_cache.h @@ -25,12 +25,15 @@ #include #include +#include #include +#include "absl/container/btree_map.h" #include "src/common/lru_cache.h" #include "src/common/concurrent/concurrent.h" #include "curvefs/src/client/common/config.h" #include "curvefs/src/client/filesystem/meta.h" +#include "curvefs/src/client/filesystem/metric.h" #include "curvefs/src/client/filesystem/message_queue.h" namespace curvefs { @@ -71,8 +74,8 @@ class DirEntryList { private: RWLock rwlock_; TimeSpec mtime_; - std::list entries_; - std::map attrs_; + std::vector entries_; + absl::btree_map index_; }; class DirCache { @@ -105,6 +108,7 @@ class DirCache { DirCacheOption option_; std::shared_ptr lru_; std::shared_ptr mq_; + std::shared_ptr metric_; }; } // namespace filesystem diff --git a/curvefs/src/client/filesystem/metric.h b/curvefs/src/client/filesystem/metric.h new file mode 100644 index 0000000000..3627349eda --- /dev/null +++ b/curvefs/src/client/filesystem/metric.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-13 + * Author: Jingli Chen (Wine93) + */ + +#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_METRIC_H_ +#define CURVEFS_SRC_CLIENT_FILESYSTEM_METRIC_H_ + +#include + +#include + +namespace curvefs { +namespace client { +namespace filesystem { + +class DirCacheMetric { + public: + DirCacheMetric() = default; + + void AddEntries(int64_t n) { + metric_.nentries << n; + } + + private: + struct Metric { + Metric() : nentries("filesystem_dircache", "nentries") {} + bvar::Adder nentries; + }; + + Metric metric_; +}; + +class OpenfilesMetric { + public: + OpenfilesMetric() = default; + + void AddOpenfiles(int64_t n) { + metric_.nfiles << n; + } + + private: + struct Metric { + Metric() : nfiles("filesystem_openfiles", "nfiles") {} + bvar::Adder nfiles; + }; + + Metric metric_; +}; + +} // namespace filesystem +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_FILESYSTEM_METRIC_H_ diff --git a/curvefs/src/client/filesystem/openfile.cpp b/curvefs/src/client/filesystem/openfile.cpp index 49ba785834..944ac77785 100644 --- a/curvefs/src/client/filesystem/openfile.cpp +++ b/curvefs/src/client/filesystem/openfile.cpp @@ -31,8 +31,9 @@ OpenFiles::OpenFiles(OpenFilesOption option, std::shared_ptr deferSync) : rwlock_(), option_(option), - deferSync_(deferSync), - files_(std::make_shared(option.lruSize)) { + deferSync_(deferSync) { + files_ = std::make_shared(option.lruSize); + metric_ = std::make_shared(); LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize; } @@ -48,6 +49,7 @@ void OpenFiles::Delete(Ino ino, deferSync_->Push(file->inode); } files_->Remove(ino); + metric_->AddOpenfiles(-1); VLOG(1) << "Delete open file cache: ino = " << ino << ", refs = " << file->refs @@ -81,6 +83,7 @@ void OpenFiles::Open(Ino ino, std::shared_ptr inode) { file = std::make_shared(inode); file->refs++; files_->Put(ino, file); + metric_->AddOpenfiles(1); VLOG(1) << "Insert open file cache: ino = " << ino << ", refs = " << file->refs diff --git a/curvefs/src/client/filesystem/openfile.h b/curvefs/src/client/filesystem/openfile.h index 53ce0ae26e..7a88f8626f 100644 --- a/curvefs/src/client/filesystem/openfile.h +++ b/curvefs/src/client/filesystem/openfile.h @@ -30,6 +30,7 @@ #include "curvefs/src/client/filesystem/meta.h" #include "curvefs/src/client/filesystem/defer_sync.h" #include "curvefs/src/client/filesystem/dir_cache.h" +#include "curvefs/src/client/filesystem/metric.h" namespace curvefs { namespace client { @@ -77,6 +78,7 @@ class OpenFiles { OpenFilesOption option_; std::shared_ptr deferSync_; std::shared_ptr files_; + std::shared_ptr metric_; }; } // namespace filesystem diff --git a/curvefs/src/client/filesystem/rpc_client.cpp b/curvefs/src/client/filesystem/rpc_client.cpp index 4f5639d1f3..a7ec92764d 100644 --- a/curvefs/src/client/filesystem/rpc_client.cpp +++ b/curvefs/src/client/filesystem/rpc_client.cpp @@ -80,6 +80,10 @@ CURVEFS_ERROR RPCClient::ReadDir(Ino ino, LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc << ", ino = " << ino; return rc; + } else if (dentries.size() == 0) { + VLOG(3) << "rpc(readdir::ListDentry) success and directory is empty" + << ", ino = " << ino; + return rc; } std::set inos; From f3fa22ad7042a8aa54dc6c8f393129544aaba757 Mon Sep 17 00:00:00 2001 From: Wine93 Date: Mon, 16 Oct 2023 17:08:06 +0800 Subject: [PATCH 17/27] curvefs/client: let curve specified xattr works even if disable xattr. Signed-off-by: Wine93 --- curvefs/conf/client.conf | 6 +- curvefs/src/client/BUILD | 12 ++ curvefs/src/client/common/common.h | 6 - curvefs/src/client/common/config.cpp | 3 +- curvefs/src/client/common/config.h | 2 +- curvefs/src/client/curve_fuse_op.cpp | 24 +-- curvefs/src/client/filesystem/xattr.h | 76 +++++++++ curvefs/src/client/fuse_client.cpp | 87 ++++++---- curvefs/src/client/fuse_common.h | 1 + curvefs/src/client/fuse_s3_client.cpp | 4 +- curvefs/src/client/rpcclient/BUILD | 1 + curvefs/src/client/xattr_manager.cpp | 96 ++++++----- curvefs/src/common/define.h | 23 ++- curvefs/src/mds/BUILD | 1 + curvefs/src/metaserver/BUILD | 1 + curvefs/src/metaserver/inode_manager.cpp | 13 +- .../client/filesystem/filesystem_test.cpp | 28 +++- .../test/client/filesystem/helper/builder.h | 2 +- .../client/filesystem/rpc_client_test.cpp | 30 +++- curvefs/test/client/rpcclient/BUILD | 1 + .../rpcclient/metaserver_client_test.cpp | 8 +- curvefs/test/client/test_fuse_s3_client.cpp | 154 +++++++++--------- .../test/client/test_inode_cache_manager.cpp | 12 +- .../test/metaserver/inode_manager_test.cpp | 32 ++-- .../test/metaserver/inode_storage_test.cpp | 32 ++-- curvefs/test/metaserver/metastore_test.cpp | 80 ++++----- curvefs/test/metaserver/partition_test.cpp | 8 +- 27 files changed, 445 insertions(+), 298 deletions(-) create mode 100644 curvefs/src/client/filesystem/xattr.h diff --git a/curvefs/conf/client.conf b/curvefs/conf/client.conf index b7b00baceb..9b3d449ec9 100644 --- a/curvefs/conf/client.conf +++ b/curvefs/conf/client.conf @@ -128,15 +128,15 @@ fuseClient.throttle.burstReadIopsSecs=180 #### filesystem metadata # { -# fs.disableXattr: +# fs.disableXAttr: # if you want to get better metadata performance, -# you can mount fs with |fs.disableXattr| is true +# you can mount fs with |fs.disableXAttr| is true # # fs.lookupCache.negativeTimeoutSec: # entry which not found will be cached if |timeout| > 0 fs.cto=true fs.maxNameLength=255 -fs.disableXattr=false +fs.disableXAttr=true fs.accessLogging=true fs.kernelCache.attrTimeoutSec=3600 fs.kernelCache.dirAttrTimeoutSec=3600 diff --git a/curvefs/src/client/BUILD b/curvefs/src/client/BUILD index b296af86f8..677e32b0b6 100644 --- a/curvefs/src/client/BUILD +++ b/curvefs/src/client/BUILD @@ -32,6 +32,16 @@ cc_binary( ], ) +cc_library( + name = "filesystem_xattr", + hdrs = ["filesystem/xattr.h"], + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//external:bvar", + ], +) + cc_library( name = "fuse_client_lib", srcs = glob( @@ -64,6 +74,7 @@ cc_library( "//curvefs/src/common:curvefs_common", "//curvefs/src/client/lease:curvefs_lease", "//curvefs/src/client/kvclient:memcached_client_lib", + ":filesystem_xattr", "//external:brpc", "//external:gflags", "//external:glog", @@ -80,6 +91,7 @@ cc_library( "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/meta:type_traits", "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/container:btree", "@com_google_googletest//:gtest_prod", "@spdlog//:spdlog", ], diff --git a/curvefs/src/client/common/common.h b/curvefs/src/client/common/common.h index 0a1dd00d09..a76312b117 100644 --- a/curvefs/src/client/common/common.h +++ b/curvefs/src/client/common/common.h @@ -65,12 +65,6 @@ enum class MetaServerOpType { std::ostream &operator<<(std::ostream &os, MetaServerOpType optype); -const uint32_t MAX_XATTR_NAME_LENGTH = 255; -const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024; - -const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op"; -const char kCurveFsWarmupXAttrList[] = "curvefs.warmup.op.list"; - constexpr size_t kMinWarmupOpArgsNum = 1; constexpr size_t kWarmupAddArgsNum = 6; constexpr size_t kWarmupCancelArgsNum = 2; diff --git a/curvefs/src/client/common/config.cpp b/curvefs/src/client/common/config.cpp index 5437fde2b5..ad4b29b97c 100644 --- a/curvefs/src/client/common/config.cpp +++ b/curvefs/src/client/common/config.cpp @@ -317,8 +317,7 @@ void InitKVClientManagerOpt(Configuration *conf, void InitFileSystemOption(Configuration* c, FileSystemOption* option) { c->GetValueFatalIfFail("fs.cto", &option->cto); c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto); - c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr); - FLAGS_fs_disableXattr = option->disableXattr; + c->GetValueFatalIfFail("fs.disableXAttr", &option->disableXAttr); c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength); c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging); { // kernel cache option diff --git a/curvefs/src/client/common/config.h b/curvefs/src/client/common/config.h index 30c6417ea7..3d0c91dd70 100644 --- a/curvefs/src/client/common/config.h +++ b/curvefs/src/client/common/config.h @@ -219,7 +219,7 @@ struct DeferSyncOption { struct FileSystemOption { bool cto; - bool disableXattr; + bool disableXAttr; uint32_t maxNameLength; uint32_t blockSize = 0x10000u; KernelCacheOption kernelCacheOption; diff --git a/curvefs/src/client/curve_fuse_op.cpp b/curvefs/src/client/curve_fuse_op.cpp index 44e3f16a71..78c567f589 100644 --- a/curvefs/src/client/curve_fuse_op.cpp +++ b/curvefs/src/client/curve_fuse_op.cpp @@ -47,6 +47,7 @@ #include "curvefs/src/common/metric_utils.h" #include "src/common/configuration.h" #include "src/common/gflags_helper.h" +#include "curvefs/src/client/filesystem/xattr.h" using ::curve::common::Configuration; using ::curvefs::client::CURVEFS_ERROR; @@ -77,6 +78,8 @@ using ::curvefs::client::rpcclient::MDSBaseClient; using ::curvefs::client::rpcclient::MdsClientImpl; using ::curvefs::client::warmup::WarmupProgress; using ::curvefs::common::LatencyUpdater; +using ::curvefs::client::filesystem::IsWarmupXAttr; +using ::curvefs::client::filesystem::IsListWarmupXAttr; using ::curvefs::common::FLAGS_vlog_level; @@ -901,21 +904,6 @@ void FuseOpStatFs(fuse_req_t req, fuse_ino_t ino) { return fs->ReplyStatfs(req, &stbuf); } -const char* warmupXAttr = ::curvefs::client::common::kCurveFsWarmupXAttr; -const char* warmupListXAttr = - ::curvefs::client::common::kCurveFsWarmupXAttrList; - -bool IsWamupReq(const char* name) { - if (strlen(name) < strlen(warmupXAttr)) { - return false; - } - return strncmp(name, warmupXAttr, strlen(warmupXAttr)) == 0; -} - -bool IsWarmupListReq(const char* name) { - return IsWamupReq(name) && strcmp(name, warmupListXAttr) == 0; -} - void FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino, const char* name, @@ -930,7 +918,7 @@ void FuseOpSetXattr(fuse_req_t req, ino, name, size, flags, StrErr(rc)); }); - if (IsWamupReq(name)) { + if (IsWarmupXAttr(name)) { return TriggerWarmup(req, ino, name, value, size); } rc = client->FuseOpSetXattr(req, ino, name, value, size, flags); @@ -951,9 +939,9 @@ void FuseOpGetXattr(fuse_req_t req, ino, name, size, StrErr(rc), value.size()); }); - if (IsWarmupListReq(name)) { + if (IsListWarmupXAttr(name)) { return ListWarmup(req, size); - } else if (IsWamupReq(name)) { + } else if (IsWarmupXAttr(name)) { return QueryWarmup(req, ino, size); } diff --git a/curvefs/src/client/filesystem/xattr.h b/curvefs/src/client/filesystem/xattr.h new file mode 100644 index 0000000000..26bc3a6c2e --- /dev/null +++ b/curvefs/src/client/filesystem/xattr.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-19 + * Author: Jingli Chen (Wine93) + */ + +#include +#include +#include + +#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_XATTR_H_ +#define CURVEFS_SRC_CLIENT_FILESYSTEM_XATTR_H_ + +namespace curvefs { +namespace client { +namespace filesystem { + +const uint32_t MAX_XATTR_NAME_LENGTH = 255; +const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024; + +const char XATTR_DIR_FILES[] = "curve.dir.files"; +const char XATTR_DIR_SUBDIRS[] = "curve.dir.subdirs"; +const char XATTR_DIR_ENTRIES[] = "curve.dir.entries"; +const char XATTR_DIR_FBYTES[] = "curve.dir.fbytes"; +const char XATTR_DIR_RFILES[] = "curve.dir.rfiles"; +const char XATTR_DIR_RSUBDIRS[] = "curve.dir.rsubdirs"; +const char XATTR_DIR_RENTRIES[] = "curve.dir.rentries"; +const char XATTR_DIR_RFBYTES[] = "curve.dir.rfbytes"; +const char XATTR_DIR_PREFIX[] = "curve.dir"; +const char XATTR_WARMUP_OP[] = "curvefs.warmup.op"; +const char XATTR_WARMUP_OP_LIST[] = "curvefs.warmup.op.list"; + +inline bool IsSpecialXAttr(const std::string& key) { + static std::map xattrs { + { XATTR_DIR_FILES, true }, + { XATTR_DIR_SUBDIRS, true }, + { XATTR_DIR_ENTRIES, true }, + { XATTR_DIR_FBYTES, true }, + { XATTR_DIR_RFILES, true }, + { XATTR_DIR_RSUBDIRS, true }, + { XATTR_DIR_RENTRIES, true }, + { XATTR_DIR_RFBYTES, true }, + { XATTR_DIR_PREFIX, true }, + }; + return xattrs.find(key) != xattrs.end(); +} + +inline bool IsWarmupXAttr(const std::string& key) { + return key == XATTR_WARMUP_OP; +} + +inline bool IsListWarmupXAttr(const std::string& key) { + return key == XATTR_WARMUP_OP_LIST; +} + +} // namespace filesystem +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_FILESYSTEM_XATTR_H_ diff --git a/curvefs/src/client/fuse_client.cpp b/curvefs/src/client/fuse_client.cpp index 0fe8d192da..b944b37f15 100644 --- a/curvefs/src/client/fuse_client.cpp +++ b/curvefs/src/client/fuse_client.cpp @@ -36,6 +36,7 @@ #include "curvefs/proto/mds.pb.h" #include "curvefs/src/client/common/common.h" #include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/client/filesystem/xattr.h" #include "curvefs/src/client/fuse_common.h" #include "curvefs/src/client/client_operator.h" #include "curvefs/src/client/inode_wrapper.h" @@ -53,12 +54,22 @@ using ::curvefs::common::S3Info; using ::curvefs::common::Volume; using ::curvefs::mds::topology::PartitionTxId; using ::curvefs::mds::FSStatusCode_Name; -using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH; -using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH; using ::curvefs::client::filesystem::ExternalMember; using ::curvefs::client::filesystem::DirEntry; using ::curvefs::client::filesystem::DirEntryList; using ::curvefs::client::filesystem::FileOut; +using ::curvefs::client::filesystem::MAX_XATTR_NAME_LENGTH; +using ::curvefs::client::filesystem::MAX_XATTR_VALUE_LENGTH; +using ::curvefs::client::filesystem::XATTR_DIR_FILES; +using ::curvefs::client::filesystem::XATTR_DIR_SUBDIRS; +using ::curvefs::client::filesystem::XATTR_DIR_ENTRIES; +using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; +using ::curvefs::client::filesystem::XATTR_DIR_RFILES; +using ::curvefs::client::filesystem::XATTR_DIR_RFBYTES; +using ::curvefs::client::filesystem::XATTR_DIR_RSUBDIRS; +using ::curvefs::client::filesystem::XATTR_DIR_RENTRIES; +using ::curvefs::client::filesystem::XATTR_DIR_PREFIX; +using ::curvefs::client::filesystem::IsSpecialXAttr; #define RETURN_IF_UNSUCCESS(action) \ do { \ @@ -318,7 +329,7 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req, // update parent summary info const Inode *inode = inodeWrapper->GetInodeLocked(); XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(length)}); for (const auto &it : inode->parent()) { auto tret = xattrManager_->UpdateParentInodeXattr( @@ -463,13 +474,13 @@ CURVEFS_ERROR FuseClient::MakeNode( if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); if (type == FsFileType::TYPE_DIRECTORY) { - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); } else { - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true); if (tret != CURVEFS_ERROR::OK) { @@ -542,13 +553,13 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent, if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); if (FsFileType::TYPE_DIRECTORY == type) { - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); } else { - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, false); if (tret != CURVEFS_ERROR::OK) { @@ -633,13 +644,13 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); if (type == FsFileType::TYPE_DIRECTORY) { - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); } else { - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true); if (tret != CURVEFS_ERROR::OK) { @@ -1015,7 +1026,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, // update parent summary info const Inode* inode = inodeWrapper->GetInodeLocked(); XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(std::abs(changeSize))}); bool direction = changeSize > 0; for (const auto &it : inode->parent()) { @@ -1044,7 +1055,8 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino, (void)req; VLOG(9) << "FuseOpGetXattr, ino: " << ino << ", name: " << name << ", size = " << size; - if (common::FLAGS_fs_disableXattr) { + + if (option_.fileSystemOption.disableXAttr && !IsSpecialXAttr(name)) { return CURVEFS_ERROR::NODATA; } @@ -1087,6 +1099,11 @@ CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino, VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name << ", size = " << size << ", strvalue: " << strvalue; + + if (option_.fileSystemOption.disableXAttr && !IsSpecialXAttr(name)) { + return CURVEFS_ERROR::NODATA; + } + if (strname.length() > MAX_XATTR_NAME_LENGTH || size > MAX_XATTR_VALUE_LENGTH) { LOG(ERROR) << "xattr length is too long, name = " << name @@ -1135,10 +1152,10 @@ CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino, // add summary xattr key if (inodeAttr.type() == FsFileType::TYPE_DIRECTORY) { - *realSize += strlen(XATTRRFILES) + 1; - *realSize += strlen(XATTRRSUBDIRS) + 1; - *realSize += strlen(XATTRRENTRIES) + 1; - *realSize += strlen(XATTRRFBYTES) + 1; + *realSize += strlen(XATTR_DIR_RFILES) + 1; + *realSize += strlen(XATTR_DIR_RSUBDIRS) + 1; + *realSize += strlen(XATTR_DIR_RENTRIES) + 1; + *realSize += strlen(XATTR_DIR_RFBYTES) + 1; } if (size == 0) { @@ -1150,14 +1167,14 @@ CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino, value += tsize; } if (inodeAttr.type() == FsFileType::TYPE_DIRECTORY) { - memcpy(value, XATTRRFILES, strlen(XATTRRFILES) + 1); - value += strlen(XATTRRFILES) + 1; - memcpy(value, XATTRRSUBDIRS, strlen(XATTRRSUBDIRS) + 1); - value += strlen(XATTRRSUBDIRS) + 1; - memcpy(value, XATTRRENTRIES, strlen(XATTRRENTRIES) + 1); - value += strlen(XATTRRENTRIES) + 1; - memcpy(value, XATTRRFBYTES, strlen(XATTRRFBYTES) + 1); - value += strlen(XATTRRFBYTES) + 1; + memcpy(value, XATTR_DIR_RFILES, strlen(XATTR_DIR_RFILES) + 1); + value += strlen(XATTR_DIR_RFILES) + 1; + memcpy(value, XATTR_DIR_RSUBDIRS, strlen(XATTR_DIR_RSUBDIRS) + 1); + value += strlen(XATTR_DIR_RSUBDIRS) + 1; + memcpy(value, XATTR_DIR_RENTRIES, strlen(XATTR_DIR_RENTRIES) + 1); + value += strlen(XATTR_DIR_RENTRIES) + 1; + memcpy(value, XATTR_DIR_RFBYTES, strlen(XATTR_DIR_RFBYTES) + 1); + value += strlen(XATTR_DIR_RFBYTES) + 1; } return CURVEFS_ERROR::OK; } @@ -1215,9 +1232,9 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true); if (tret != CURVEFS_ERROR::OK) { @@ -1276,9 +1293,9 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr( newparent, xattr, true); diff --git a/curvefs/src/client/fuse_common.h b/curvefs/src/client/fuse_common.h index 280d96aa16..816bc7b5fb 100644 --- a/curvefs/src/client/fuse_common.h +++ b/curvefs/src/client/fuse_common.h @@ -28,6 +28,7 @@ #include #include +#include #ifdef __cplusplus extern "C" { diff --git a/curvefs/src/client/fuse_s3_client.cpp b/curvefs/src/client/fuse_s3_client.cpp index 5534857bab..216a65bf0b 100644 --- a/curvefs/src/client/fuse_s3_client.cpp +++ b/curvefs/src/client/fuse_s3_client.cpp @@ -27,6 +27,7 @@ #include #include "curvefs/src/client/kvclient/memcache_client.h" +#include "curvefs/src/client/filesystem/xattr.h" namespace curvefs { namespace client { @@ -46,6 +47,7 @@ using curvefs::client::common::FLAGS_supportKVcache; using curvefs::client::common::FLAGS_enableCto; using curvefs::mds::topology::MemcacheClusterInfo; using curvefs::mds::topology::MemcacheServerInfo; +using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) { FuseClientOption opt(option); @@ -204,7 +206,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, if (enableSumInDir_ && changeSize != 0) { const Inode* inode = inodeWrapper->GetInodeLocked(); XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(changeSize)}); for (const auto &it : inode->parent()) { auto tret = xattrManager_->UpdateParentInodeXattr(it, xattr, true); diff --git a/curvefs/src/client/rpcclient/BUILD b/curvefs/src/client/rpcclient/BUILD index c0e6927f96..13cdfccf1c 100644 --- a/curvefs/src/client/rpcclient/BUILD +++ b/curvefs/src/client/rpcclient/BUILD @@ -30,6 +30,7 @@ cc_library( "//curvefs/proto:space_cc_proto", "//curvefs/src/client/common", "//curvefs/src/client/metric:client_metric", + "//curvefs/src/client:filesystem_xattr", "//curvefs/src/common:curvefs_common", "//curvefs/src/common:metric_utils", "//external:brpc", diff --git a/curvefs/src/client/xattr_manager.cpp b/curvefs/src/client/xattr_manager.cpp index 61500e38eb..8ed6c350ea 100644 --- a/curvefs/src/client/xattr_manager.cpp +++ b/curvefs/src/client/xattr_manager.cpp @@ -23,6 +23,7 @@ #include "curvefs/src/client/xattr_manager.h" #include "curvefs/src/client/common/common.h" +#include "curvefs/src/client/filesystem/xattr.h" #include "src/common/string_util.h" namespace curvefs { @@ -31,16 +32,25 @@ namespace client { using ::curve::common::StringToUll; using ::curve::common::Thread; using ::curvefs::client::common::AddUllStringToFirst; +using ::curvefs::client::filesystem::XATTR_DIR_FILES; +using ::curvefs::client::filesystem::XATTR_DIR_SUBDIRS; +using ::curvefs::client::filesystem::XATTR_DIR_ENTRIES; +using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; +using ::curvefs::client::filesystem::XATTR_DIR_RFILES; +using ::curvefs::client::filesystem::XATTR_DIR_RSUBDIRS; +using ::curvefs::client::filesystem::XATTR_DIR_RENTRIES; +using ::curvefs::client::filesystem::XATTR_DIR_RFBYTES; +using ::curvefs::client::filesystem::XATTR_DIR_PREFIX; bool IsSummaryInfo(const char *name) { - return std::strstr(name, SUMMARYPREFIX); + return std::strstr(name, XATTR_DIR_PREFIX); } bool IsOneLayer(const char *name) { - if (std::strcmp(name, XATTRFILES) == 0 || - std::strcmp(name, XATTRSUBDIRS) == 0 || - std::strcmp(name, XATTRENTRIES) == 0 || - std::strcmp(name, XATTRFBYTES) == 0) { + if (std::strcmp(name, XATTR_DIR_FILES) == 0 || + std::strcmp(name, XATTR_DIR_SUBDIRS) == 0 || + std::strcmp(name, XATTR_DIR_ENTRIES) == 0 || + std::strcmp(name, XATTR_DIR_FBYTES) == 0) { return true; } return false; @@ -77,13 +87,13 @@ CURVEFS_ERROR XattrManager::CalOneLayerSumInfo(InodeAttr* attr) { summaryInfo.entries++; summaryInfo.fbytes += it.length(); } - (*attr->mutable_xattr())[XATTRFILES] = + (*attr->mutable_xattr())[XATTR_DIR_FILES] = std::to_string(summaryInfo.files); - (*attr->mutable_xattr())[XATTRSUBDIRS] = + (*attr->mutable_xattr())[XATTR_DIR_SUBDIRS] = std::to_string(summaryInfo.subdirs); - (*attr->mutable_xattr())[XATTRENTRIES] = + (*attr->mutable_xattr())[XATTR_DIR_ENTRIES] = std::to_string(summaryInfo.entries); - (*attr->mutable_xattr())[XATTRFBYTES] = + (*attr->mutable_xattr())[XATTR_DIR_FBYTES] = std::to_string(summaryInfo.fbytes + attr->length()); } return ret; @@ -91,7 +101,7 @@ CURVEFS_ERROR XattrManager::CalOneLayerSumInfo(InodeAttr* attr) { CURVEFS_ERROR XattrManager::FastCalOneLayerSumInfo(InodeAttr *attr) { if (!AddUllStringToFirst( - &(attr->mutable_xattr()->find(XATTRFBYTES)->second), + &(attr->mutable_xattr()->find(XATTR_DIR_FBYTES)->second), attr->length(), true)) { return CURVEFS_ERROR::INTERNAL; } @@ -265,13 +275,13 @@ CURVEFS_ERROR XattrManager::CalAllLayerSumInfo(InodeAttr *attr) { summaryInfo.fbytes -= it.second; } - attr->mutable_xattr()->insert({XATTRRFILES, + attr->mutable_xattr()->insert({XATTR_DIR_RFILES, std::to_string(summaryInfo.files)}); - attr->mutable_xattr()->insert({XATTRRSUBDIRS, + attr->mutable_xattr()->insert({XATTR_DIR_RSUBDIRS, std::to_string(summaryInfo.subdirs)}); - attr->mutable_xattr()->insert({XATTRRENTRIES, + attr->mutable_xattr()->insert({XATTR_DIR_RENTRIES, std::to_string(summaryInfo.entries)}); - attr->mutable_xattr()->insert({XATTRRFBYTES, + attr->mutable_xattr()->insert({XATTR_DIR_RFBYTES, std::to_string(summaryInfo.fbytes + attr->length())}); return CURVEFS_ERROR::OK; } @@ -305,30 +315,30 @@ void XattrManager::ConcurrentGetInodeXattr( if (tret == CURVEFS_ERROR::OK) { SummaryInfo summaryInfo; for (const auto &it : xattrs) { - if (it.xattrinfos().count(XATTRFILES)) { + if (it.xattrinfos().count(XATTR_DIR_FILES)) { if (!AddUllStringToFirst(&summaryInfo.files, - it.xattrinfos().find(XATTRFILES)->second)) { + it.xattrinfos().find(XATTR_DIR_FILES)->second)) { ret->store(false); return; } } - if (it.xattrinfos().count(XATTRSUBDIRS)) { + if (it.xattrinfos().count(XATTR_DIR_SUBDIRS)) { if (!AddUllStringToFirst(&summaryInfo.subdirs, - it.xattrinfos().find(XATTRSUBDIRS)->second)) { + it.xattrinfos().find(XATTR_DIR_SUBDIRS)->second)) { ret->store(false); return; } } - if (it.xattrinfos().count(XATTRENTRIES)) { + if (it.xattrinfos().count(XATTR_DIR_ENTRIES)) { if (!AddUllStringToFirst(&summaryInfo.entries, - it.xattrinfos().find(XATTRENTRIES)->second)) { + it.xattrinfos().find(XATTR_DIR_ENTRIES)->second)) { ret->store(false); return; } } - if (it.xattrinfos().count(XATTRFBYTES)) { + if (it.xattrinfos().count(XATTR_DIR_FBYTES)) { if (!AddUllStringToFirst(&summaryInfo.fbytes, - it.xattrinfos().find(XATTRFBYTES)->second)) { + it.xattrinfos().find(XATTR_DIR_FBYTES)->second)) { ret->store(false); return; } @@ -337,16 +347,16 @@ void XattrManager::ConcurrentGetInodeXattr( // record summary info to target inode std::lock_guard guard(*inodeMutex); if (!(AddUllStringToFirst( - &(attr->mutable_xattr()->find(XATTRRFILES)->second), + &(attr->mutable_xattr()->find(XATTR_DIR_RFILES)->second), summaryInfo.files, true) && AddUllStringToFirst( - &(attr->mutable_xattr()->find(XATTRRSUBDIRS)->second), + &(attr->mutable_xattr()->find(XATTR_DIR_RSUBDIRS)->second), summaryInfo.subdirs, true) && AddUllStringToFirst( - &(attr->mutable_xattr()->find(XATTRRENTRIES)->second), + &(attr->mutable_xattr()->find(XATTR_DIR_RENTRIES)->second), summaryInfo.entries, true) && AddUllStringToFirst( - &(attr->mutable_xattr()->find(XATTRRFBYTES)->second), + &(attr->mutable_xattr()->find(XATTR_DIR_RFBYTES)->second), summaryInfo.fbytes, true))) { ret->store(false); return; @@ -368,20 +378,20 @@ CURVEFS_ERROR XattrManager::FastCalAllLayerSumInfo(InodeAttr *attr) { iStack.emplace(ino); // add the size of itself first if (!AddUllStringToFirst( - &(attr->mutable_xattr()->find(XATTRFBYTES)->second), + &(attr->mutable_xattr()->find(XATTR_DIR_FBYTES)->second), attr->length(), true)) { return CURVEFS_ERROR::INTERNAL; } // add first layer summary to all layer summary info - attr->mutable_xattr()->insert({XATTRRFILES, - attr->xattr().find(XATTRFILES)->second}); - attr->mutable_xattr()->insert({XATTRRSUBDIRS, - attr->xattr().find(XATTRSUBDIRS)->second}); - attr->mutable_xattr()->insert({XATTRRENTRIES, - attr->xattr().find(XATTRENTRIES)->second}); - attr->mutable_xattr()->insert({XATTRRFBYTES, - attr->xattr().find(XATTRFBYTES)->second}); + attr->mutable_xattr()->insert({XATTR_DIR_RFILES, + attr->xattr().find(XATTR_DIR_FILES)->second}); + attr->mutable_xattr()->insert({XATTR_DIR_RSUBDIRS, + attr->xattr().find(XATTR_DIR_SUBDIRS)->second}); + attr->mutable_xattr()->insert({XATTR_DIR_RENTRIES, + attr->xattr().find(XATTR_DIR_ENTRIES)->second}); + attr->mutable_xattr()->insert({XATTR_DIR_RFBYTES, + attr->xattr().find(XATTR_DIR_FBYTES)->second}); std::vector threadpool; Atomic inflightNum(0); @@ -512,13 +522,13 @@ CURVEFS_ERROR XattrManager::UpdateParentXattrAfterRename(uint64_t parent, return rc; } XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); if (dentry.type() == FsFileType::TYPE_DIRECTORY) { - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); } else { - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); // update src parent @@ -547,13 +557,13 @@ CURVEFS_ERROR XattrManager::UpdateParentXattrAfterRename(uint64_t parent, renameOp->GetOldInode(&oldInode, &oldInodeSize, &oldInodeType); if (oldInode != 0 && oldInodeSize >= 0) { XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); if (oldInodeType == FsFileType::TYPE_DIRECTORY) { - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); } else { - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, std::to_string(oldInodeSize)}); rc = UpdateParentInodeXattr(newparent, xattr, false); diff --git a/curvefs/src/common/define.h b/curvefs/src/common/define.h index eb9a23be2d..c07af5a9b9 100644 --- a/curvefs/src/common/define.h +++ b/curvefs/src/common/define.h @@ -24,19 +24,24 @@ #define CURVEFS_SRC_COMMON_DEFINE_H_ #include +#include "curvefs/src/client/filesystem/xattr.h" + namespace curvefs { const uint64_t ROOTINODEID = 1; const uint64_t RECYCLEINODEID = 2; const char RECYCLENAME[] = ".recycle"; -const char XATTRFILES[] = "curve.dir.files"; -const char XATTRSUBDIRS[] = "curve.dir.subdirs"; -const char XATTRENTRIES[] = "curve.dir.entries"; -const char XATTRFBYTES[] = "curve.dir.fbytes"; -const char XATTRRFILES[] = "curve.dir.rfiles"; -const char XATTRRSUBDIRS[] = "curve.dir.rsubdirs"; -const char XATTRRENTRIES[] = "curve.dir.rentries"; -const char XATTRRFBYTES[] = "curve.dir.rfbytes"; -const char SUMMARYPREFIX[] = "curve.dir"; +using ::curvefs::client::filesystem::XATTR_DIR_FILES; +using ::curvefs::client::filesystem::XATTR_DIR_SUBDIRS; +using ::curvefs::client::filesystem::XATTR_DIR_ENTRIES; +using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; +using ::curvefs::client::filesystem::XATTR_DIR_RFILES; +using ::curvefs::client::filesystem::XATTR_DIR_RSUBDIRS; +using ::curvefs::client::filesystem::XATTR_DIR_RENTRIES; +using ::curvefs::client::filesystem::XATTR_DIR_RFBYTES; +using ::curvefs::client::filesystem::XATTR_DIR_PREFIX; +using ::curvefs::client::filesystem::XATTR_WARMUP_OP; +using ::curvefs::client::filesystem::XATTR_WARMUP_OP_LIST; + } // namespace curvefs #endif // CURVEFS_SRC_COMMON_DEFINE_H_ diff --git a/curvefs/src/mds/BUILD b/curvefs/src/mds/BUILD index c6d003f470..bf46e8d63c 100644 --- a/curvefs/src/mds/BUILD +++ b/curvefs/src/mds/BUILD @@ -68,6 +68,7 @@ cc_library( "//curvefs/src/mds/schedule/scheduleService:curvefs_schedule_service", "//curvefs/src/mds/space:curvefs_mds_space", "//curvefs/src/mds/topology:curvefs_topology", + "//curvefs/src/client:filesystem_xattr", "//external:brpc", "//external:gflags", "//external:glog", diff --git a/curvefs/src/metaserver/BUILD b/curvefs/src/metaserver/BUILD index 04c90247dd..df1696159a 100644 --- a/curvefs/src/metaserver/BUILD +++ b/curvefs/src/metaserver/BUILD @@ -61,6 +61,7 @@ cc_library( "//curvefs/src/metaserver/common:fs_metaserver_common", "//curvefs/src/volume:volume", "//curvefs/src/client:fuse_client_lib", + "//curvefs/src/client:filesystem_xattr", "//external:braft", "//src/common:curve_common", "//src/fs:lfs", diff --git a/curvefs/src/metaserver/inode_manager.cpp b/curvefs/src/metaserver/inode_manager.cpp index d5e4fce007..ee4a6d675c 100644 --- a/curvefs/src/metaserver/inode_manager.cpp +++ b/curvefs/src/metaserver/inode_manager.cpp @@ -36,10 +36,15 @@ #include "curvefs/src/metaserver/storage/storage.h" #include "src/common/concurrent/name_lock.h" #include "src/common/timeutility.h" +#include "curvefs/src/client/filesystem/xattr.h" using ::curve::common::NameLockGuard; using ::curve::common::TimeUtility; using ::google::protobuf::util::MessageDifferencer; +using ::curvefs::client::filesystem::XATTR_DIR_FILES; +using ::curvefs::client::filesystem::XATTR_DIR_SUBDIRS; +using ::curvefs::client::filesystem::XATTR_DIR_ENTRIES; +using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; #define CHECK_APPLIED() \ do { \ @@ -196,10 +201,10 @@ void InodeManager::GenerateInodeInternal(uint64_t inodeId, if (FsFileType::TYPE_DIRECTORY == param.type) { inode->set_nlink(2); // set summary xattr - inode->mutable_xattr()->insert({XATTRFILES, "0"}); - inode->mutable_xattr()->insert({XATTRSUBDIRS, "0"}); - inode->mutable_xattr()->insert({XATTRENTRIES, "0"}); - inode->mutable_xattr()->insert({XATTRFBYTES, "0"}); + inode->mutable_xattr()->insert({XATTR_DIR_FILES, "0"}); + inode->mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "0"}); + inode->mutable_xattr()->insert({XATTR_DIR_ENTRIES, "0"}); + inode->mutable_xattr()->insert({XATTR_DIR_FBYTES, "0"}); } else { inode->set_nlink(1); } diff --git a/curvefs/test/client/filesystem/filesystem_test.cpp b/curvefs/test/client/filesystem/filesystem_test.cpp index e8087818fe..940f9d9f16 100644 --- a/curvefs/test/client/filesystem/filesystem_test.cpp +++ b/curvefs/test/client/filesystem/filesystem_test.cpp @@ -248,14 +248,34 @@ TEST_F(FileSystemTest, ReadDir_Basic) { fi.fh = handler->fh; // CASE 1: readdir success - EXPECT_CALL_RETURN_ListDentry(*builder.GetDentryManager(), - CURVEFS_ERROR::OK); - EXPECT_CALL_RETURN_BatchGetInodeAttrAsync(*builder.GetInodeManager(), - CURVEFS_ERROR::OK); + EXPECT_CALL_INVOKE_ListDentry(*builder.GetDentryManager(), + [&](uint64_t parent, + std::list* dentries, + uint32_t limit, + bool only, + uint32_t nlink) -> CURVEFS_ERROR { + dentries->push_back(MkDentry(1, "test")); + return CURVEFS_ERROR::OK; + }); + EXPECT_CALL_INVOKE_BatchGetInodeAttrAsync(*builder.GetInodeManager(), + [&](uint64_t parentId, + std::set* inos, + std::map* attrs) -> CURVEFS_ERROR { + for (const auto& ino : *inos) { + auto attr = MkAttr(ino, AttrOption().mtime(123, ino)); + attrs->emplace(ino, attr); + } + return CURVEFS_ERROR::OK; + }); + DirEntry dirEntry; auto entries = std::make_shared(); auto rc = fs->ReadDir(Request(), 1, &fi, &entries); ASSERT_EQ(rc, CURVEFS_ERROR::OK); + ASSERT_EQ(entries->Size(), 1); + ASSERT_TRUE(entries->Get(1, &dirEntry)); + ASSERT_EQ(dirEntry.ino, 1); + ASSERT_EQ(dirEntry.name, "test"); } TEST_F(FileSystemTest, ReadDir_CheckEntries) { diff --git a/curvefs/test/client/filesystem/helper/builder.h b/curvefs/test/client/filesystem/helper/builder.h index 56c4b70874..84d4e6a22e 100644 --- a/curvefs/test/client/filesystem/helper/builder.h +++ b/curvefs/test/client/filesystem/helper/builder.h @@ -200,7 +200,7 @@ class FileSystemBuilder { }; option.cto = true; - common::FLAGS_fs_disableXattr = option.disableXattr = true; + option.disableXAttr = true; option.maxNameLength = 255; option.blockSize = 0x10000u; option.kernelCacheOption = kernelCacheOption; diff --git a/curvefs/test/client/filesystem/rpc_client_test.cpp b/curvefs/test/client/filesystem/rpc_client_test.cpp index 9419af8579..9763e15787 100644 --- a/curvefs/test/client/filesystem/rpc_client_test.cpp +++ b/curvefs/test/client/filesystem/rpc_client_test.cpp @@ -106,14 +106,34 @@ TEST_F(RPCClientTest, ReadDir_Basic) { // CASE 1: ok { - EXPECT_CALL_RETURN_ListDentry(*builder.GetDentryManager(), - CURVEFS_ERROR::OK); - EXPECT_CALL_RETURN_BatchGetInodeAttrAsync(*builder.GetInodeManager(), - CURVEFS_ERROR::OK); - + EXPECT_CALL_INVOKE_ListDentry(*builder.GetDentryManager(), + [&](uint64_t parent, + std::list* dentries, + uint32_t limit, + bool only, + uint32_t nlink) -> CURVEFS_ERROR { + dentries->push_back(MkDentry(1, "test")); + return CURVEFS_ERROR::OK; + }); + EXPECT_CALL_INVOKE_BatchGetInodeAttrAsync(*builder.GetInodeManager(), + [&](uint64_t parentId, + std::set* inos, + std::map* attrs) -> CURVEFS_ERROR { + for (const auto& ino : *inos) { + auto attr = MkAttr(ino, AttrOption().mtime(123, ino)); + attrs->emplace(ino, attr); + } + return CURVEFS_ERROR::OK; + }); + + DirEntry dirEntry; auto entries = std::make_shared(); auto rc = rpc->ReadDir(100, &entries); ASSERT_EQ(rc, CURVEFS_ERROR::OK); + ASSERT_EQ(entries->Size(), 1); + ASSERT_TRUE(entries->Get(1, &dirEntry)); + ASSERT_EQ(dirEntry.ino, 1); + ASSERT_EQ(dirEntry.name, "test"); } // CASE 2: inode not exist diff --git a/curvefs/test/client/rpcclient/BUILD b/curvefs/test/client/rpcclient/BUILD index fa2a3d4649..0503eac796 100644 --- a/curvefs/test/client/rpcclient/BUILD +++ b/curvefs/test/client/rpcclient/BUILD @@ -27,6 +27,7 @@ cc_test( "//external:gtest", "@com_google_googletest//:gtest_main", "//curvefs/src/client/rpcclient:rpcclient", + "//curvefs/src/client:filesystem_xattr", "//curvefs/proto:curvefs_topology_cc_proto", "//curvefs/src/common:curvefs_common", "//curvefs/test/utils:curvefs_test_utils", diff --git a/curvefs/test/client/rpcclient/metaserver_client_test.cpp b/curvefs/test/client/rpcclient/metaserver_client_test.cpp index 4fdae51576..bfffefce54 100644 --- a/curvefs/test/client/rpcclient/metaserver_client_test.cpp +++ b/curvefs/test/client/rpcclient/metaserver_client_test.cpp @@ -973,10 +973,10 @@ TEST_F(MetaServerClientImplTest, test_BatchGetXAttr) { curvefs::metaserver::XAttr out; out.set_fsid(fsid); out.set_inodeid(inodeId1); - out.mutable_xattrinfos()->insert({XATTRFILES, "1"}); - out.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); - out.mutable_xattrinfos()->insert({XATTRENTRIES, "2"}); - out.mutable_xattrinfos()->insert({XATTRFBYTES, "100"}); + out.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); + out.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); + out.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "2"}); + out.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, "100"}); curvefs::metaserver::XAttr out1 = out; out1.set_inodeid(inodeId2); xattr.emplace_back(out); diff --git a/curvefs/test/client/test_fuse_s3_client.cpp b/curvefs/test/client/test_fuse_s3_client.cpp index d22369abfa..c8f81d093c 100644 --- a/curvefs/test/client/test_fuse_s3_client.cpp +++ b/curvefs/test/client/test_fuse_s3_client.cpp @@ -3437,10 +3437,10 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir) { inode.set_inodeid(ino); inode.set_length(4096); inode.set_type(FsFileType::TYPE_DIRECTORY); - inode.mutable_xattr()->insert({XATTRFILES, "0"}); - inode.mutable_xattr()->insert({XATTRSUBDIRS, "0"}); - inode.mutable_xattr()->insert({XATTRENTRIES, "0"}); - inode.mutable_xattr()->insert({XATTRFBYTES, "0"}); + inode.mutable_xattr()->insert({XATTR_DIR_FILES, "0"}); + inode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "0"}); + inode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "0"}); + inode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "0"}); EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _)) .WillOnce(DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK))); @@ -3502,10 +3502,10 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) { InodeAttr inode; inode.set_inodeid(ino); - inode.mutable_xattr()->insert({XATTRFILES, "aaa"}); - inode.mutable_xattr()->insert({XATTRSUBDIRS, "1"}); - inode.mutable_xattr()->insert({XATTRENTRIES, "2"}); - inode.mutable_xattr()->insert({XATTRFBYTES, "100"}); + inode.mutable_xattr()->insert({XATTR_DIR_FILES, "aaa"}); + inode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "1"}); + inode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "2"}); + inode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "100"}); // get inode failed EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _)) @@ -3562,19 +3562,19 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir) { XAttr xattr; xattr.set_fsid(fsId); xattr.set_inodeid(ino); - xattr.mutable_xattrinfos()->insert({XATTRFILES, "2"}); - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "2"}); - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "4"}); - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, "200"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "2"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "2"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "4"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, "200"}); xattrs.emplace_back(xattr); InodeAttr inode; inode.set_inodeid(ino); inode.set_nlink(3); - inode.mutable_xattr()->insert({XATTRFILES, "1"}); - inode.mutable_xattr()->insert({XATTRSUBDIRS, "1"}); - inode.mutable_xattr()->insert({XATTRENTRIES, "2"}); - inode.mutable_xattr()->insert({XATTRFBYTES, "100"}); + inode.mutable_xattr()->insert({XATTR_DIR_FILES, "1"}); + inode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "1"}); + inode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "2"}); + inode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "100"}); InodeAttr attr = inode; attr.set_nlink(2); @@ -3627,10 +3627,10 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) { XAttr xattr; xattr.set_fsid(fsId); xattr.set_inodeid(inodeId); - xattr.mutable_xattrinfos()->insert({XATTRFILES, "2"}); - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "2"}); - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "4"}); - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, "200"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "2"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "2"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "4"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, "200"}); xattrs.emplace_back(xattr); InodeAttr inode; @@ -3638,10 +3638,10 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) { inode.set_nlink(3); inode.set_length(4096); inode.set_type(FsFileType::TYPE_DIRECTORY); - inode.mutable_xattr()->insert({XATTRFILES, "1"}); - inode.mutable_xattr()->insert({XATTRSUBDIRS, "1"}); - inode.mutable_xattr()->insert({XATTRENTRIES, "2"}); - inode.mutable_xattr()->insert({XATTRFBYTES, "aaa"}); + inode.mutable_xattr()->insert({XATTR_DIR_FILES, "1"}); + inode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "1"}); + inode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "2"}); + inode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "aaa"}); // get inode failed EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _)) @@ -3654,7 +3654,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) { .WillOnce(DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK))); ret = client_->FuseOpGetXattr(req, ino, name, &value, size); ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - inode.mutable_xattr()->find(XATTRFBYTES)->second = "100"; + inode.mutable_xattr()->find(XATTR_DIR_FBYTES)->second = "100"; // list dentry failed EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _)) @@ -3680,8 +3680,8 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) { ret = client_->FuseOpGetXattr(req, ino, rname, &value, size); ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - // AddUllStringToFirst XATTRFILES failed - inode.mutable_xattr()->find(XATTRFILES)->second = "aaa"; + // AddUllStringToFirst XATTR_DIR_FILES failed + inode.mutable_xattr()->find(XATTR_DIR_FILES)->second = "aaa"; EXPECT_CALL(*inodeManager_, GetInodeAttr(_, _)) .Times(AtLeast(2)) .WillRepeatedly( @@ -3695,9 +3695,9 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) { ret = client_->FuseOpGetXattr(req, ino, rname, &value, size); ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - // AddUllStringToFirst XATTRSUBDIRS failed - inode.mutable_xattr()->find(XATTRFILES)->second = "0"; - inode.mutable_xattr()->find(XATTRSUBDIRS)->second = "aaa"; + // AddUllStringToFirst XATTR_DIR_SUBDIRS failed + inode.mutable_xattr()->find(XATTR_DIR_FILES)->second = "0"; + inode.mutable_xattr()->find(XATTR_DIR_SUBDIRS)->second = "aaa"; EXPECT_CALL(*inodeManager_, GetInodeAttr(_, _)) .Times(AtLeast(2)) .WillRepeatedly( @@ -3711,9 +3711,9 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) { ret = client_->FuseOpGetXattr(req, ino, rname, &value, size); ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - // AddUllStringToFirst XATTRENTRIES failed - inode.mutable_xattr()->find(XATTRSUBDIRS)->second = "0"; - inode.mutable_xattr()->find(XATTRENTRIES)->second = "aaa"; + // AddUllStringToFirst XATTR_DIR_ENTRIES failed + inode.mutable_xattr()->find(XATTR_DIR_SUBDIRS)->second = "0"; + inode.mutable_xattr()->find(XATTR_DIR_ENTRIES)->second = "aaa"; EXPECT_CALL(*inodeManager_, GetInodeAttr(_, _)) .Times(AtLeast(2)) .WillRepeatedly( @@ -3762,10 +3762,10 @@ TEST_F(TestFuseS3Client, FuseOpCreate_EnableSummary) { parentInode.set_inodeid(parent); parentInode.set_type(FsFileType::TYPE_DIRECTORY); parentInode.set_nlink(2); - parentInode.mutable_xattr()->insert({XATTRFILES, "1"}); - parentInode.mutable_xattr()->insert({XATTRSUBDIRS, "1"}); - parentInode.mutable_xattr()->insert({XATTRENTRIES, "2"}); - parentInode.mutable_xattr()->insert({XATTRFBYTES, "100"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_FILES, "1"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "1"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "2"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "100"}); auto parentInodeWrapper = std::make_shared(parentInode, metaClient_); @@ -3785,10 +3785,10 @@ TEST_F(TestFuseS3Client, FuseOpCreate_EnableSummary) { ASSERT_EQ(CURVEFS_ERROR::OK, ret); auto p = parentInodeWrapper->GetInodeLocked(); - ASSERT_EQ(p->xattr().find(XATTRFILES)->second, "2"); - ASSERT_EQ(p->xattr().find(XATTRSUBDIRS)->second, "1"); - ASSERT_EQ(p->xattr().find(XATTRENTRIES)->second, "3"); - ASSERT_EQ(p->xattr().find(XATTRFBYTES)->second, "4196"); + ASSERT_EQ(p->xattr().find(XATTR_DIR_FILES)->second, "2"); + ASSERT_EQ(p->xattr().find(XATTR_DIR_SUBDIRS)->second, "1"); + ASSERT_EQ(p->xattr().find(XATTR_DIR_ENTRIES)->second, "3"); + ASSERT_EQ(p->xattr().find(XATTR_DIR_FBYTES)->second, "4196"); } TEST_F(TestFuseS3Client, FuseOpWrite_EnableSummary) { @@ -3814,10 +3814,10 @@ TEST_F(TestFuseS3Client, FuseOpWrite_EnableSummary) { parentInode.set_inodeid(0); parentInode.set_type(FsFileType::TYPE_DIRECTORY); parentInode.set_nlink(2); - parentInode.mutable_xattr()->insert({XATTRFILES, "1"}); - parentInode.mutable_xattr()->insert({XATTRSUBDIRS, "0"}); - parentInode.mutable_xattr()->insert({XATTRENTRIES, "1"}); - parentInode.mutable_xattr()->insert({XATTRFBYTES, "0"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_FILES, "1"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "0"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "1"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "0"}); auto parentInodeWrapper = std::make_shared(parentInode, metaClient_); @@ -3837,10 +3837,10 @@ TEST_F(TestFuseS3Client, FuseOpWrite_EnableSummary) { ASSERT_EQ(size, fileOut.nwritten); auto p = parentInodeWrapper->GetInodeLocked(); - ASSERT_EQ(p->xattr().find(XATTRFILES)->second, "1"); - ASSERT_EQ(p->xattr().find(XATTRSUBDIRS)->second, "0"); - ASSERT_EQ(p->xattr().find(XATTRENTRIES)->second, "1"); - ASSERT_EQ(p->xattr().find(XATTRFBYTES)->second, std::to_string(size)); + ASSERT_EQ(p->xattr().find(XATTR_DIR_FILES)->second, "1"); + ASSERT_EQ(p->xattr().find(XATTR_DIR_SUBDIRS)->second, "0"); + ASSERT_EQ(p->xattr().find(XATTR_DIR_ENTRIES)->second, "1"); + ASSERT_EQ(p->xattr().find(XATTR_DIR_FBYTES)->second, std::to_string(size)); } TEST_F(TestFuseS3Client, FuseOpLink_EnableSummary) { @@ -3861,10 +3861,10 @@ TEST_F(TestFuseS3Client, FuseOpLink_EnableSummary) { Inode pinode; pinode.set_inodeid(0); pinode.set_length(0); - pinode.mutable_xattr()->insert({XATTRFILES, "0"}); - pinode.mutable_xattr()->insert({XATTRSUBDIRS, "0"}); - pinode.mutable_xattr()->insert({XATTRENTRIES, "0"}); - pinode.mutable_xattr()->insert({XATTRFBYTES, "0"}); + pinode.mutable_xattr()->insert({XATTR_DIR_FILES, "0"}); + pinode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "0"}); + pinode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "0"}); + pinode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "0"}); auto pinodeWrapper = std::make_shared(pinode, metaClient_); EXPECT_CALL(*inodeManager_, GetInode(_, _)) @@ -3882,10 +3882,10 @@ TEST_F(TestFuseS3Client, FuseOpLink_EnableSummary) { client_->FuseOpLink(req, ino, newparent, newname, &entryOut); ASSERT_EQ(CURVEFS_ERROR::OK, ret); auto p = pinodeWrapper->GetInode(); - ASSERT_EQ(p.xattr().find(XATTRFILES)->second, "1"); - ASSERT_EQ(p.xattr().find(XATTRSUBDIRS)->second, "0"); - ASSERT_EQ(p.xattr().find(XATTRENTRIES)->second, "1"); - ASSERT_EQ(p.xattr().find(XATTRFBYTES)->second, "100"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_FILES)->second, "1"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_SUBDIRS)->second, "0"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_ENTRIES)->second, "1"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_FBYTES)->second, "100"); } TEST_F(TestFuseS3Client, FuseOpUnlink_EnableSummary) { @@ -3925,10 +3925,10 @@ TEST_F(TestFuseS3Client, FuseOpUnlink_EnableSummary) { parentInode.set_inodeid(parent); parentInode.set_type(FsFileType::TYPE_DIRECTORY); parentInode.set_nlink(3); - parentInode.mutable_xattr()->insert({XATTRFILES, "1"}); - parentInode.mutable_xattr()->insert({XATTRSUBDIRS, "1"}); - parentInode.mutable_xattr()->insert({XATTRENTRIES, "2"}); - parentInode.mutable_xattr()->insert({XATTRFBYTES, "4196"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_FILES, "1"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "1"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "2"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "4196"}); InodeAttr attr; attr.set_fsid(fsId); @@ -3959,10 +3959,10 @@ TEST_F(TestFuseS3Client, FuseOpUnlink_EnableSummary) { auto p = parentInodeWrapper->GetInode(); ASSERT_EQ(3, p.nlink()); - ASSERT_EQ(p.xattr().find(XATTRFILES)->second, "0"); - ASSERT_EQ(p.xattr().find(XATTRSUBDIRS)->second, "1"); - ASSERT_EQ(p.xattr().find(XATTRENTRIES)->second, "1"); - ASSERT_EQ(p.xattr().find(XATTRFBYTES)->second, "100"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_FILES)->second, "0"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_SUBDIRS)->second, "1"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_ENTRIES)->second, "1"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_FBYTES)->second, "100"); } TEST_F(TestFuseS3Client, FuseOpOpen_Trunc_EnableSummary) { @@ -3990,10 +3990,10 @@ TEST_F(TestFuseS3Client, FuseOpOpen_Trunc_EnableSummary) { parentInode.set_inodeid(0); parentInode.set_type(FsFileType::TYPE_DIRECTORY); parentInode.set_nlink(3); - parentInode.mutable_xattr()->insert({XATTRFILES, "1"}); - parentInode.mutable_xattr()->insert({XATTRSUBDIRS, "1"}); - parentInode.mutable_xattr()->insert({XATTRENTRIES, "2"}); - parentInode.mutable_xattr()->insert({XATTRFBYTES, "4196"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_FILES, "1"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "1"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "2"}); + parentInode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "4196"}); auto parentInodeWrapper = std::make_shared(parentInode, metaClient_); @@ -4026,11 +4026,11 @@ TEST_F(TestFuseS3Client, FuseOpOpen_Trunc_EnableSummary) { ASSERT_EQ(CURVEFS_ERROR::OK, ret); auto p = parentInodeWrapper->GetInode(); - ASSERT_EQ(p.xattr().find(XATTRFILES)->second, "1"); - ASSERT_EQ(p.xattr().find(XATTRSUBDIRS)->second, "1"); - ASSERT_EQ(p.xattr().find(XATTRENTRIES)->second, "2"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_FILES)->second, "1"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_SUBDIRS)->second, "1"); + ASSERT_EQ(p.xattr().find(XATTR_DIR_ENTRIES)->second, "2"); // FIXME: (Wine93) - // ASSERT_EQ(p.xattr().find(XATTRFBYTES)->second, "100"); + // ASSERT_EQ(p.xattr().find(XATTR_DIR_FBYTES)->second, "100"); } TEST_F(TestFuseS3Client, FuseOpListXattr) { @@ -4069,9 +4069,11 @@ TEST_F(TestFuseS3Client, FuseOpListXattr) { .WillOnce(DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK))); ret = client_->FuseOpListXattr(req, ino, buf, size, &realSize); ASSERT_EQ(CURVEFS_ERROR::OK, ret); - auto expected = key.length() + 1 + strlen(XATTRRFILES) + 1 + - strlen(XATTRRSUBDIRS) + 1 + strlen(XATTRRENTRIES) + 1 + - strlen(XATTRRFBYTES) + 1; + auto expected = key.length() + 1 + + strlen(XATTR_DIR_RFILES) + 1 + + strlen(XATTR_DIR_RSUBDIRS) + 1 + + strlen(XATTR_DIR_RENTRIES) + 1 + + strlen(XATTR_DIR_RFBYTES) + 1; ASSERT_EQ(realSize, expected); realSize = 0; diff --git a/curvefs/test/client/test_inode_cache_manager.cpp b/curvefs/test/client/test_inode_cache_manager.cpp index 02b6133ebb..6b266118f0 100644 --- a/curvefs/test/client/test_inode_cache_manager.cpp +++ b/curvefs/test/client/test_inode_cache_manager.cpp @@ -392,13 +392,13 @@ TEST_F(TestInodeCacheManager, BatchGetXAttr) { XAttr xattr; xattr.set_fsid(fsId_); xattr.set_inodeid(inodeId1); - xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"}); - xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "2"}); - xattr.mutable_xattrinfos()->insert({XATTRFBYTES, "100"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_SUBDIRS, "1"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "2"}); + xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, "100"}); xattrs.emplace_back(xattr); xattr.set_inodeid(inodeId2); - xattr.mutable_xattrinfos()->find(XATTRFBYTES)->second = "200"; + xattr.mutable_xattrinfos()->find(XATTR_DIR_FBYTES)->second = "200"; xattrs.emplace_back(xattr); EXPECT_CALL(*metaClient_, BatchGetXAttr(fsId_, inodeIds, _)) @@ -415,7 +415,7 @@ TEST_F(TestInodeCacheManager, BatchGetXAttr) { ASSERT_EQ(getXAttrs.size(), 2); ASSERT_THAT(getXAttrs.begin()->inodeid(), AnyOf(inodeId1, inodeId2)); ASSERT_EQ(getXAttrs.begin()->fsid(), fsId_); - ASSERT_THAT(getXAttrs.begin()->xattrinfos().find(XATTRFBYTES)->second, + ASSERT_THAT(getXAttrs.begin()->xattrinfos().find(XATTR_DIR_FBYTES)->second, AnyOf("100", "200")); } diff --git a/curvefs/test/metaserver/inode_manager_test.cpp b/curvefs/test/metaserver/inode_manager_test.cpp index a407ccde1a..42b5734bc2 100644 --- a/curvefs/test/metaserver/inode_manager_test.cpp +++ b/curvefs/test/metaserver/inode_manager_test.cpp @@ -465,10 +465,10 @@ TEST_F(InodeManagerTest, testGetXAttr) { ASSERT_EQ(manager->CreateInode(3, param_, &inode2, logIndex_++), MetaStatusCode::OK); ASSERT_FALSE(inode2.xattr().empty()); - ASSERT_EQ(inode2.xattr().find(XATTRFILES)->second, "0"); - ASSERT_EQ(inode2.xattr().find(XATTRSUBDIRS)->second, "0"); - ASSERT_EQ(inode2.xattr().find(XATTRENTRIES)->second, "0"); - ASSERT_EQ(inode2.xattr().find(XATTRFBYTES)->second, "0"); + ASSERT_EQ(inode2.xattr().find(XATTR_DIR_FILES)->second, "0"); + ASSERT_EQ(inode2.xattr().find(XATTR_DIR_SUBDIRS)->second, "0"); + ASSERT_EQ(inode2.xattr().find(XATTR_DIR_ENTRIES)->second, "0"); + ASSERT_EQ(inode2.xattr().find(XATTR_DIR_FBYTES)->second, "0"); // GET XAttr xattr; @@ -477,16 +477,16 @@ TEST_F(InodeManagerTest, testGetXAttr) { ASSERT_EQ(xattr.fsid(), fsId); ASSERT_EQ(xattr.inodeid(), inode2.inodeid()); ASSERT_EQ(xattr.xattrinfos_size(), 4); - ASSERT_EQ(xattr.xattrinfos().find(XATTRFILES)->second, "0"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRSUBDIRS)->second, "0"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRENTRIES)->second, "0"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRFBYTES)->second, "0"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_FILES)->second, "0"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_SUBDIRS)->second, "0"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_ENTRIES)->second, "0"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_FBYTES)->second, "0"); // UPDATE - inode2.mutable_xattr()->find(XATTRFILES)->second = "1"; - inode2.mutable_xattr()->find(XATTRSUBDIRS)->second = "1"; - inode2.mutable_xattr()->find(XATTRENTRIES)->second = "2"; - inode2.mutable_xattr()->find(XATTRFBYTES)->second = "100"; + inode2.mutable_xattr()->find(XATTR_DIR_FILES)->second = "1"; + inode2.mutable_xattr()->find(XATTR_DIR_SUBDIRS)->second = "1"; + inode2.mutable_xattr()->find(XATTR_DIR_ENTRIES)->second = "2"; + inode2.mutable_xattr()->find(XATTR_DIR_FBYTES)->second = "100"; UpdateInodeRequest request = MakeUpdateInodeRequestFromInode(inode2); ASSERT_EQ(manager->UpdateInode(request, logIndex_++), MetaStatusCode::OK); @@ -495,10 +495,10 @@ TEST_F(InodeManagerTest, testGetXAttr) { ASSERT_EQ(manager->GetXAttr(fsId, inode2.inodeid(), &xattr1), MetaStatusCode::OK); ASSERT_EQ(xattr1.xattrinfos_size(), 4); - ASSERT_EQ(xattr1.xattrinfos().find(XATTRFILES)->second, "1"); - ASSERT_EQ(xattr1.xattrinfos().find(XATTRSUBDIRS)->second, "1"); - ASSERT_EQ(xattr1.xattrinfos().find(XATTRENTRIES)->second, "2"); - ASSERT_EQ(xattr1.xattrinfos().find(XATTRFBYTES)->second, "100"); + ASSERT_EQ(xattr1.xattrinfos().find(XATTR_DIR_FILES)->second, "1"); + ASSERT_EQ(xattr1.xattrinfos().find(XATTR_DIR_SUBDIRS)->second, "1"); + ASSERT_EQ(xattr1.xattrinfos().find(XATTR_DIR_ENTRIES)->second, "2"); + ASSERT_EQ(xattr1.xattrinfos().find(XATTR_DIR_FBYTES)->second, "100"); } TEST_F(InodeManagerTest, testCreateManageInode) { diff --git a/curvefs/test/metaserver/inode_storage_test.cpp b/curvefs/test/metaserver/inode_storage_test.cpp index 279831ce54..4ea0bc3f54 100644 --- a/curvefs/test/metaserver/inode_storage_test.cpp +++ b/curvefs/test/metaserver/inode_storage_test.cpp @@ -314,30 +314,30 @@ TEST_F(InodeStorageTest, testGetXAttr) { inode.set_mode(777); inode.set_nlink(2); inode.set_type(FsFileType::TYPE_DIRECTORY); - inode.mutable_xattr()->insert({XATTRFILES, "1"}); - inode.mutable_xattr()->insert({XATTRSUBDIRS, "1"}); - inode.mutable_xattr()->insert({XATTRENTRIES, "2"}); - inode.mutable_xattr()->insert({XATTRFBYTES, "100"}); + inode.mutable_xattr()->insert({XATTR_DIR_FILES, "1"}); + inode.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "1"}); + inode.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "2"}); + inode.mutable_xattr()->insert({XATTR_DIR_FBYTES, "100"}); - inode.mutable_xattr()->insert({XATTRRFILES, "100"}); - inode.mutable_xattr()->insert({XATTRRSUBDIRS, "100"}); - inode.mutable_xattr()->insert({XATTRRENTRIES, "200"}); - inode.mutable_xattr()->insert({XATTRRFBYTES, "1000"}); + inode.mutable_xattr()->insert({XATTR_DIR_RFILES, "100"}); + inode.mutable_xattr()->insert({XATTR_DIR_RSUBDIRS, "100"}); + inode.mutable_xattr()->insert({XATTR_DIR_RENTRIES, "200"}); + inode.mutable_xattr()->insert({XATTR_DIR_RFBYTES, "1000"}); ASSERT_EQ(storage.Insert(inode, logIndex_++), MetaStatusCode::OK); XAttr xattr; ASSERT_EQ(storage.GetXAttr(Key4Inode(1, 1), &xattr), MetaStatusCode::OK); ASSERT_FALSE(xattr.xattrinfos().empty()); - ASSERT_EQ(xattr.xattrinfos().find(XATTRFILES)->second, "1"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRSUBDIRS)->second, "1"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRENTRIES)->second, "2"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRFBYTES)->second, "100"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_FILES)->second, "1"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_SUBDIRS)->second, "1"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_ENTRIES)->second, "2"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_FBYTES)->second, "100"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRRFILES)->second, "100"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRRSUBDIRS)->second, "100"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRRENTRIES)->second, "200"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRRFBYTES)->second, "1000"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_RFILES)->second, "100"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_RSUBDIRS)->second, "100"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_RENTRIES)->second, "200"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_RFBYTES)->second, "1000"); } TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) { diff --git a/curvefs/test/metaserver/metastore_test.cpp b/curvefs/test/metaserver/metastore_test.cpp index f8910d046f..750a32fa6e 100644 --- a/curvefs/test/metaserver/metastore_test.cpp +++ b/curvefs/test/metaserver/metastore_test.cpp @@ -1498,10 +1498,10 @@ TEST_F(MetastoreTest, testBatchGetXAttr) { updateRequest.set_partitionid(partitionId); updateRequest.set_fsid(fsId); updateRequest.set_inodeid(inodeId1); - updateRequest.mutable_xattr()->insert({XATTRFILES, "1"}); - updateRequest.mutable_xattr()->insert({XATTRSUBDIRS, "2"}); - updateRequest.mutable_xattr()->insert({XATTRENTRIES, "3"}); - updateRequest.mutable_xattr()->insert({XATTRFBYTES, "100"}); + updateRequest.mutable_xattr()->insert({XATTR_DIR_FILES, "1"}); + updateRequest.mutable_xattr()->insert({XATTR_DIR_SUBDIRS, "2"}); + updateRequest.mutable_xattr()->insert({XATTR_DIR_ENTRIES, "3"}); + updateRequest.mutable_xattr()->insert({XATTR_DIR_FBYTES, "100"}); ret = metastore.UpdateInode(&updateRequest, &updateResponse, logIndex_++); ASSERT_EQ(updateResponse.statuscode(), MetaStatusCode::OK); @@ -1518,48 +1518,40 @@ TEST_F(MetastoreTest, testBatchGetXAttr) { ASSERT_EQ(batchResponse.statuscode(), MetaStatusCode::OK); ASSERT_EQ(batchResponse.xattr_size(), 2); if (batchResponse.xattr(0).inodeid() == inodeId1) { - ASSERT_EQ(batchResponse.xattr(0).xattrinfos().find(XATTRFILES)->second, - "1"); - ASSERT_EQ( - batchResponse.xattr(0).xattrinfos().find(XATTRSUBDIRS)->second, - "2"); - ASSERT_EQ( - batchResponse.xattr(0).xattrinfos().find(XATTRENTRIES)->second, - "3"); - ASSERT_EQ(batchResponse.xattr(0).xattrinfos().find(XATTRFBYTES)->second, - "100"); - ASSERT_EQ(batchResponse.xattr(1).xattrinfos().find(XATTRFILES)->second, - "0"); - ASSERT_EQ( - batchResponse.xattr(1).xattrinfos().find(XATTRSUBDIRS)->second, - "0"); - ASSERT_EQ( - batchResponse.xattr(1).xattrinfos().find(XATTRENTRIES)->second, - "0"); - ASSERT_EQ(batchResponse.xattr(1).xattrinfos().find(XATTRFBYTES)->second, - "0"); + ASSERT_EQ(batchResponse.xattr(0).xattrinfos() + .find(XATTR_DIR_FILES)->second, "1"); + ASSERT_EQ(batchResponse.xattr(0).xattrinfos() + .find(XATTR_DIR_SUBDIRS)->second, "2"); + ASSERT_EQ(batchResponse.xattr(0).xattrinfos() + .find(XATTR_DIR_ENTRIES)->second, "3"); + ASSERT_EQ(batchResponse.xattr(0).xattrinfos() + .find(XATTR_DIR_FBYTES)->second, "100"); + ASSERT_EQ(batchResponse.xattr(1).xattrinfos() + .find(XATTR_DIR_FILES)->second, "0"); + ASSERT_EQ(batchResponse.xattr(1).xattrinfos() + .find(XATTR_DIR_SUBDIRS)->second, "0"); + ASSERT_EQ(batchResponse.xattr(1).xattrinfos() + .find(XATTR_DIR_ENTRIES)->second, "0"); + ASSERT_EQ(batchResponse.xattr(1).xattrinfos() + .find(XATTR_DIR_FBYTES)->second, "0"); } else { - ASSERT_EQ(batchResponse.xattr(1).xattrinfos().find(XATTRFILES)->second, - "1"); - ASSERT_EQ( - batchResponse.xattr(1).xattrinfos().find(XATTRSUBDIRS)->second, - "2"); - ASSERT_EQ( - batchResponse.xattr(1).xattrinfos().find(XATTRENTRIES)->second, - "3"); - ASSERT_EQ(batchResponse.xattr(1).xattrinfos().find(XATTRFBYTES)->second, - "100"); - ASSERT_EQ(batchResponse.xattr(0).xattrinfos().find(XATTRFILES)->second, - "0"); - ASSERT_EQ( - batchResponse.xattr(0).xattrinfos().find(XATTRSUBDIRS)->second, - "0"); - ASSERT_EQ( - batchResponse.xattr(0).xattrinfos().find(XATTRENTRIES)->second, - "0"); - ASSERT_EQ(batchResponse.xattr(0).xattrinfos().find(XATTRFBYTES)->second, - "0"); + ASSERT_EQ(batchResponse.xattr(1).xattrinfos() + .find(XATTR_DIR_FILES)->second, "1"); + ASSERT_EQ(batchResponse.xattr(1).xattrinfos() + .find(XATTR_DIR_SUBDIRS)->second, "2"); + ASSERT_EQ(batchResponse.xattr(1).xattrinfos() + .find(XATTR_DIR_ENTRIES)->second, "3"); + ASSERT_EQ(batchResponse.xattr(1).xattrinfos() + .find(XATTR_DIR_FBYTES)->second, "100"); + ASSERT_EQ(batchResponse.xattr(0).xattrinfos() + .find(XATTR_DIR_FILES)->second, "0"); + ASSERT_EQ(batchResponse.xattr(0).xattrinfos() + .find(XATTR_DIR_SUBDIRS)->second, "0"); + ASSERT_EQ(batchResponse.xattr(0).xattrinfos() + .find(XATTR_DIR_ENTRIES)->second, "0"); + ASSERT_EQ(batchResponse.xattr(0).xattrinfos() + .find(XATTR_DIR_FBYTES)->second, "0"); } } diff --git a/curvefs/test/metaserver/partition_test.cpp b/curvefs/test/metaserver/partition_test.cpp index 7256a136ba..9d29461e39 100644 --- a/curvefs/test/metaserver/partition_test.cpp +++ b/curvefs/test/metaserver/partition_test.cpp @@ -466,10 +466,10 @@ TEST_F(PartitionTest, testGetXAttr) { ASSERT_EQ(partition1.GetXAttr(1, 100, &xattr), MetaStatusCode::OK); ASSERT_EQ(xattr.inodeid(), 100); ASSERT_EQ(xattr.fsid(), 1); - ASSERT_EQ(xattr.xattrinfos().find(XATTRFILES)->second, "0"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRSUBDIRS)->second, "0"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRENTRIES)->second, "0"); - ASSERT_EQ(xattr.xattrinfos().find(XATTRFBYTES)->second, "0"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_FILES)->second, "0"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_SUBDIRS)->second, "0"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_ENTRIES)->second, "0"); + ASSERT_EQ(xattr.xattrinfos().find(XATTR_DIR_FBYTES)->second, "0"); } } // namespace metaserver From d616bd335f4a5f733a31294d3fa1e65b9098c8e4 Mon Sep 17 00:00:00 2001 From: Wine93 Date: Fri, 20 Oct 2023 10:50:50 +0800 Subject: [PATCH 18/27] curvefs/client: now we support hadoop sdk. Signed-off-by: Wine93 --- .clang-format | 1 + .github/workflows/clang-format.yml | 2 +- .gitignore | 8 + .obm.cfg | 2 +- Makefile | 6 + WORKSPACE | 25 +- curvefs/conf/client.conf | 9 + curvefs/sdk/.clang-format | 2 + curvefs/sdk/README.md | 18 + curvefs/sdk/java/native/BUILD | 36 + ..._opencurve_curve_fs_libfs_CurveFSMount.cpp | 688 +++++++++++++++ ...io_opencurve_curve_fs_libfs_CurveFSMount.h | 221 +++++ curvefs/sdk/java/pom.xml | 92 ++ .../fs/flink/CurveFileSystemFactory.java | 48 ++ .../fs/flink/CurveFileSystemTableFactory.java | 10 + .../io/opencurve/curve/fs/hadoop/CurveFS.java | 52 ++ .../curve/fs/hadoop/CurveFSInputStream.java | 251 ++++++ .../curve/fs/hadoop/CurveFSOutputStream.java | 174 ++++ .../curve/fs/hadoop/CurveFSProto.java | 57 ++ .../curve/fs/hadoop/CurveFSTalker.java | 182 ++++ .../curve/fs/hadoop/CurveFileSystem.java | 388 +++++++++ .../io/opencurve/curve/fs/hadoop/Main.java | 7 + .../curve/fs/hadoop/permission/Group.java | 107 +++ .../curve/fs/hadoop/permission/Helper.java | 105 +++ .../fs/hadoop/permission/Permission.java | 66 ++ .../curve/fs/hadoop/permission/User.java | 99 +++ .../curve/fs/libfs/CurveFSMount.java | 228 +++++ .../curve/fs/libfs/CurveFSNativeLoader.java | 106 +++ .../opencurve/curve/fs/libfs/CurveFSStat.java | 50 ++ .../curve/fs/libfs/CurveFSStatVFS.java | 33 + ...org.apache.flink.core.fs.FileSystemFactory | 1 + .../org.apache.flink.table.factories.Factory | 1 + .../opencurve/curve/fs/others/TestOthers.java | 15 + curvefs/sdk/libcurvefs/BUILD | 44 + curvefs/sdk/libcurvefs/README.md | 29 + curvefs/sdk/libcurvefs/examples/Makefile | 24 + curvefs/sdk/libcurvefs/examples/append.c | 48 ++ curvefs/sdk/libcurvefs/examples/chmod.c | 24 + curvefs/sdk/libcurvefs/examples/common.h | 76 ++ curvefs/sdk/libcurvefs/examples/fstat.c | 36 + curvefs/sdk/libcurvefs/examples/ls.c | 49 ++ curvefs/sdk/libcurvefs/examples/mkdir.c | 25 + curvefs/sdk/libcurvefs/examples/read.c | 44 + curvefs/sdk/libcurvefs/examples/rename.c | 24 + curvefs/sdk/libcurvefs/examples/rmdir.c | 25 + curvefs/sdk/libcurvefs/examples/stat.c | 27 + curvefs/sdk/libcurvefs/examples/statfs.c | 29 + curvefs/sdk/libcurvefs/examples/touch.c | 26 + curvefs/sdk/libcurvefs/examples/unlink.c | 25 + curvefs/sdk/libcurvefs/examples/write.c | 42 + curvefs/sdk/libcurvefs/libcurvefs.cpp | 256 ++++++ curvefs/sdk/libcurvefs/libcurvefs.h | 152 ++++ curvefs/src/client/BUILD | 14 + curvefs/src/client/client_operator.cpp | 6 +- curvefs/src/client/common/config.cpp | 37 +- curvefs/src/client/common/config.h | 22 +- curvefs/src/client/curve_fuse_op.cpp | 17 +- curvefs/src/client/filesystem/.clang-format | 26 + curvefs/src/client/filesystem/defer_sync.cpp | 11 + curvefs/src/client/filesystem/defer_sync.h | 2 + curvefs/src/client/filesystem/dir_cache.cpp | 9 + curvefs/src/client/filesystem/dir_cache.h | 2 + curvefs/src/client/filesystem/error.cpp | 20 +- curvefs/src/client/filesystem/error.h | 18 +- curvefs/src/client/filesystem/filesystem.cpp | 27 +- curvefs/src/client/filesystem/filesystem.h | 28 +- .../src/client/filesystem/lookup_cache.cpp | 2 +- curvefs/src/client/filesystem/meta.cpp | 4 + curvefs/src/client/filesystem/meta.h | 2 + curvefs/src/client/filesystem/rpc_client.cpp | 2 +- curvefs/src/client/filesystem/utils.cpp | 2 +- curvefs/src/client/filesystem/utils.h | 2 +- curvefs/src/client/fuse_client.cpp | 141 ++- curvefs/src/client/fuse_client.h | 22 +- curvefs/src/client/fuse_common.h | 4 +- curvefs/src/client/fuse_s3_client.cpp | 16 +- curvefs/src/client/fuse_volume_client.cpp | 4 +- curvefs/src/client/inode_cache_manager.cpp | 2 +- curvefs/src/client/logger/BUILD | 32 + curvefs/src/client/logger/access_log.cpp | 46 + .../{filesystem => logger}/access_log.h | 24 +- curvefs/src/client/logger/error_log.h | 61 ++ curvefs/src/client/metric/client_metric.h | 1 + curvefs/src/client/s3/client_s3_adaptor.cpp | 2 +- .../src/client/s3/client_s3_cache_manager.cpp | 4 +- curvefs/src/client/sdk_helper.cpp | 169 ++++ curvefs/src/client/sdk_helper.h | 87 ++ curvefs/src/client/vfs/.clang-format | 26 + curvefs/src/client/vfs/BUILD | 32 + curvefs/src/client/vfs/README.md | 21 + curvefs/src/client/vfs/cache.cpp | 126 +++ curvefs/src/client/vfs/cache.h | 102 +++ curvefs/src/client/vfs/config.cpp | 107 +++ curvefs/src/client/vfs/config.h | 63 ++ curvefs/src/client/vfs/handlers.cpp | 60 ++ curvefs/src/client/vfs/handlers.h | 69 ++ curvefs/src/client/vfs/meta.h | 65 ++ curvefs/src/client/vfs/operations.cpp | 243 ++++++ curvefs/src/client/vfs/operations.h | 208 +++++ curvefs/src/client/vfs/permission.cpp | 113 +++ curvefs/src/client/vfs/permission.h | 77 ++ curvefs/src/client/vfs/typedef.h | 87 ++ curvefs/src/client/vfs/utils.cpp | 115 +++ curvefs/src/client/vfs/utils.h | 67 ++ curvefs/src/client/vfs/vfs.cpp | 804 ++++++++++++++++++ curvefs/src/client/vfs/vfs.h | 157 ++++ curvefs/src/client/warmup/warmup_manager.cpp | 2 +- curvefs/src/client/xattr_manager.cpp | 99 ++- curvefs/src/common/define.h | 10 +- curvefs/src/metaserver/inode_manager.cpp | 8 +- curvefs/src/metaserver/metacli_manager.cpp | 2 + curvefs/src/metaserver/metacli_manager.h | 3 + .../space/inode_volume_space_deallocate.cpp | 6 +- .../space/volume_deallocate_worker.cpp | 6 +- curvefs/test/client/client_operator_test.cpp | 2 +- .../client/client_s3_adaptor_Integration.cpp | 11 +- .../test/client/file_cache_manager_test.cpp | 2 +- curvefs/test/client/filesystem/.clang-format | 26 + curvefs/test/client/filesystem/BUILD | 9 +- curvefs/test/client/filesystem/error_test.cpp | 6 +- .../client/filesystem/filesystem_test.cpp | 43 +- curvefs/test/client/filesystem/helper/BUILD | 31 + .../test/client/filesystem/helper/builder.h | 2 + .../client/filesystem/rpc_client_test.cpp | 20 +- curvefs/test/client/filesystem/utils_test.cpp | 10 +- curvefs/test/client/fs_cache_manager_test.cpp | 2 +- .../test/client/test_dentry_cache_manager.cpp | 2 +- curvefs/test/client/test_fuse_s3_client.cpp | 162 ++-- .../test/client/test_fuse_volume_client.cpp | 18 +- curvefs/test/client/test_inodeWrapper.cpp | 2 +- .../test/client/test_inode_cache_manager.cpp | 8 +- curvefs/test/client/vfs/.clang-format | 26 + curvefs/test/client/vfs/BUILD | 30 + curvefs/test/client/vfs/attr_cache_test.cpp | 333 ++++++++ curvefs/test/client/vfs/config_test.cpp | 119 +++ curvefs/test/client/vfs/entry_cache_test.cpp | 327 +++++++ curvefs/test/client/vfs/handlers_test.cpp | 100 +++ curvefs/test/client/vfs/helper/BUILD | 33 + curvefs/test/client/vfs/helper/builder.h | 115 +++ curvefs/test/client/vfs/helper/expect.h | 127 +++ curvefs/test/client/vfs/helper/helper.h | 46 + .../test/client/vfs/helper/mem_operations.cpp | 419 +++++++++ .../test/client/vfs/helper/mem_operations.h | 159 ++++ .../test/client/vfs/helper/mock_fuse_client.h | 212 +++++ curvefs/test/client/vfs/meta_test.cpp | 57 ++ curvefs/test/client/vfs/operations_test.cpp | 537 ++++++++++++ curvefs/test/client/vfs/permission_test.cpp | 255 ++++++ curvefs/test/client/vfs/utils_test.cpp | 205 +++++ curvefs/test/client/vfs/vfs_test.cpp | 435 ++++++++++ curvefs/test/client/volume/BUILD.bazel | 1 + .../volume/default_volume_storage_test.cpp | 10 +- curvefs/test/metaserver/BUILD | 1 + curvefs/test/metaserver/metastore_test.cpp | 81 +- thirdparties/fmt.BUILD | 2 +- thirdparties/glog/glog.patch | 13 +- thirdparties/incbin.BUILD | 22 + thirdparties/jni.BUILD | 26 + thirdparties/spdlog.BUILD | 2 +- util/build_in_image.sh | 2 + util/init-hadoop.sh | 83 ++ util/playground.sh | 25 +- util/sdk.sh | 74 ++ 162 files changed, 11420 insertions(+), 454 deletions(-) create mode 100644 curvefs/sdk/.clang-format create mode 100644 curvefs/sdk/README.md create mode 100644 curvefs/sdk/java/native/BUILD create mode 100644 curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.cpp create mode 100644 curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.h create mode 100644 curvefs/sdk/java/pom.xml create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemFactory.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemTableFactory.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFS.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSInputStream.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSOutputStream.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSProto.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSTalker.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/Main.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Group.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Helper.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Permission.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/User.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSMount.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSNativeLoader.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStat.java create mode 100644 curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStatVFS.java create mode 100644 curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory create mode 100644 curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory create mode 100644 curvefs/sdk/java/src/test/java/io/opencurve/curve/fs/others/TestOthers.java create mode 100644 curvefs/sdk/libcurvefs/BUILD create mode 100644 curvefs/sdk/libcurvefs/README.md create mode 100644 curvefs/sdk/libcurvefs/examples/Makefile create mode 100644 curvefs/sdk/libcurvefs/examples/append.c create mode 100644 curvefs/sdk/libcurvefs/examples/chmod.c create mode 100644 curvefs/sdk/libcurvefs/examples/common.h create mode 100644 curvefs/sdk/libcurvefs/examples/fstat.c create mode 100644 curvefs/sdk/libcurvefs/examples/ls.c create mode 100644 curvefs/sdk/libcurvefs/examples/mkdir.c create mode 100644 curvefs/sdk/libcurvefs/examples/read.c create mode 100644 curvefs/sdk/libcurvefs/examples/rename.c create mode 100644 curvefs/sdk/libcurvefs/examples/rmdir.c create mode 100644 curvefs/sdk/libcurvefs/examples/stat.c create mode 100644 curvefs/sdk/libcurvefs/examples/statfs.c create mode 100644 curvefs/sdk/libcurvefs/examples/touch.c create mode 100644 curvefs/sdk/libcurvefs/examples/unlink.c create mode 100644 curvefs/sdk/libcurvefs/examples/write.c create mode 100644 curvefs/sdk/libcurvefs/libcurvefs.cpp create mode 100644 curvefs/sdk/libcurvefs/libcurvefs.h create mode 100644 curvefs/src/client/filesystem/.clang-format create mode 100644 curvefs/src/client/logger/BUILD create mode 100644 curvefs/src/client/logger/access_log.cpp rename curvefs/src/client/{filesystem => logger}/access_log.h (75%) create mode 100644 curvefs/src/client/logger/error_log.h create mode 100644 curvefs/src/client/sdk_helper.cpp create mode 100644 curvefs/src/client/sdk_helper.h create mode 100644 curvefs/src/client/vfs/.clang-format create mode 100644 curvefs/src/client/vfs/BUILD create mode 100644 curvefs/src/client/vfs/README.md create mode 100644 curvefs/src/client/vfs/cache.cpp create mode 100644 curvefs/src/client/vfs/cache.h create mode 100644 curvefs/src/client/vfs/config.cpp create mode 100644 curvefs/src/client/vfs/config.h create mode 100644 curvefs/src/client/vfs/handlers.cpp create mode 100644 curvefs/src/client/vfs/handlers.h create mode 100644 curvefs/src/client/vfs/meta.h create mode 100644 curvefs/src/client/vfs/operations.cpp create mode 100644 curvefs/src/client/vfs/operations.h create mode 100644 curvefs/src/client/vfs/permission.cpp create mode 100644 curvefs/src/client/vfs/permission.h create mode 100644 curvefs/src/client/vfs/typedef.h create mode 100644 curvefs/src/client/vfs/utils.cpp create mode 100644 curvefs/src/client/vfs/utils.h create mode 100644 curvefs/src/client/vfs/vfs.cpp create mode 100644 curvefs/src/client/vfs/vfs.h create mode 100644 curvefs/test/client/filesystem/.clang-format create mode 100644 curvefs/test/client/filesystem/helper/BUILD create mode 100644 curvefs/test/client/vfs/.clang-format create mode 100644 curvefs/test/client/vfs/BUILD create mode 100644 curvefs/test/client/vfs/attr_cache_test.cpp create mode 100644 curvefs/test/client/vfs/config_test.cpp create mode 100644 curvefs/test/client/vfs/entry_cache_test.cpp create mode 100644 curvefs/test/client/vfs/handlers_test.cpp create mode 100644 curvefs/test/client/vfs/helper/BUILD create mode 100644 curvefs/test/client/vfs/helper/builder.h create mode 100644 curvefs/test/client/vfs/helper/expect.h create mode 100644 curvefs/test/client/vfs/helper/helper.h create mode 100644 curvefs/test/client/vfs/helper/mem_operations.cpp create mode 100644 curvefs/test/client/vfs/helper/mem_operations.h create mode 100644 curvefs/test/client/vfs/helper/mock_fuse_client.h create mode 100644 curvefs/test/client/vfs/meta_test.cpp create mode 100644 curvefs/test/client/vfs/operations_test.cpp create mode 100644 curvefs/test/client/vfs/permission_test.cpp create mode 100644 curvefs/test/client/vfs/utils_test.cpp create mode 100644 curvefs/test/client/vfs/vfs_test.cpp create mode 100644 thirdparties/incbin.BUILD create mode 100644 thirdparties/jni.BUILD create mode 100644 util/init-hadoop.sh create mode 100644 util/sdk.sh diff --git a/.clang-format b/.clang-format index fa13380a3d..da0d029bcd 100644 --- a/.clang-format +++ b/.clang-format @@ -4,6 +4,7 @@ BasedOnStyle: Google AccessModifierOffset: -3 DerivePointerAlignment: false IndentWidth: 4 +AlignAfterOpenBracket: DontAlign --- Language: Proto BasedOnStyle: Google diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index b0ceec4da5..fa4c88720e 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -14,7 +14,7 @@ jobs: # fetch everything to be able to compare with any ref fetch-depth: 0 - - name: Check + - name: Check env: LANG: "C.UTF-8" LC_ALL: "C.UTF-8" diff --git a/.gitignore b/.gitignore index 208afe07e7..92dc59bcf5 100755 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,11 @@ test/integration/snapshotcloneserver/config/*.conf *.deb *.whl + +*.class +curvefs/sdk/java/target/ +curvefs/sdk/java/native/build +curvefs/sdk/libcurvefs/examples/bin +curvefs/sdk/output/ +hadoop-test/ +nnbench-test/ diff --git a/.obm.cfg b/.obm.cfg index 012870a5fd..ada426f8e5 100644 --- a/.obm.cfg +++ b/.obm.cfg @@ -1,2 +1,2 @@ -container_name: curve-build-playground-master +container_name: curve-build-playground.master container_image: opencurvedocker/curve-base:build-debian11 diff --git a/Makefile b/Makefile index c71b5ab045..749e838ba2 100644 --- a/Makefile +++ b/Makefile @@ -124,3 +124,9 @@ docker: format: @bash util/format.sh $(commit_id) + +init-hadoop: + @bash util/init-hadoop.sh + +sdk: + @bash util/sdk.sh diff --git a/WORKSPACE b/WORKSPACE index 70772b05f5..a423f1c46a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -142,7 +142,7 @@ git_repository( patches = [ "//:thirdparties/brpc/brpc.patch", "//:thirdparties/brpc/fix-gcc11.patch", - "//:thirdparties/brpc/0001-bvar-warning-on-conflict-bvar-name.patch", + "//:thirdparties/brpc/0001-bvar-warning-on-conflict-bvar-name.patch", ], patch_args = ["-p1"], ) @@ -192,7 +192,6 @@ new_local_repository( path = "thirdparties/memcache/libmemcached-1.1.2", ) - http_archive( name = "aws", urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"], @@ -259,6 +258,21 @@ http_archive( build_file = "//:thirdparties/spdlog.BUILD", ) +# incbin +new_git_repository( + name = "incbin", + remote = "https://github.com/graphitemaster/incbin.git", + commit = "6e576cae5ab5810f25e2631f2e0b80cbe7dc8cbf", + build_file = "//:thirdparties/incbin.BUILD", +) + +# config +new_local_repository( + name = "config", + build_file = "//:thirdparties/config.BUILD", + path = "thirdparties/config", +) + # Bazel platform rules. http_archive( name = "platforms", @@ -274,6 +288,13 @@ new_local_repository( path = "thirdparties/rocksdb", ) +# jni +new_local_repository( + name = "jni", + build_file = "//:thirdparties/jni.BUILD", + path = "thirdparties", +) + # Hedron's Compile Commands Extractor for Bazel # https://github.com/hedronvision/bazel-compile-commands-extractor http_archive( diff --git a/curvefs/conf/client.conf b/curvefs/conf/client.conf index 9b3d449ec9..755d624920 100644 --- a/curvefs/conf/client.conf +++ b/curvefs/conf/client.conf @@ -126,6 +126,15 @@ fuseClient.throttle.burstReadIops=0 # the times that read burst Iops can continue, default 180s fuseClient.throttle.burstReadIopsSecs=180 +#### vfs (virtual filesystem) +#{ +vfs.userPermission.uid=0 +vfs.userPermission.gids=0 +vfs.userPermission.umask=0022 +vfs.entryCache.lruSize=2000000 +vfs.attrCache.lruSize=2000000 +#} + #### filesystem metadata # { # fs.disableXAttr: diff --git a/curvefs/sdk/.clang-format b/curvefs/sdk/.clang-format new file mode 100644 index 0000000000..47a38a93f2 --- /dev/null +++ b/curvefs/sdk/.clang-format @@ -0,0 +1,2 @@ +DisableFormat: true +SortIncludes: Never diff --git a/curvefs/sdk/README.md b/curvefs/sdk/README.md new file mode 100644 index 0000000000..c58aa8feba --- /dev/null +++ b/curvefs/sdk/README.md @@ -0,0 +1,18 @@ +Hadoop SDK +=== + +How to build +--- + +``` bash +$ git clone git@github.com:opencurve/curve.git +$ cd curve +$ make dep stor=fs +$ make sdk +``` + +It will generate a jar after build success: + +``` +Build SDK success => /curve/curvefs/sdk/output/curvefs-hadoop-1.0-SNAPSHOT.jar +``` diff --git a/curvefs/sdk/java/native/BUILD b/curvefs/sdk/java/native/BUILD new file mode 100644 index 0000000000..3361aeea44 --- /dev/null +++ b/curvefs/sdk/java/native/BUILD @@ -0,0 +1,36 @@ +# +# Copyright (c) 2023 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_DEFAULT_COPTS") + +cc_binary( + name = "curvefs_jni", + srcs = glob([ + "*.h", + "*.cpp", + ]), + visibility = ["//visibility:public"], + copts = CURVE_DEFAULT_COPTS, + linkopts = [ + "-Wl,-rpath=/tmp/libcurvefs,--disable-new-dtags", + ], + deps = [ + "@com_google_absl//absl/cleanup", + "@jni//:copy_jni_hdr_lib", + "//curvefs/sdk/libcurvefs:curvefs_lib", + ], + linkshared = True, +) diff --git a/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.cpp b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.cpp new file mode 100644 index 0000000000..d9c0abed81 --- /dev/null +++ b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.cpp @@ -0,0 +1,688 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-08 + * Author: Jingli Chen (Wine93) + */ + +#include +#include + +#include "absl/cleanup/cleanup.h" +#include "curvefs/sdk/libcurvefs/libcurvefs.h" +#include "curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.h" + +/* Cached field IDs for io.opencurve.curve.fs.CurveStat */ +static jfieldID curvestat_mode_fid; +static jfieldID curvestat_uid_fid; +static jfieldID curvestat_gid_fid; +static jfieldID curvestat_size_fid; +static jfieldID curvestat_blksize_fid; +static jfieldID curvestat_blocks_fid; +static jfieldID curvestat_a_time_fid; +static jfieldID curvestat_m_time_fid; +static jfieldID curvestat_is_file_fid; +static jfieldID curvestat_is_directory_fid; +static jfieldID curvestat_is_symlink_fid; + +/* Cached field IDs for io.opencurve.curve.fs.CurveStatVFS */ +static jfieldID curvestatvfs_bsize_fid; +static jfieldID curvestatvfs_frsize_fid; +static jfieldID curvestatvfs_blocks_fid; +static jfieldID curvestatvfs_bavail_fid; +static jfieldID curvestatvfs_files_fid; +static jfieldID curvestatvfs_fsid_fid; +static jfieldID curvestatvfs_namemax_fid; + +/* + * Setup cached field IDs + */ +static void setup_field_ids(JNIEnv* env) { + jclass curvestat_cls; + jclass curvestatvfs_cls; + +/* + * Get a fieldID from a class with a specific type + * + * clz: jclass + * field: field in clz + * type: integer, long, etc.. + * + * This macro assumes some naming convention that is used + * only in this file: + * + * GETFID(curvestat, mode, I) gets translated into + * curvestat_mode_fid = env->GetFieldID(curvestat_cls, "mode", "I"); + */ +#define GETFID(clz, field, type) do { \ + clz ## _ ## field ## _fid = env->GetFieldID(clz ## _cls, #field, #type); \ + if (!clz ## _ ## field ## _fid) \ + return; \ + } while (0) + + /* Cache CurveStat fields */ + + curvestat_cls = env->FindClass("io/opencurve/curve/fs/libfs/CurveFSStat"); + if (!curvestat_cls) { + return; + } + + GETFID(curvestat, mode, I); + GETFID(curvestat, uid, I); + GETFID(curvestat, gid, I); + GETFID(curvestat, size, J); + GETFID(curvestat, blksize, J); + GETFID(curvestat, blocks, J); + GETFID(curvestat, a_time, J); + GETFID(curvestat, m_time, J); + GETFID(curvestat, is_file, Z); + GETFID(curvestat, is_directory, Z); + GETFID(curvestat, is_symlink, Z); + + /* Cache CurveStatVFS fields */ + + curvestatvfs_cls = + env->FindClass("io/opencurve/curve/fs/libfs/CurveFSStatVFS"); + if (!curvestatvfs_cls) { + return; + } + + GETFID(curvestatvfs, bsize, J); + GETFID(curvestatvfs, frsize, J); + GETFID(curvestatvfs, blocks, J); + GETFID(curvestatvfs, bavail, J); + GETFID(curvestatvfs, files, J); + GETFID(curvestatvfs, fsid, J); + GETFID(curvestatvfs, namemax, J); + +#undef GETFID +} + +static void fill_curvestat(JNIEnv* env, + jobject j_curvestat, + struct stat* stat) { + env->SetIntField(j_curvestat, curvestat_mode_fid, stat->st_mode); + env->SetIntField(j_curvestat, curvestat_uid_fid, stat->st_uid); + env->SetIntField(j_curvestat, curvestat_gid_fid, stat->st_gid); + env->SetLongField(j_curvestat, curvestat_size_fid, stat->st_size); + env->SetLongField(j_curvestat, curvestat_blksize_fid, stat->st_blksize); + env->SetLongField(j_curvestat, curvestat_blocks_fid, stat->st_blocks); + + // mtime + uint64_t time = stat->st_mtim.tv_sec; + time *= 1000; + time += stat->st_mtim.tv_nsec / 1000000; + env->SetLongField(j_curvestat, curvestat_m_time_fid, time); + + // atime + time = stat->st_atim.tv_sec; + time *= 1000; + time += stat->st_atim.tv_nsec / 1000000; + env->SetLongField(j_curvestat, curvestat_a_time_fid, time); + + env->SetBooleanField(j_curvestat, curvestat_is_file_fid, + S_ISREG(stat->st_mode) ? JNI_TRUE : JNI_FALSE); + + env->SetBooleanField(j_curvestat, curvestat_is_directory_fid, + S_ISDIR(stat->st_mode) ? JNI_TRUE : JNI_FALSE); + + env->SetBooleanField(j_curvestat, curvestat_is_symlink_fid, + S_ISLNK(stat->st_mode) ? JNI_TRUE : JNI_FALSE); +} + +static void fill_curvestatvfs(JNIEnv* env, + jobject j_curvestatvfs, + struct statvfs st) { + env->SetLongField(j_curvestatvfs, curvestatvfs_bsize_fid, st.f_bsize); + env->SetLongField(j_curvestatvfs, curvestatvfs_frsize_fid, st.f_frsize); + env->SetLongField(j_curvestatvfs, curvestatvfs_blocks_fid, st.f_blocks); + env->SetLongField(j_curvestatvfs, curvestatvfs_bavail_fid, st.f_bavail); + env->SetLongField(j_curvestatvfs, curvestatvfs_files_fid, st.f_files); + env->SetLongField(j_curvestatvfs, curvestatvfs_fsid_fid, st.f_fsid); + env->SetLongField(j_curvestatvfs, curvestatvfs_namemax_fid, st.f_namemax); +} + +/* Map io_opencurve_curve_fs_libfs_CurveFSMount_O_* open flags to values in libc */ +static inline uint32_t fixup_open_flags(jint jflags) { + uint32_t flags = 0; + +#define FIXUP_OPEN_FLAG(name) \ + if (jflags & io_opencurve_curve_fs_libfs_CurveFSMount_##name) \ + flags |= name; + + FIXUP_OPEN_FLAG(O_RDONLY) + FIXUP_OPEN_FLAG(O_RDWR) + FIXUP_OPEN_FLAG(O_APPEND) + FIXUP_OPEN_FLAG(O_CREAT) + FIXUP_OPEN_FLAG(O_TRUNC) + FIXUP_OPEN_FLAG(O_EXCL) + FIXUP_OPEN_FLAG(O_WRONLY) + FIXUP_OPEN_FLAG(O_DIRECTORY) + +#undef FIXUP_OPEN_FLAG + + return flags; +} + +#define CURVEFS_SETATTR_MODE (1 << 0) +#define CURVEFS_SETATTR_UID (1 << 1) +#define CURVEFS_SETATTR_GID (1 << 2) +#define CURVEFS_SETATTR_SIZE (1 << 3) +#define CURVEFS_SETATTR_ATIME (1 << 4) +#define CURVEFS_SETATTR_MTIME (1 << 5) +#define CURVEFS_SETATTR_ATIME_NOW (1 << 7) +#define CURVEFS_SETATTR_MTIME_NOW (1 << 8) +#define CURVEFS_SETATTR_CTIME (1 << 10) + +/* Map JAVA_SETATTR_* to values in curve lib */ +static inline int fixup_attr_mask(jint jmask) { + int mask = 0; + +#define FIXUP_ATTR_MASK(name) \ + if (jmask & io_opencurve_curve_fs_libfs_CurveFSMount_##name) \ + mask |= CURVEFS_##name; + + FIXUP_ATTR_MASK(SETATTR_MODE) + FIXUP_ATTR_MASK(SETATTR_UID) + FIXUP_ATTR_MASK(SETATTR_GID) + FIXUP_ATTR_MASK(SETATTR_MTIME) + FIXUP_ATTR_MASK(SETATTR_ATIME) + +#undef FIXUP_ATTR_MASK + return mask; +} + +/* + * Exception throwing helper. Adapted from Apache Hadoop header + * org_apache_hadoop.h by adding the do {} while (0) construct. + */ +#define THROW(env, exception_name, message) \ + do { \ + jclass ecls = env->FindClass(exception_name); \ + if (ecls) { \ + int ret = env->ThrowNew(ecls, message); \ + if (ret < 0) { \ + printf("(CurveFS) Fatal Error\n"); \ + } \ + env->DeleteLocalRef(ecls); \ + } \ + } while (0) + +static void handle_error(JNIEnv* env, int rc) { + switch (rc) { + case ENOENT: + THROW(env, "java/io/FileNotFoundException", ""); + return; + case EEXIST: + THROW(env, "org/apache/hadoop/fs/FileAlreadyExistsException", ""); + return; + case ENOTDIR: + THROW(env, "org/apache/hadoop/fs/ParentNotDirectoryException", ""); + return; + default: + break; + } + + THROW(env, "java/io/IOException", strerror(rc)); +} + +// nativeCurveFSCreate: curvefs_create +JNIEXPORT jlong +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSCreate + (JNIEnv* env, jobject) { + setup_field_ids(env); + uintptr_t instance = curvefs_create(); + return reinterpret_cast(instance); +} + +// nativeCurveFSRelease: curvefs_release +JNIEXPORT void +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRelease + (JNIEnv* env, jobject, jlong j_instance) { + uintptr_t instance = static_cast(j_instance); + return curvefs_release(instance); +} + +// nativeCurveFSConfSet: curvefs_conf_set +JNIEXPORT void +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSConfSet + (JNIEnv* env, jclass, jlong j_instance, jstring j_key, jstring j_value) { + uintptr_t instance = static_cast(j_instance); + const char* key = env->GetStringUTFChars(j_key, NULL); + const char* value = env->GetStringUTFChars(j_value, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_key, key); + env->ReleaseStringUTFChars(j_value, value); + }); + + return curvefs_conf_set(instance, key, value); +} + +// nativeCurveFSMount: curvefs_mount +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMount + (JNIEnv* env, jclass, jlong j_instance, + jstring j_fsname, jstring j_mountpoint) { + uintptr_t instance = static_cast(j_instance); + const char* fsname = env->GetStringUTFChars(j_fsname, NULL); + const char* mountpoint = env->GetStringUTFChars(j_mountpoint, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_fsname, fsname); + env->ReleaseStringUTFChars(j_mountpoint, mountpoint); + }); + + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSUmount: curvefs_umount +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUmount + (JNIEnv* env, jclass, jlong j_instance, + jstring j_fsname, jstring j_mountpoint) { + uintptr_t instance = static_cast(j_instance); + const char* fsname = env->GetStringUTFChars(j_fsname, NULL); + const char* mountpoint = env->GetStringUTFChars(j_mountpoint, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_fsname, fsname); + env->ReleaseStringUTFChars(j_mountpoint, mountpoint); + }); + + int rc = curvefs_umonut(instance, fsname, mountpoint); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSMkDirs: curvefs_mkdirs +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMkDirs + (JNIEnv* env, jclass, jlong j_instance, jstring j_path, jint j_mode) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + uint16_t mode = static_cast(j_mode); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_mkdirs(instance, path, mode); + if (rc == EEXIST) { + rc = 0; + } else if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSRmDir: curvefs_rmdir +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRmDir + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_rmdir(instance, path); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSListDir: curvefs_opendir/curvefs_readdir/curvefs_closedir +JNIEXPORT jobjectArray +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSListDir + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + // curvefs_opendir + dir_stream_t dir_stream; + auto rc = curvefs_opendir(instance, path, &dir_stream); + if (rc != 0) { + handle_error(env, rc); + return NULL; + } + + // curvefs_readdir + std::vector dirents; + dirent_t dirent; + for ( ;; ) { + ssize_t n = curvefs_readdir(instance, &dir_stream, &dirent); + if (n < 0) { + handle_error(env, rc); + return NULL; + } else if (n == 0) { + break; + } + dirents.push_back(dirent); + } + + // closedir + rc = curvefs_closedir(instance, &dir_stream); + if (rc != 0) { + handle_error(env, rc); + return NULL; + } + + // extract entry name + jobjectArray j_names = env->NewObjectArray( + dirents.size(), env->FindClass("java/lang/String"), NULL); + + for (int i = 0; i < dirents.size(); i++) { + jstring j_name = env->NewStringUTF(dirents[i].name); + env->SetObjectArrayElement(j_names, i, j_name); + env->DeleteLocalRef(j_name); + } + return j_names; +} + +// nativeCurveFSOpen: curvefs_open +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSOpen + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jint j_flags, jint j_mode) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + uint32_t flags = fixup_open_flags(j_flags); + uint16_t mode = static_cast(j_mode); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int fd = curvefs_open(instance, path, flags, mode); + if (fd < 0) { + handle_error(env, fd); + } + return fd; +} + +// nativeCurveFSLSeek: curvefs_lseek +JNIEXPORT jlong +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLSeek + (JNIEnv* env, jclass, + jlong j_instance, jint j_fd, jlong j_offset, jint j_whence) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + uint64_t offset = static_cast(j_offset); + + int whence; + switch (j_whence) { + case io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_SET: + whence = SEEK_SET; + break; + case io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_CUR: + whence = SEEK_CUR; + break; + case io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_END: + whence = SEEK_END; + break; + default: + return -1; + } + + int rc = curvefs_lseek(instance, fd, offset, whence); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativieCurveFSRead: curvefs_read +JNIEXPORT jlong +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSRead + (JNIEnv* env, jclass, jlong j_instance, jint j_fd, + jbyteArray j_buffer, jlong j_size, jlong j_offset) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + jbyte* c_buffer = env->GetByteArrayElements(j_buffer, NULL); + char* buffer = reinterpret_cast(c_buffer); + size_t count = static_cast(j_size); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseByteArrayElements(j_buffer, c_buffer, 0); + }); + + ssize_t n = curvefs_read(instance, fd, buffer, count); + if (n < 0) { + handle_error(env, n); + } + return static_cast(n); +} + +// nativieCurveFSWrite: curvefs_write +JNIEXPORT jlong +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSWrite + (JNIEnv* env, jclass, jlong j_instance, jint j_fd, + jbyteArray j_buffer, jlong j_size, jlong j_offset) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + jbyte* c_buffer = env->GetByteArrayElements(j_buffer, NULL); + char* buffer = reinterpret_cast(c_buffer); + size_t count = static_cast(j_size); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseByteArrayElements(j_buffer, c_buffer, 0); + }); + + ssize_t n = curvefs_write(instance, fd, buffer, count); + if (n < 0) { + handle_error(env, n); + } + return static_cast(n); +} + +// nativeCurveFSFSync: curvefs_fsync +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFSync + (JNIEnv* env, jclass, jlong j_instance, jint j_fd) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + + int rc = curvefs_fsync(instance, fd); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSClose: curvefs_close +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSClose + (JNIEnv* env, jclass, jlong j_instance, jint j_fd) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + + int rc = curvefs_close(instance, fd); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSUnlink: curvefs_unlink +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUnlink + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_unlink(instance, path); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSStatFs: curvefs_statfs +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSStatFs + (JNIEnv* env, jclass, + jlong j_instance, jobject j_curvestatvfs) { + uintptr_t instance = static_cast(j_instance); + + struct statvfs statvfs; + int rc = curvefs_statfs(instance, &statvfs); + if (rc != 0) { + handle_error(env, rc); + return rc; + } + + fill_curvestatvfs(env, j_curvestatvfs, statvfs); + return rc; +} + +// nativeCurveFSLstat: curvefs_lstat +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLstat + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jobject j_curvestat) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + // curvefs_lstat + struct stat stat; + auto rc = curvefs_lstat(instance, path, &stat); + if (rc != 0) { + handle_error(env, rc); + return rc; + } + + fill_curvestat(env, j_curvestat, &stat); + return rc; +} + +// nativeCurveFSFStat: curvefs_fstat +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFStat + (JNIEnv* env, jclass, jlong j_instance, jint j_fd, jobject j_curvestat) { + uintptr_t instance = static_cast(j_instance); + int fd = static_cast(j_fd); + + // curvefs_fstat + struct stat stat; + auto rc = curvefs_fstat(instance, fd, &stat); + if (rc != 0) { + handle_error(env, rc); + return rc; + } + + fill_curvestat(env, j_curvestat, &stat); + return rc; +} + +// nativeCurveFSSetAttr: curvefs_setattr +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSSetAttr + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jobject j_curvestat, jint j_mask) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + int to_set = fixup_attr_mask(j_mask); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + struct stat stat; + memset(&stat, 0, sizeof(stat)); + stat.st_mode = env->GetIntField(j_curvestat, curvestat_mode_fid); + stat.st_uid = env->GetIntField(j_curvestat, curvestat_uid_fid); + stat.st_gid = env->GetIntField(j_curvestat, curvestat_gid_fid); + uint64_t mtime_msec = env->GetLongField(j_curvestat, curvestat_m_time_fid); + uint64_t atime_msec = env->GetLongField(j_curvestat, curvestat_a_time_fid); + stat.st_mtim.tv_sec = mtime_msec / 1000; + stat.st_mtim.tv_nsec = (mtime_msec % 1000) * 1000000; + stat.st_atim.tv_sec = atime_msec / 1000; + stat.st_atim.tv_nsec = (atime_msec % 1000) * 1000000; + + int rc = curvefs_setattr(instance, path, &stat, to_set); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSChmod: curvefs_chmod +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChmod + (JNIEnv* env, jclass, jlong j_instance, jstring j_path, jint j_mode) { + uintptr_t instance = static_cast(j_instance); + uint16_t mode = static_cast(j_mode); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_chmod(instance, path, mode); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSChown: curvefs_chown +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChown + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jint j_uid, jint j_gid) { + uintptr_t instance = static_cast(j_instance); + uint32_t uid = static_cast(j_uid); + uint32_t gid = static_cast(j_gid); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_chown(instance, path, uid, gid); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFSRename: curvefs_rename +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRename + (JNIEnv* env, jclass, jlong j_instance, jstring j_src, jstring j_dst) { + uintptr_t instance = static_cast(j_instance); + const char* src = env->GetStringUTFChars(j_src, NULL); + const char* dst = env->GetStringUTFChars(j_dst, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_src, src); + env->ReleaseStringUTFChars(j_dst, dst); + }); + + int rc = curvefs_rename(instance, src, dst); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} diff --git a/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.h b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.h new file mode 100644 index 0000000000..05d0a21e3f --- /dev/null +++ b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.h @@ -0,0 +1,221 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class io_opencurve_curve_fs_libfs_CurveFSMount */ + +#ifndef _Included_io_opencurve_curve_fs_libfs_CurveFSMount +#define _Included_io_opencurve_curve_fs_libfs_CurveFSMount +#ifdef __cplusplus +extern "C" { +#endif +#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_RDONLY +#define io_opencurve_curve_fs_libfs_CurveFSMount_O_RDONLY 1L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_RDWR +#define io_opencurve_curve_fs_libfs_CurveFSMount_O_RDWR 2L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_APPEND +#define io_opencurve_curve_fs_libfs_CurveFSMount_O_APPEND 4L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_CREAT +#define io_opencurve_curve_fs_libfs_CurveFSMount_O_CREAT 8L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_TRUNC +#define io_opencurve_curve_fs_libfs_CurveFSMount_O_TRUNC 16L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_EXCL +#define io_opencurve_curve_fs_libfs_CurveFSMount_O_EXCL 32L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_WRONLY +#define io_opencurve_curve_fs_libfs_CurveFSMount_O_WRONLY 64L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_DIRECTORY +#define io_opencurve_curve_fs_libfs_CurveFSMount_O_DIRECTORY 128L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_SET +#define io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_SET 0L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_CUR +#define io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_CUR 1L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_END +#define io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_END 2L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_MODE +#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_MODE 1L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_UID +#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_UID 2L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_GID +#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_GID 4L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_MTIME +#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_MTIME 8L +#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_ATIME +#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_ATIME 16L +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSCreate + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSCreate + (JNIEnv *, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSRelease + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRelease + (JNIEnv *, jobject, jlong); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSConfSet + * Signature: (JLjava/lang/String;Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSConfSet + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSMount + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMount + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSUmount + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUmount + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSMkDirs + * Signature: (JLjava/lang/String;I)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMkDirs + (JNIEnv *, jclass, jlong, jstring, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSRmDir + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRmDir + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSListDir + * Signature: (JLjava/lang/String;)[Ljava/lang/String; + */ +JNIEXPORT jobjectArray JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSListDir + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSOpen + * Signature: (JLjava/lang/String;II)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSOpen + (JNIEnv *, jclass, jlong, jstring, jint, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSLSeek + * Signature: (JIJI)J + */ +JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLSeek + (JNIEnv *, jclass, jlong, jint, jlong, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativieCurveFSRead + * Signature: (JI[BJJ)J + */ +JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSRead + (JNIEnv *, jclass, jlong, jint, jbyteArray, jlong, jlong); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativieCurveFSWrite + * Signature: (JI[BJJ)J + */ +JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSWrite + (JNIEnv *, jclass, jlong, jint, jbyteArray, jlong, jlong); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSFSync + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFSync + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSClose + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSClose + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSUnlink + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUnlink + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSStatFs + * Signature: (JLio/opencurve/curve/fs/libfs/CurveFSStatVFS;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSStatFs + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSLstat + * Signature: (JLjava/lang/String;Lio/opencurve/curve/fs/libfs/CurveFSStat;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLstat + (JNIEnv *, jclass, jlong, jstring, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSFStat + * Signature: (JILio/opencurve/curve/fs/libfs/CurveFSStat;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFStat + (JNIEnv *, jclass, jlong, jint, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSSetAttr + * Signature: (JLjava/lang/String;Lio/opencurve/curve/fs/libfs/CurveFSStat;I)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSSetAttr + (JNIEnv *, jclass, jlong, jstring, jobject, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSChmod + * Signature: (JLjava/lang/String;I)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChmod + (JNIEnv *, jclass, jlong, jstring, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSChown + * Signature: (JLjava/lang/String;II)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChown + (JNIEnv *, jclass, jlong, jstring, jint, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFSMount + * Method: nativeCurveFSRename + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRename + (JNIEnv *, jclass, jlong, jstring, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/curvefs/sdk/java/pom.xml b/curvefs/sdk/java/pom.xml new file mode 100644 index 0000000000..b3f15f585c --- /dev/null +++ b/curvefs/sdk/java/pom.xml @@ -0,0 +1,92 @@ + + + + 4.0.0 + + io.opencurve.curve.fs.hadoop + curvefs-hadoop + 1.0-SNAPSHOT + + + 8 + 8 + UTF-8 + 2.7.3 + 0.80.5 + 1.15.4 + + + + + ali-maven + http://maven.aliyun.com/nexus/content/groups/public + + + + + + + native/build + + + + + + + net.java.dev.jna + jna-platform + 4.0.0 + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + + org.apache.commons + commons-compress + 1.24.0 + + + + org.apache.flink + flink-hadoop-fs + ${flink.version} + provided + + + + org.apache.flink + flink-core + ${flink.version} + provided + + + + junit + junit + 4.13.1 + test + + + + org.apache.flink + flink-connector-files + ${flink.version} + provided + + + + org.apache.flink + flink-table-common + 1.15.4 + provided + + + + diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemFactory.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemFactory.java new file mode 100644 index 0000000000..ac42dfaf82 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemFactory.java @@ -0,0 +1,48 @@ +package io.opencurve.curve.fs.flink; + +import io.opencurve.curve.fs.hadoop.CurveFileSystem; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.core.fs.FileSystemFactory; +import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.net.URI; + +public class CurveFileSystemFactory implements FileSystemFactory { + private org.apache.hadoop.conf.Configuration conf; + + private static final String CURVE_FS_CONFIG_PREFIXES = "curvefs."; + private static final String FLINK_CONFIG_PREFIXES = "fs."; + public static String SCHEME = "curvefs"; + + @Override + public void configure(org.apache.flink.configuration.Configuration config) { + conf = new Configuration(); + if (config != null) { + for (String key : config.keySet()) { + if (key.startsWith(CURVE_FS_CONFIG_PREFIXES) || key.startsWith(FLINK_CONFIG_PREFIXES)) { + String value = config.getString(key, null); + if (value != null) { + if (CurveFileSystem.class.getCanonicalName().equals(value.trim())) { + SCHEME = key.split("\\.")[1]; + } + conf.set(key, value); + } + } + } + } + } + + @Override + public String getScheme() { + return SCHEME; + } + + @Override + public FileSystem create(URI uri) throws IOException { + CurveFileSystem fs = new CurveFileSystem(); + fs.initialize(uri, conf); + return new HadoopFileSystem(fs); + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemTableFactory.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemTableFactory.java new file mode 100644 index 0000000000..da68151bbd --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/flink/CurveFileSystemTableFactory.java @@ -0,0 +1,10 @@ +package io.opencurve.curve.fs.flink; + +import org.apache.flink.connector.file.table.FileSystemTableFactory; + +public class CurveFileSystemTableFactory extends FileSystemTableFactory { + @Override + public String factoryIdentifier() { + return CurveFileSystemFactory.SCHEME; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFS.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFS.java new file mode 100644 index 0000000000..f15008a363 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFS.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.DelegateToFileSystem; +import io.opencurve.curve.fs.flink.CurveFileSystemFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * The CurveFS implementation of AbstractFileSystem. + * This impl delegates to the old FileSystem + */ +public class CurveFS extends DelegateToFileSystem { + /** + * This constructor has the signature needed by + * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}. + * + * @param theUri which must be that of localFs + * @param conf + * @throws IOException + * @throws URISyntaxException + */ + CurveFS(final URI theUri, final Configuration conf) throws IOException, + URISyntaxException { + super(theUri, new CurveFileSystem(conf), conf, CurveFileSystemFactory.SCHEME, true); + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSInputStream.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSInputStream.java new file mode 100644 index 0000000000..2dd1be8d1d --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSInputStream.java @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSInputStream; +import io.opencurve.curve.fs.libfs.CurveFSMount; + +import java.io.IOException; + +/** + *

+ * An {@link FSInputStream} for a CurveFileSystem and corresponding + * Curve instance. + */ +public class CurveFSInputStream extends FSInputStream { + private static final Log LOG = LogFactory.getLog(CurveFSInputStream.class); + private boolean closed; + + private int fileHandle; + + private long fileLength; + + private CurveFSProto curve; + + private byte[] buffer; + private int bufPos = 0; + private int bufValid = 0; + private long curvePos = 0; + + /** + * Create a new CurveInputStream. + * @param conf The system configuration. Unused. + * @param fh The filehandle provided by Curve to reference. + * @param flength The current length of the file. If the length changes + * you will need to close and re-open it to access the new data. + */ + public CurveFSInputStream(Configuration conf, CurveFSProto curvefs, + int fh, long flength, int bufferSize) { + // Whoever's calling the constructor is responsible for doing the actual curve_open + // call and providing the file handle. + fileLength = flength; + fileHandle = fh; + closed = false; + curve = curvefs; + buffer = new byte[1<<21]; + LOG.debug("CurveInputStream constructor: initializing stream with fh " + + fh + " and file length " + flength); + } + + /** Curve likes things to be closed before it shuts down, + * so closing the IOStream stuff voluntarily in a finalizer is good + */ + protected void finalize() throws Throwable { + try { + if (!closed) { + close(); + } + } finally { + super.finalize(); + } + } + + private synchronized boolean fillBuffer() throws IOException { + bufValid = curve.read(fileHandle, buffer, buffer.length, -1); + bufPos = 0; + if (bufValid < 0) { + int err = bufValid; + + bufValid = 0; + + // attempt to reset to old position. If it fails, too bad. + curve.lseek(fileHandle, curvePos, CurveFSMount.SEEK_SET); + throw new IOException("Failed to fill read buffer! Error code:" + err); + } + curvePos += bufValid; + return (bufValid != 0); + } + + /* + * Get the current position of the stream. + */ + public synchronized long getPos() throws IOException { + return curvePos - bufValid + bufPos; + } + + /** + * Find the number of bytes remaining in the file. + */ + @Override + public synchronized int available() throws IOException { + if (closed) { + throw new IOException("file is closed"); + } + return (int) (fileLength - getPos()); + } + + public synchronized void seek(long targetPos) throws IOException { + LOG.trace("CurveInputStream.seek: Seeking to position " + targetPos + " on fd " + + fileHandle); + if (targetPos > fileLength) { + throw new IOException("CurveInputStream.seek: failed seek to position " + + targetPos + " on fd " + fileHandle + + ": Cannot seek after EOF " + fileLength); + } + long oldPos = curvePos; + + curvePos = curve.lseek(fileHandle, targetPos, CurveFSMount.SEEK_SET); + bufValid = 0; + bufPos = 0; + if (curvePos < 0) { + curvePos = oldPos; + throw new IOException("Curve failed to seek to new position!"); + } + } + + /** + * Failovers are handled by the Curve code at a very low level; + * if there are issues that can be solved by changing sources + * they'll be dealt with before anybody even tries to call this method! + * @return false. + */ + public synchronized boolean seekToNewSource(long targetPos) { + return false; + } + + /** + * Read a byte from the file. + * @return the next byte. + */ + @Override + public synchronized int read() throws IOException { + LOG.trace( + "CurveInputStream.read: Reading a single byte from fd " + fileHandle + + " by calling general read function"); + + byte result[] = new byte[1]; + + if (getPos() >= fileLength) { + return -1; + } + + if (-1 == read(result, 0, 1)) { + return -1; + } + + if (result[0] < 0) { + return 256 + (int) result[0]; + } else { + return result[0]; + } + } + + /** + * Read a specified number of bytes from the file into a byte[]. + * @param buf the byte array to read into. + * @param off the offset to start at in the file + * @param len the number of bytes to read + * @return 0 if successful, otherwise an error code. + * @throws IOException on bad input. + */ + @Override + public synchronized int read(byte buf[], int off, int len) throws IOException { + LOG.trace( + "CurveInputStream.read: Reading " + len + " bytes from fd " + fileHandle); + + if (closed) { + throw new IOException( + "CurveInputStream.read: cannot read " + len + " bytes from fd " + + fileHandle + ": stream closed"); + } + + // ensure we're not past the end of the file + if (getPos() >= fileLength) { + LOG.debug( + "CurveInputStream.read: cannot read " + len + " bytes from fd " + + fileHandle + ": current position is " + getPos() + + " and file length is " + fileLength); + + return -1; + } + + int totalRead = 0; + int initialLen = len; + int read; + + do { + read = Math.min(len, bufValid - bufPos); + try { + System.arraycopy(buffer, bufPos, buf, off, read); + } catch (IndexOutOfBoundsException ie) { + throw new IOException( + "CurveInputStream.read: Indices out of bounds:" + "read length is " + + len + ", buffer offset is " + off + ", and buffer size is " + + buf.length); + } catch (ArrayStoreException ae) { + throw new IOException( + "Uh-oh, CurveInputStream failed to do an array" + + "copy due to type mismatch..."); + } catch (NullPointerException ne) { + throw new IOException( + "CurveInputStream.read: cannot read " + len + "bytes from fd:" + + fileHandle + ": buf is null"); + } + bufPos += read; + len -= read; + off += read; + totalRead += read; + } while (len > 0 && fillBuffer()); + + LOG.trace( + "CurveInputStream.read: Reading " + initialLen + " bytes from fd " + + fileHandle + ": succeeded in reading " + totalRead + " bytes"); + return totalRead; + } + + /** + * Close the CurveInputStream and release the associated filehandle. + */ + @Override + public void close() throws IOException { + LOG.trace("CurveOutputStream.close:enter"); + if (!closed) { + curve.close(fileHandle); + closed = true; + LOG.trace("CurveOutputStream.close:exit"); + } + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSOutputStream.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSOutputStream.java new file mode 100644 index 0000000000..06855bcd7d --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSOutputStream.java @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import io.opencurve.curve.fs.libfs.CurveFSMount; + +import java.io.IOException; +import java.io.OutputStream; + +/** + *

+ * An {@link OutputStream} for a CurveFileSystem and corresponding + * Curve instance. + * + * TODO: + * - When libcurvefs-jni supports ByteBuffer interface we can get rid of the + * use of the buffer here to reduce memory copies and just use buffers in + * libcurvefs. Currently it might be useful to reduce JNI crossings, but not + * much more. + */ +public class CurveFSOutputStream extends OutputStream { + private boolean closed; + + private CurveFSProto curve; + + private int fileHandle; + + private byte[] buffer; + private int bufUsed = 0; + + /** + * Construct the CurveOutputStream. + * @param conf The FileSystem configuration. + * @param fh The Curve filehandle to connect to. + */ + public CurveFSOutputStream(Configuration conf, CurveFSProto curvefs, + int fh, int bufferSize) { + curve = curvefs; + fileHandle = fh; + closed = false; + buffer = new byte[1<<21]; + } + + /** + * Close the Curve file handle if close() wasn't explicitly called. + */ + protected void finalize() throws Throwable { + try { + if (!closed) { + close(); + } + } finally { + super.finalize(); + } + } + + /** + * Ensure that the stream is opened. + */ + private synchronized void checkOpen() throws IOException { + if (closed) { + throw new IOException("operation on closed stream (fd=" + fileHandle + ")"); + } + } + + /** + * Get the current position in the file. + * @return The file offset in bytes. + */ + public synchronized long getPos() throws IOException { + checkOpen(); + return curve.lseek(fileHandle, 0, CurveFSMount.SEEK_CUR); + } + + @Override + public synchronized void write(int b) throws IOException { + byte buf[] = new byte[1]; + buf[0] = (byte) b; + write(buf, 0, 1); + } + + @Override + public synchronized void write(byte buf[], int off, int len) throws IOException { + checkOpen(); + + while (len > 0) { + int remaining = Math.min(len, buffer.length - bufUsed); + System.arraycopy(buf, off, buffer, bufUsed, remaining); + + bufUsed += remaining; + off += remaining; + len -= remaining; + + if (buffer.length == bufUsed) { + flushBuffer(); + } + } + } + + /* + * Moves data from the buffer into libcurvefs. + */ + private synchronized void flushBuffer() throws IOException { + if (bufUsed == 0) { + return; + } + + while (bufUsed > 0) { + int ret = curve.write(fileHandle, buffer, bufUsed, -1); + if (ret < 0) { + throw new IOException("curve.write: ret=" + ret); + } + + if (ret == bufUsed) { + bufUsed = 0; + return; + } + + assert(ret > 0); + assert(ret < bufUsed); + + /* + * TODO: handle a partial write by shifting the remainder of the data in + * the buffer back to the beginning and retrying the write. It would + * probably be better to use a ByteBuffer 'view' here, and I believe + * using a ByteBuffer has some other performance benefits but we'll + * likely need to update the libcurvefs-jni implementation. + */ + int remaining = bufUsed - ret; + System.arraycopy(buffer, ret, buffer, 0, remaining); + bufUsed -= ret; + } + + assert(bufUsed == 0); + } + + @Override + public synchronized void flush() throws IOException { + checkOpen(); + flushBuffer(); // buffer -> libcurvefs + curve.fsync(fileHandle); // libcurvefs -> cluster + } + + @Override + public synchronized void close() throws IOException { + checkOpen(); + flush(); + curve.close(fileHandle); + closed = true; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSProto.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSProto.java new file mode 100644 index 0000000000..df8ab69b21 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSProto.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import io.opencurve.curve.fs.libfs.CurveFSStat; +import io.opencurve.curve.fs.libfs.CurveFSStatVFS; + +import java.io.IOException; +import java.net.URI; + +abstract class CurveFSProto { + // init* + abstract void initialize(URI uri, Configuration conf) throws IOException; + abstract void shutdown() throws IOException; + // directory* + abstract void mkdirs(Path path, int mode) throws IOException; + abstract void rmdir(Path path) throws IOException; + abstract String[] listdir(Path path) throws IOException; + // file* + abstract int open(Path path, int flags, int mode) throws IOException; + abstract long lseek(int fd, long offset, int whence) throws IOException; + abstract int write(int fd, byte[] buf, long size, long offset) throws IOException; + abstract int read(int fd, byte[] buf, long size, long offset) throws IOException; + abstract void fsync(int fd) throws IOException; + abstract void close(int fd) throws IOException; + abstract void unlink(Path path) throws IOException; + // others + abstract void statfs(Path path, CurveFSStatVFS stat) throws IOException; + abstract void lstat(Path path, CurveFSStat stat) throws IOException; + abstract void fstat(int fd, CurveFSStat stat) throws IOException; + abstract void setattr(Path path, CurveFSStat stat, int mask) throws IOException; + abstract void chmod(Path path, int mode) throws IOException; + abstract void chown(Path path, int uid, int gid) throws IOException; + abstract void rename(Path src, Path dst) throws IOException; +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSTalker.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSTalker.java new file mode 100644 index 0000000000..90366cc210 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSTalker.java @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import org.apache.commons.logging.Log; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import io.opencurve.curve.fs.libfs.CurveFSMount; +import io.opencurve.curve.fs.libfs.CurveFSStat; +import io.opencurve.curve.fs.libfs.CurveFSStatVFS; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.util.Map; + +class CurveFSTalker extends CurveFSProto { + private CurveFSMount mount; + private String fsname = null; + private boolean inited = false; + + private static final String PREFIX_KEY = "curvefs"; + + CurveFSTalker(Configuration conf, Log log) { + mount = null; + } + + private String tostr(Path path) { + if (null == path) { + return "/"; + } + return path.toUri().getPath(); + } + + private void loadCfg(Configuration conf) { + Map m = conf.getValByRegex("^" + PREFIX_KEY + "\\..*"); + for (Map.Entry entry : m.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.equals(PREFIX_KEY + ".name")) { + fsname = value; + } else { + mount.confSet(key.substring(PREFIX_KEY.length() + 1), value); + } + } + } + + @Override + void initialize(URI uri, Configuration conf) throws IOException { + mount = new CurveFSMount(); + loadCfg(conf); + if (null == fsname || fsname.isEmpty()) { + throw new IOException("curvefs.name is not set"); + } + mount.mount(fsname, "/"); + inited = true; + } + + @Override + void shutdown() throws IOException { + if (inited) { + mount.umount(fsname, "/"); + mount = null; + inited = false; + } + } + + @Override + void mkdirs(Path path, int mode) throws IOException { + mount.mkdirs(tostr(path), mode); + } + + @Override + void rmdir(Path path) throws IOException { + mount.rmdir(tostr(path)); + } + + @Override + String[] listdir(Path path) throws IOException { + CurveFSStat stat = new CurveFSStat(); + try { + mount.lstat(tostr(path), stat); + } catch (FileNotFoundException e) { + return null; + } + if (!stat.isDir()) { + return null; + } + + return mount.listdir(tostr(path)); + } + + @Override + int open(Path path, int flags, int mode) throws IOException { + return mount.open(tostr(path), flags, mode); + } + + @Override + long lseek(int fd, long offset, int whence) throws IOException { + return mount.lseek(fd, offset, whence); + } + + @Override + int write(int fd, byte[] buf, long size, long offset) throws IOException { + return mount.write(fd, buf, size, offset); + } + + @Override + int read(int fd, byte[] buf, long size, long offset) throws IOException { + return mount.read(fd, buf, size, offset); + } + + @Override + void fsync(int fd) throws IOException { + mount.fsync(fd); + } + + @Override + void close(int fd) throws IOException { + mount.close(fd); + } + + @Override + void unlink(Path path) throws IOException { + mount.unlink(tostr(path)); + } + + @Override + void statfs(Path path, CurveFSStatVFS stat) throws IOException { + mount.statfs(tostr(path), stat); + } + + @Override + void lstat(Path path, CurveFSStat stat) throws IOException { + mount.lstat(tostr(path), stat); + } + + @Override + void fstat(int fd, CurveFSStat stat) throws IOException { + mount.fstat(fd, stat); + } + + @Override + void setattr(Path path, CurveFSStat stat, int mask) throws IOException { + mount.setattr(tostr(path), stat, mask); + } + + @Override + void chmod(Path path, int mode) throws IOException { + mount.chmod(tostr(path), mode); + } + + @Override + void chown(Path path, int uid, int gid) throws IOException { + mount.chown(tostr(path), uid, gid); + } + + @Override + void rename(Path src, Path dst) throws IOException { + mount.rename(tostr(src), tostr(dst)); + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java new file mode 100644 index 0000000000..09df042c1b --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java @@ -0,0 +1,388 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: NetEase Media Bigdata + */ + +package io.opencurve.curve.fs.hadoop; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Progressable; +import io.opencurve.curve.fs.libfs.CurveFSMount; +import io.opencurve.curve.fs.libfs.CurveFSStat; +import io.opencurve.curve.fs.libfs.CurveFSStatVFS; +import io.opencurve.curve.fs.hadoop.permission.Permission; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; + +public class CurveFileSystem extends FileSystem { + private static final Log LOG = LogFactory.getLog(CurveFileSystem.class); + + private URI uri; + private Path workingDir; + private CurveFSProto curve = null; + private Permission perm = null; + + public CurveFileSystem() {} + + public CurveFileSystem(Configuration conf) { + setConf(conf); + } + + private Path makeAbsolute(Path path) { + if (path.isAbsolute()) { + return path; + } + return new Path(workingDir, path); + } + + public URI getUri() { + return uri; + } + + public String getScheme() { + return uri.getScheme(); + } + + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + super.initialize(uri, conf); + + if (curve == null) { + curve = new CurveFSTalker(conf, LOG); + } + if (perm == null) { + perm = new Permission(); + } + + perm.initialize(conf); + curve.initialize(uri, conf); + setConf(conf); + this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); + this.workingDir = getHomeDirectory(); + } + + + public FSDataInputStream open(Path path, int bufferSize) throws IOException { + path = makeAbsolute(path); + + // throws filenotfoundexception if path is a directory + int fd = curve.open(path, CurveFSMount.O_RDONLY, 0); + + /* get file size */ + CurveFSStat stat = new CurveFSStat(); + curve.fstat(fd, stat); + + CurveFSInputStream istream = new CurveFSInputStream(getConf(), curve, fd, stat.size, bufferSize); + return new FSDataInputStream(istream); + } + + @Override + public void close() throws IOException { + super.close(); // this method does stuff, make sure it's run! + curve.shutdown(); + } + + public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) throws IOException { + path = makeAbsolute(path); + + if (progress != null) { + progress.progress(); + } + + int fd = curve.open(path, CurveFSMount.O_WRONLY | CurveFSMount.O_APPEND, 0); + if (progress != null) { + progress.progress(); + } + + CurveFSOutputStream ostream = new CurveFSOutputStream(getConf(), curve, fd, bufferSize); + return new FSDataOutputStream(ostream, statistics); + } + + public Path getWorkingDirectory() { + return workingDir; + } + + @Override + public void setWorkingDirectory(Path dir) { + workingDir = makeAbsolute(dir); + } + + @Override + public boolean mkdirs(Path path, FsPermission perms) throws IOException { + path = makeAbsolute(path); + curve.mkdirs(path, (int) perms.toShort()); + return true; + } + + @Override + public boolean mkdirs(Path f) throws IOException { + FsPermission perms = FsPermission.getDirDefault().applyUMask(FsPermission.getUMask(getConf()));; + return mkdirs(f, perms); + } + + public FileStatus getFileStatus(Path path) throws IOException { + path = makeAbsolute(path); + + CurveFSStat stat = new CurveFSStat(); + curve.lstat(path, stat); + String owner = perm.getUsername(stat.uid);; + String group = perm.getGroupname(stat.gid);; + + FileStatus status = new FileStatus( + stat.size, stat.isDir(), 1, stat.blksize, + stat.m_time, stat.a_time, + new FsPermission((short) stat.mode), owner, group, + path.makeQualified(this)); + return status; + } + + + public FileStatus[] listStatus(Path path) throws IOException { + path = makeAbsolute(path); + + if (isFile(path)) { + return new FileStatus[]{getFileStatus(path)}; + } + + String[] dirlist = curve.listdir(path); + if (dirlist != null) { + FileStatus[] status = new FileStatus[dirlist.length]; + for (int i = 0; i < status.length; i++) { + status[i] = getFileStatus(new Path(path, dirlist[i])); + } + curve.shutdown(); + return status; + } else { + throw new FileNotFoundException("File " + path + " does not exist."); + } + + } + + @Override + public void setPermission(Path path, FsPermission permission) throws IOException { + path = makeAbsolute(path); + curve.chmod(path, permission.toShort()); + } + + @Override + public void setTimes(Path path, long mtime, long atime) throws IOException { + path = makeAbsolute(path); + + CurveFSStat stat = new CurveFSStat(); + + int mask = 0; + if (mtime != -1) { + stat.m_time = mtime; + mask |= CurveFSMount.SETATTR_MTIME; + } + + if (atime != -1) { + stat.a_time = atime; + mask |= CurveFSMount.SETATTR_ATIME; + } + + curve.setattr(path, stat, mask); + } + + public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, + short replication, long blockSize, Progressable progress) throws IOException { + + path = makeAbsolute(path); + + boolean exists = exists(path); + + if (progress != null) { + progress.progress(); + } + + int flags = CurveFSMount.O_WRONLY | CurveFSMount.O_CREAT; + + if (exists) { + if (overwrite) { + flags |= CurveFSMount.O_TRUNC; + } else { + throw new FileAlreadyExistsException(); + } + } else { + Path parent = path.getParent(); + if (parent != null) { + if (!mkdirs(parent)) { + throw new IOException("mkdirs failed for " + parent.toString()); + } + } + } + + if (progress != null) { + progress.progress(); + } + + int fd = curve.open(path, flags, (int) permission.toShort()); + + if (progress != null) { + progress.progress(); + } + + OutputStream ostream = new CurveFSOutputStream(getConf(), curve, fd, bufferSize); + return new FSDataOutputStream(ostream, statistics); + } + + @Override + public void setOwner(Path path, String username, String groupname) throws IOException { + CurveFSStat stat = new CurveFSStat(); + curve.lstat(path, stat); + + int uid = stat.uid; + int gid = stat.gid; + if (username != null) { + uid = perm.getUid(username); + } + if (groupname != null) { + gid = perm.getGid(groupname); + } + + curve.chown(path, uid, gid); + } + + @Deprecated + public FSDataOutputStream createNonRecursive(Path path, FsPermission permission, + boolean overwrite, + int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { + path = makeAbsolute(path); + + Path parent = path.getParent(); + + if (parent != null) { + CurveFSStat stat = new CurveFSStat(); + curve.lstat(parent, stat); // handles FileNotFoundException case + if (stat.isFile()) { + throw new FileAlreadyExistsException(parent.toString()); + } + } + + return this.create(path, permission, overwrite, + bufferSize, replication, blockSize, progress); + } + + @Override + public boolean rename(Path src, Path dst) throws IOException { + src = makeAbsolute(src); + dst = makeAbsolute(dst); + + try { + CurveFSStat stat = new CurveFSStat(); + curve.lstat(dst, stat); + if (stat.isDir()) { + return rename(src, new Path(dst, src.getName())); + } + return false; + } catch (FileNotFoundException e) { + } + + try { + curve.rename(src, dst); + } catch (FileNotFoundException e) { + throw e; + } catch (Exception e) { + return false; + } + return true; + } + + @Deprecated + public boolean delete(Path path) throws IOException { + return delete(path, false); + } + + public boolean delete(Path path, boolean recursive) throws IOException { + path = makeAbsolute(path); + + /* path exists? */ + FileStatus status; + try { + status = getFileStatus(path); + } catch (FileNotFoundException e) { + return false; + } + + /* we're done if its a file */ + if (status.isFile()) { + curve.unlink(path); + return true; + } + + /* get directory contents */ + FileStatus[] dirlist = listStatus(path); + if (dirlist == null) { + return false; + } + + if (!recursive && dirlist.length > 0) { + throw new IOException("Directory " + path.toString() + "is not empty."); + } + + for (FileStatus fs : dirlist) { + if (!delete(fs.getPath(), recursive)) { + return false; + } + } + + curve.rmdir(path); + return true; + } + + @Override + public FsStatus getStatus(Path p) throws IOException { + CurveFSStatVFS stat = new CurveFSStatVFS(); + curve.statfs(p, stat); + + FsStatus status = new FsStatus(stat.bsize * stat.blocks, + stat.bsize * (stat.blocks - stat.bavail), + stat.bsize * stat.bavail); + return status; + } + + @Override + public short getDefaultReplication() { + return 1; + } + + @Override + public long getDefaultBlockSize() { + return super.getDefaultBlockSize(); + } + + @Override + protected int getDefaultPort() { + return super.getDefaultPort(); + } + + @Override + public String getCanonicalServiceName() { + return null; // Does not support Token + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/Main.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/Main.java new file mode 100644 index 0000000000..d488e309dc --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/Main.java @@ -0,0 +1,7 @@ +package io.opencurve.curve.fs.hadoop; + +public class Main { + public static void main(String[] args) { + System.out.println("Hello world!"); + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Group.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Group.java new file mode 100644 index 0000000000..d1605a863a --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Group.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: Xianfei Cao (caoxianfei1) + */ + +package io.opencurve.curve.fs.hadoop.permission; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; + +import java.util.HashMap; +import java.io.IOException; +import java.util.List; + +public class Group { + private HashMap groupnames; + private HashMap groupIDs; + private UserGroupInformation ugi; + + private static final String SUPER_GROUPNAME = "supergroup"; + private static final int SUPER_GID = 0; + + public Group() { + groupnames = new HashMap(); + groupIDs = new HashMap(); + } + + private int finger(String groupname) { + return Math.abs(groupname.hashCode()); + } + + private void addGroup(String groupname, int gid) { + groupnames.put(groupname, gid); + groupIDs.put(gid, groupname); + } + + private void loadGroupFromSystem() throws IOException { + List groups = Helper.getAllGroups(); + for (Entry group : groups) { + if (group.id == 0) { + group.id = finger(group.name); + } + addGroup(group.name, group.id); + } + addGroup(SUPER_GROUPNAME, SUPER_GID); + } + + private void loadGroupFromFile() throws IOException { + // implement it + } + + public void initialize(Path path) throws IOException { + this.ugi = UserGroupInformation.getCurrentUser(); + if (path != null) { + loadGroupFromFile(); + } else { + loadGroupFromSystem(); + } + } + + public int[] getCurrentGids() { + String[] groups = {"nogroup"}; + if (ugi.getGroupNames().length > 0) { + groups = ugi.getGroupNames(); + } + + int[] gids = new int[groups.length]; + for (int i = 0; i < groups.length; i++) { + gids[i] = getGid(groups[i]); + } + return gids; + } + + public int getGid(String groupname) { + Integer gid = groupnames.get(groupname); + if (null == gid) { + gid = finger(groupname); + addGroup(groupname, gid); + } + return gid; + } + + public String getGroupname(int gid) { + String groupname = groupIDs.get(gid); + if (null == groupname || groupname.isEmpty()) { + return String.valueOf(gid); + } + return groupname; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Helper.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Helper.java new file mode 100644 index 0000000000..fe505123aa --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Helper.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: Xianfei Cao (caoxianfei1) + */ + +package io.opencurve.curve.fs.hadoop.permission; + +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.Pointer; +import com.sun.jna.Structure; +import io.opencurve.curve.fs.hadoop.permission.Helper.CLibrary.passwd; +import io.opencurve.curve.fs.hadoop.permission.Helper.CLibrary.group; + +import java.util.Arrays; +import java.util.List; +import java.util.ArrayList; + +class Entry { + String name; + int id; + + public Entry(String name, int id) { + this.name = name; + this.id = id; + } +} + +public class Helper { + public interface CLibrary extends Library { + CLibrary INSTANCE = (CLibrary)Native.loadLibrary("c", CLibrary.class); + passwd getpwent(); + void endpwent(); + group getgrent(); + void endgrent(); + + public static class passwd extends Structure { + public String pw_name; + public String pw_passwd; + public int pw_uid; + public int pw_gid; + public String pw_gecos; + public String pw_dir; + public String pw_shell; + + protected List getFieldOrder() { + return Arrays.asList(new String[] { + "pw_name", "pw_passwd", "pw_uid", "pw_gid", "pw_gecos", "pw_dir", "pw_shell", + }); + } + } + + public static class group extends Structure { + public String gr_name; + public String gr_passwd; + public int gr_gid; + public Pointer gr_mem; + + protected List getFieldOrder() { + return Arrays.asList(new String[] { + "gr_name", "gr_passwd", "gr_gid", "gr_mem", + }); + } + } + } + + public static List getAllUsers() { + CLibrary lib = CLibrary.INSTANCE; + List users = new ArrayList(); + passwd usr = new passwd(); + while ((usr = lib.getpwent()) != null) { + users.add(new Entry(usr.pw_name, usr.pw_uid)); + } + lib.endpwent(); + return users; + } + + public static List getAllGroups() { + CLibrary lib = CLibrary.INSTANCE; + List groups = new ArrayList(); + group grp = new group(); + while ((grp = lib.getgrent()) != null) { + groups.add(new Entry(grp.gr_name, grp.gr_gid)); + } + lib.endgrent(); + return groups; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Permission.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Permission.java new file mode 100644 index 0000000000..82ce9c52e0 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/Permission.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: Xianfei Cao (caoxianfei1) + */ + +package io.opencurve.curve.fs.hadoop.permission; + +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; + +public class Permission { + private static User user = null; + private static Group group = null; + + public Permission() { + user = new User(); + group = new Group(); + } + + public void initialize(Configuration conf) throws IOException { + user.initialize(null); + group.initialize(null); + } + + public int getCurrentUid() { + return user.getCurrentUid(); + } + + public int getUid(String username) { + return user.getUid(username); + } + + public String getUsername(int uid) { + return user.getUsername(uid); + } + + public int[] getCurrentGids() { + return group.getCurrentGids(); + } + + public int getGid(String groupname) { + return group.getGid(groupname); + } + + public String getGroupname(int gid) { + return group.getGroupname(gid); + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/User.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/User.java new file mode 100644 index 0000000000..7b892caf53 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/permission/User.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-01 + * Author: Xianfei Cao (caoxianfei1) + */ + +package io.opencurve.curve.fs.hadoop.permission; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; + +import java.util.HashMap; +import java.io.IOException; +import java.util.List; + +public class User { + private HashMap usernames; + private HashMap userIDs; + private UserGroupInformation ugi; + + private static final String SUPER_USERNAME = "hdfs"; + private static final int SUPER_UID = 0; + + public User() { + usernames = new HashMap(); + userIDs = new HashMap(); + } + + private int finger(String groupname) { + return Math.abs(groupname.hashCode()); + } + + private void addUser(String username, int uid) { + usernames.put(username, uid); + userIDs.put(uid, username); + } + + private void loadUserFromSystem() throws IOException { + List users = Helper.getAllUsers(); + for (Entry user : users) { + if (user.id == 0) { + user.id = finger(user.name); + } + addUser(user.name, user.id); + } + addUser(SUPER_USERNAME, SUPER_UID); + } + + private void loadUserFromFile() throws IOException { + // TODO: implement it + } + + public void initialize(Path path) throws IOException { + this.ugi = UserGroupInformation.getCurrentUser(); + if (path != null) { + loadUserFromFile(); + } else { + loadUserFromSystem(); + } + } + + public int getCurrentUid() { + String username = ugi.getShortUserName(); + return getUid(username); + } + + public int getUid(String username) { + Integer gid = usernames.get(username); + if (null == gid) { + gid = finger(username); + addUser(username, gid); + } + return gid; + } + + public String getUsername(int uid) { + String username = userIDs.get(uid); + if (null == username || username.isEmpty()) { + return String.valueOf(uid); + } + return username; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSMount.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSMount.java new file mode 100644 index 0000000000..7b423dcc0f --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSMount.java @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-07 + * Author: Jingli Chen (Wine93) + */ +package io.opencurve.curve.fs.libfs; + +import java.io.IOException; + +public class CurveFSMount { + // init + private native long nativeCurveFSCreate(); + private native void nativeCurveFSRelease(long instancePtr); + private static native void nativeCurveFSConfSet(long instancePtr, String key, String value); + private static native int nativeCurveFSMount(long instancePtr, String fsname, String mountpoint); + private static native int nativeCurveFSUmount(long instancePtr, String fsname, String mountpoint); + // dir* + private static native int nativeCurveFSMkDirs(long instancePtr, String path, int mode); + private static native int nativeCurveFSRmDir(long instancePtr, String path); + private static native String[] nativeCurveFSListDir(long instancePtr, String path); + // file* + private static native int nativeCurveFSOpen(long instancePtr, String path, int flags, int mode); + private static native long nativeCurveFSLSeek(long instancePtr, int fd, long offset, int whence); + private static native long nativieCurveFSRead(long instancePtr, int fd, byte[] buffer, long size, long offset); + private static native long nativieCurveFSWrite(long instancePtr, int fd, byte[] buffer, long size, long offset); + private static native int nativeCurveFSFSync(long instancePtr, int fd); + private static native int nativeCurveFSClose(long instancePtr, int fd); + private static native int nativeCurveFSUnlink(long instancePtr, String path); + // others + private static native int nativeCurveFSStatFs(long instancePtr, CurveFSStatVFS statvfs); + private static native int nativeCurveFSLstat(long instancePtr, String path, CurveFSStat stat); + private static native int nativeCurveFSFStat(long instancePtr, int fd, CurveFSStat stat); + private static native int nativeCurveFSSetAttr(long instancePtr, String path, CurveFSStat stat, int mask); + private static native int nativeCurveFSChmod(long instancePtr, String path, int mode); + private static native int nativeCurveFSChown(long instancePtr, String path, int uid, int gid); + private static native int nativeCurveFSRename(long instancePtr, String src, String dst); + + /* + * Flags for open(). + * + * Must be synchronized with JNI if changed. + */ + public static final int O_RDONLY = 1; + public static final int O_RDWR = 2; + public static final int O_APPEND = 4; + public static final int O_CREAT = 8; + public static final int O_TRUNC = 16; + public static final int O_EXCL = 32; + public static final int O_WRONLY = 64; + public static final int O_DIRECTORY = 128; + + /* + * Whence flags for seek(). + * + * Must be synchronized with JNI if changed. + */ + public static final int SEEK_SET = 0; + public static final int SEEK_CUR = 1; + public static final int SEEK_END = 2; + + /* + * Attribute flags for setattr(). + * + * Must be synchronized with JNI if changed. + */ + public static final int SETATTR_MODE = 1; + public static final int SETATTR_UID = 2; + public static final int SETATTR_GID = 4; + public static final int SETATTR_MTIME = 8; + public static final int SETATTR_ATIME = 16; + + private static final String CURVEFS_DEBUG_ENV_VAR = "CURVEFS_DEBUG"; + private static final String CLASS_NAME = "io.opencurve.curve.fs.CurveFSMount"; + + private long instancePtr; + + private static void accessLog(String name, String... args) { + String value = System.getenv(CURVEFS_DEBUG_ENV_VAR); + if (!Boolean.valueOf(value)) { + return; + } + + String params = String.join(",", args); + String message = String.format("%s.%s(%s)", CLASS_NAME, name, params); + System.out.println(message); + } + + static { + accessLog("loadLibrary"); + try { + CurveFSNativeLoader.getInstance().loadLibrary(); + } catch(Exception e) {} + } + + protected void finalize() throws Throwable { + accessLog("finalize"); + } + + public CurveFSMount() { + accessLog("CurveMount"); + instancePtr = nativeCurveFSCreate(); + } + + public void confSet(String key, String value) { + accessLog("confSet", key, value); + nativeCurveFSConfSet(instancePtr, key, value); + } + + public void mount(String fsname, String mountpoint) throws IOException { + accessLog("mount"); + nativeCurveFSMount(instancePtr, fsname, mountpoint); + } + + public void umount(String fsname, String mountpoint) throws IOException { + accessLog("umount"); + nativeCurveFSUmount(instancePtr, fsname, mountpoint); + } + + public void shutdown() throws IOException { + accessLog("shutdown"); + } + + // directory* + public void mkdirs(String path, int mode) throws IOException { + accessLog("mkdirs", path.toString()); + nativeCurveFSMkDirs(instancePtr, path, mode); + } + + public void rmdir(String path) throws IOException { + accessLog("rmdir", path.toString()); + nativeCurveFSRmDir(instancePtr, path); + } + + public String[] listdir(String path) throws IOException { + accessLog("listdir", path.toString()); + return nativeCurveFSListDir(instancePtr, path); + } + + // file* + public int open(String path, int flags, int mode) throws IOException { + accessLog("open", path.toString()); + return nativeCurveFSOpen(instancePtr, path, flags, mode); + } + + public long lseek(int fd, long offset, int whence) throws IOException { + accessLog("lseek", String.valueOf(fd), String.valueOf(offset), String.valueOf(whence)); + return nativeCurveFSLSeek(instancePtr, fd, offset, whence); + } + + public int read(int fd, byte[] buf, long size, long offset) throws IOException { + accessLog("read", String.valueOf(fd), String.valueOf(size), String.valueOf(size)); + long rc = nativieCurveFSRead(instancePtr, fd, buf, size, offset); + return (int) rc; + } + + public int write(int fd, byte[] buf, long size, long offset) throws IOException { + accessLog("write", String.valueOf(fd), String.valueOf(size), String.valueOf(size)); + long rc = nativieCurveFSWrite(instancePtr, fd, buf, size, offset); + return (int) rc; + } + + public void fsync(int fd) throws IOException { + accessLog("fsync", String.valueOf(fd)); + nativeCurveFSFSync(instancePtr, fd); + } + + public void close(int fd) throws IOException { + accessLog("close", String.valueOf(fd)); + nativeCurveFSClose(instancePtr, fd); + } + + public void unlink(String path) throws IOException { + accessLog("unlink", path.toString()); + nativeCurveFSUnlink(instancePtr, path); + } + + // others + public void statfs(String path, CurveFSStatVFS statvfs) throws IOException { + accessLog("statfs", path.toString()); + nativeCurveFSStatFs(instancePtr, statvfs); + } + + public void lstat(String path, CurveFSStat stat) throws IOException { + accessLog("lstat", path.toString()); + nativeCurveFSLstat(instancePtr, path, stat); + } + + public void fstat(int fd, CurveFSStat stat) throws IOException { + accessLog("fstat", String.valueOf(fd)); + nativeCurveFSFStat(instancePtr, fd, stat); + } + + public void setattr(String path, CurveFSStat stat, int mask) throws IOException { + accessLog("setattr", path.toString()); + nativeCurveFSSetAttr(instancePtr, path, stat, mask); + } + + public void chmod(String path, int mode) throws IOException { + accessLog("chmod", path.toString()); + nativeCurveFSChmod(instancePtr, path, mode); + } + + public void chown(String path, int uid, int gid) throws IOException { + accessLog("chown", path.toString(), String.valueOf(uid), String.valueOf(gid)); + nativeCurveFSChown(instancePtr, path, uid, gid); + } + + public void rename(String src, String dst) throws IOException { + accessLog("rename", src.toString(), dst.toString()); + nativeCurveFSRename(instancePtr, src, dst); + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSNativeLoader.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSNativeLoader.java new file mode 100644 index 0000000000..7ba7a535e8 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSNativeLoader.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-07 + * Author: Jingli Chen (Wine93) + */ + +package io.opencurve.curve.fs.libfs; + +import java.net.URL; +import java.net.URLConnection; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.utils.IOUtils; + +public class CurveFSNativeLoader { + boolean initialized = false; + private static final CurveFSNativeLoader instance = new CurveFSNativeLoader(); + + private static final String TMP_DIR = "/tmp"; + private static final String CURVEFS_LIBRARY_PATH = "/tmp/libcurvefs"; + private static final String RESOURCE_TAR_NAME = "libcurvefs.tar"; + private static final String JNI_LIBRARY_NAME = "libcurvefs_jni.so"; + + private CurveFSNativeLoader() {} + + public static CurveFSNativeLoader getInstance() { + return instance; + } + + public long getJarModifiedTime() throws IOException { + URL location = CurveFSNativeLoader.class.getProtectionDomain().getCodeSource().getLocation(); + URLConnection conn = location.openConnection(); + return conn.getLastModified(); + } + + public void descompress(InputStream in, String dest) throws IOException { + File dir = new File(dest); + if (!dir.exists()) { + dir.mkdirs(); + } + + ArchiveEntry entry; + TarArchiveInputStream reader = new TarArchiveInputStream(in); + while ((entry = reader.getNextTarEntry()) != null) { + if (entry.isDirectory()) { + continue; + } + + String path = TMP_DIR + File.separator + entry.getName(); + File file = new File(path); + IOUtils.copy(reader, new FileOutputStream(file)); + } + reader.close(); + } + + public void loadJniLibrary() throws IOException { + File libFile = new File(CURVEFS_LIBRARY_PATH, JNI_LIBRARY_NAME); + System.load(libFile.getAbsolutePath()); + } + + public synchronized void loadLibrary() throws IOException { + if (initialized) { + return; + } + + long jarModifiedTime = getJarModifiedTime(); + File libDir = new File(CURVEFS_LIBRARY_PATH); + if (libDir.exists() && libDir.lastModified() == jarModifiedTime) { + loadJniLibrary(); + initialized = true; + return; + } + + InputStream reader = CurveFSNativeLoader.class.getResourceAsStream("/" + RESOURCE_TAR_NAME); + if (reader == null) { + throw new IOException("Cannot get resource " + RESOURCE_TAR_NAME + " from Jar file."); + } + descompress(reader, CURVEFS_LIBRARY_PATH); + reader.close(); + + libDir.setLastModified(jarModifiedTime); + loadJniLibrary(); + initialized = true; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStat.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStat.java new file mode 100644 index 0000000000..d188ca93f5 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStat.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-07 + * Author: Jingli Chen (Wine93) + */ + +package io.opencurve.curve.fs.libfs; + +public class CurveFSStat { + public boolean is_file; /* S_ISREG */ + public boolean is_directory; /* S_ISDIR */ + public boolean is_symlink; /* S_ISLNK */ + + public int mode; + public int uid; + public int gid; + public long size; + public long blksize; + public long blocks; + public long a_time; + public long m_time; + + public boolean isFile() { + return is_file; + } + + public boolean isDir() { + return is_directory; + } + + public boolean isSymlink() { + return is_symlink; + } +} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStatVFS.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStatVFS.java new file mode 100644 index 0000000000..5ee2c1b8b0 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStatVFS.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-07 + * Author: Jingli Chen (Wine93) + */ + +package io.opencurve.curve.fs.libfs; + +public class CurveFSStatVFS { + public long bsize; + public long frsize; + public long blocks; + public long bavail; + public long files; + public long fsid; + public long namemax; +} diff --git a/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory b/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory new file mode 100644 index 0000000000..49f845958d --- /dev/null +++ b/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.core.fs.FileSystemFactory @@ -0,0 +1 @@ +io.opencurve.curve.fs.flink.CurveFileSystemFactory diff --git a/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 0000000000..767cc341af --- /dev/null +++ b/curvefs/sdk/java/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1 @@ +io.opencurve.curve.fs.flink.CurveFileSystemTableFactory diff --git a/curvefs/sdk/java/src/test/java/io/opencurve/curve/fs/others/TestOthers.java b/curvefs/sdk/java/src/test/java/io/opencurve/curve/fs/others/TestOthers.java new file mode 100644 index 0000000000..754e6cc797 --- /dev/null +++ b/curvefs/sdk/java/src/test/java/io/opencurve/curve/fs/others/TestOthers.java @@ -0,0 +1,15 @@ +package io.opencurve.curve.fs.others; + +import junit.framework.TestCase; +import io.opencurve.curve.fs.hadoop.CurveFileSystem; + +import java.io.IOException; +import java.net.URL; + +public class TestOthers extends TestCase { + public void testHelloWorld() throws IOException { + URL location = CurveFileSystem.class.getProtectionDomain().getCodeSource().getLocation(); + System.out.println("Hello World"); + System.out.println(location.getPath()); + } +} diff --git a/curvefs/sdk/libcurvefs/BUILD b/curvefs/sdk/libcurvefs/BUILD new file mode 100644 index 0000000000..44f0f8c703 --- /dev/null +++ b/curvefs/sdk/libcurvefs/BUILD @@ -0,0 +1,44 @@ +# +# Copyright (c) 2023 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_DEFAULT_COPTS") + +cc_binary( + name = "curvefs", + srcs = [ + "libcurvefs.h", + "libcurvefs.cpp", + ], + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//curvefs/src/client/vfs:vfs", + ], + linkshared = True, +) + +cc_library( + name = "curvefs_lib", + srcs = [ + "libcurvefs.h", + "libcurvefs.cpp", + ], + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//curvefs/src/client/vfs:vfs", + ], +) diff --git a/curvefs/sdk/libcurvefs/README.md b/curvefs/sdk/libcurvefs/README.md new file mode 100644 index 0000000000..839fe28f20 --- /dev/null +++ b/curvefs/sdk/libcurvefs/README.md @@ -0,0 +1,29 @@ +libcurvefs +=== + +SDK C/C++ library for CurveFS. + +Example +=== + +```c +#include "libcurvefs.h" + +int instance = curvefs_create(); +curvefs_conf_set(instance, "s3.ak", "xxx") +curvefs_conf_set(instance, "s3.sk", "xxx") + +... + +int rc = curvefs_mount(instance, "fsname", "/); +if (rc != 0) { + // mount failed +} + +rc = curvefs_mkdir(instance_ptr, "/mydir") +if (rc != 0) { + // mkdir failed +} +``` + +See [examples](examples) for more examples. diff --git a/curvefs/sdk/libcurvefs/examples/Makefile b/curvefs/sdk/libcurvefs/examples/Makefile new file mode 100644 index 0000000000..2e4acf367d --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/Makefile @@ -0,0 +1,24 @@ +# Copyright (C) 2024 Jingli Chen (Wine93), NetEase Inc. + +root= $(shell echo $${PWD%/curvefs/sdk/libcurvefs/examples}) +so_path?= $(root)/bazel-bin/curvefs/sdk/libcurvefs +hrd_opt?= -I/curve +build_opt?= -L$(so_path) +link_opt?= -Wl,-rpath=$(so_path) -Wall +flags?= $(hrd_opt) $(build_opt) $(link_opt) +targets= mkdir rmdir ls touch read write unlink append rename stat statfs fstat chmod + +libcurvefs: + mkdir -p bin + bazel build \ + --config=gcc7-later \ + --compilation_mode=opt \ + --copt -g \ + --copt -DUNINSTALL_SIGSEGV=1 \ + --copt -DCLIENT_CONF_PATH="\"$(root)/curvefs/conf/client.conf\"" \ + //curvefs/sdk/java/native:curvefs_jni + +$(targets): libcurvefs + gcc ${flags} $@.c -o bin/$@ -lcurvefs + +all: $(targets) diff --git a/curvefs/sdk/libcurvefs/examples/append.c b/curvefs/sdk/libcurvefs/examples/append.c new file mode 100644 index 0000000000..79aaa3f41b --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/append.c @@ -0,0 +1,48 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 2); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_WRONLY, 0777); + if (fd != 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_lseek(instance, fd, 0, SEEK_END); + if (rc != 0) { + fprintf(stderr, "lseek failed: retcode = %d\n", rc); + return rc; + } + + ssize_t n = curvefs_write(instance, fd, argv[2], strlen(argv[2])); + if (n < 0) { + rc = n; + fprintf(stderr, "write failed: retcode = %d\n", rc); + return rc; + } else if (n != strlen(argv[2])) { + fprintf(stderr, "write failed: %zd != %zu\n", n, strlen(argv[2])); + return -1; + } + + rc = curvefs_close(instance, fd); + if (rc != 0) { + fprintf(stderr, "close failed: retcode = %d\n", rc); + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/chmod.c b/curvefs/sdk/libcurvefs/examples/chmod.c new file mode 100644 index 0000000000..f6da02ee77 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/chmod.c @@ -0,0 +1,24 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 2); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_chmod(instance, argv[1], atoi(argv[2])); + if (rc != 0) { + fprintf(stderr, "chmod failed: retcode = %d\n", rc); + return rc; + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/common.h b/curvefs/sdk/libcurvefs/examples/common.h new file mode 100644 index 0000000000..dc262eb0b2 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/common.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CURVEFS_SDK_LIBCURVEFS_EXAMPLES_COMMON_H_ +#define CURVEFS_SDK_LIBCURVEFS_EXAMPLES_COMMON_H_ + +#include +#include +#include + +#include "curvefs/sdk/libcurvefs/libcurvefs.h" + +const char* KEY_FSNAME = "CURVEFS_FSNAME"; +const char* KEY_S3_AK = "s3.ak"; +const char* KEY_S3_SK = "s3.sk"; +const char* KEY_S3_ENDPOINT = "s3.endpoint"; +const char* KEY_S3_BUCKET = "s3.bucket_name"; +const char* KEY_MDS_ADDRS = "mdsOpt.rpcRetryOpt.addrs"; + +char* +require_string(const char* name) { + char* value = getenv(name); + if (strlen(value) == 0) { + fprintf(stderr, "require %s\n", name); + exit(1); + } + return value; +} + +char* +get_filesystem_name() { + return require_string(KEY_FSNAME); +} + +char* +get_mountpoint() { + return "/"; +} + +void +load_cfg_from_environ(uintptr_t instance) { + curvefs_conf_set(instance, KEY_S3_AK, require_string(KEY_S3_AK)); + curvefs_conf_set(instance, KEY_S3_SK, require_string(KEY_S3_SK)); + curvefs_conf_set(instance, KEY_S3_ENDPOINT, + require_string(KEY_S3_ENDPOINT)); + curvefs_conf_set(instance, KEY_S3_BUCKET, require_string(KEY_S3_BUCKET)); + curvefs_conf_set(instance, KEY_MDS_ADDRS, require_string(KEY_MDS_ADDRS)); + curvefs_conf_set(instance, "fs.accessLogging", "true"); + curvefs_conf_set(instance, "client.loglevel", "6"); + curvefs_conf_set(instance, "diskCache.diskCacheType", "0"); +} + +void +exact_args(int argc, int number) { + if (--argc == number) { + return; + } + + fprintf(stderr, "requires exactly %d argument[s]\n", number); + exit(1); +} + +#endif // CURVEFS_SDK_LIBCURVEFS_EXAMPLES_COMMON_H_ diff --git a/curvefs/sdk/libcurvefs/examples/fstat.c b/curvefs/sdk/libcurvefs/examples/fstat.c new file mode 100644 index 0000000000..5745f19e05 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/fstat.c @@ -0,0 +1,36 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_WRONLY, 0777); + if (fd != 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + return rc; + } + + struct stat stat; + rc = curvefs_fstat(instance, fd, &stat); + if (rc != 0) { + fprintf(stderr, "fstat failed: retcode = %d\n", rc); + return rc; + } + + printf("ino=%d, size=%d\n", stat.st_ino, stat.st_size); + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/ls.c b/curvefs/sdk/libcurvefs/examples/ls.c new file mode 100644 index 0000000000..71df5f30be --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/ls.c @@ -0,0 +1,49 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + // mount + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + // opendir + dir_stream_t dir_stream; + rc = curvefs_opendir(instance, argv[1], &dir_stream); + if (rc != 0) { + fprintf(stderr, "opendir failed: retcode = %d\n", rc); + return rc; + } + + // readdir + dirent_t dirent; + for ( ;; ) { + ssize_t n = curvefs_readdir(instance, &dir_stream, &dirent); + if (n < 0) { + rc = n; + fprintf(stderr, "readdir failed: retcode = %d\n", rc); + break; + } else if (n == 0) { + break; + } + + printf("%s: ino=%d size=%d\n", dirent.name, + dirent.stat.st_ino, + dirent.stat.st_size); + } + + rc = curvefs_closedir(instance, &dir_stream); + if (rc != 0) { + fprintf(stderr, "closedir failed: retcode = %d\n", rc); + } + return rc; +} diff --git a/curvefs/sdk/libcurvefs/examples/mkdir.c b/curvefs/sdk/libcurvefs/examples/mkdir.c new file mode 100644 index 0000000000..c2c5b4bce6 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/mkdir.c @@ -0,0 +1,25 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_mkdir(instance, argv[1], 0755); + if (rc != 0) { + fprintf(stderr, "mkdir failed: retcode = %d\n", rc); + return rc; + } + + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/read.c b/curvefs/sdk/libcurvefs/examples/read.c new file mode 100644 index 0000000000..3dcbd5d4ee --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/read.c @@ -0,0 +1,44 @@ +#include + +#include "common.h" + +#define MAX_BUFFER_SIZE 4096 + 5 + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_WRONLY, 0777); + if (fd != 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + return rc; + } + + char buffer[MAX_BUFFER_SIZE]; + for ( ;; ) { + ssize_t n = curvefs_read(instance, fd, buffer, sizeof(buffer)); + if (n < 0) { + rc = n; + fprintf(stderr, "read failed: retcode = %d\n", rc); + return rc; + } else if (n == 0) { + break; + } + + buffer[n] = '\0'; + printf("%s", buffer); + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/rename.c b/curvefs/sdk/libcurvefs/examples/rename.c new file mode 100644 index 0000000000..77f824531b --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/rename.c @@ -0,0 +1,24 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 2); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_rename(instance, argv[1], argv[2]); + if (rc != 0) { + fprintf(stderr, "rename failed: retcode = %d\n", rc); + return rc; + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/rmdir.c b/curvefs/sdk/libcurvefs/examples/rmdir.c new file mode 100644 index 0000000000..7ae25a80a7 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/rmdir.c @@ -0,0 +1,25 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_rmdir(instance, argv[1]); + if (rc != 0) { + fprintf(stderr, "rmdir failed: retcode = %d\n", rc); + return rc; + } + + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/stat.c b/curvefs/sdk/libcurvefs/examples/stat.c new file mode 100644 index 0000000000..34589bc6fa --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/stat.c @@ -0,0 +1,27 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + struct stat stat; + rc = curvefs_lstat(instance, argv[1], &stat); + if (rc != 0) { + fprintf(stderr, "stat failed: retcode = %d\n", rc); + return rc; + } + + printf("ino=%d, size=%d\n", stat.st_ino, stat.st_size); + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/statfs.c b/curvefs/sdk/libcurvefs/examples/statfs.c new file mode 100644 index 0000000000..81f0aee290 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/statfs.c @@ -0,0 +1,29 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 0); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + struct statvfs statvfs; + rc = curvefs_statfs(instance, &statvfs); + if (rc != 0) { + fprintf(stderr, "statvfs failed: retcode = %d\n", rc); + return rc; + } + + printf("fsid = %d\n", statvfs.f_fsid); + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/touch.c b/curvefs/sdk/libcurvefs/examples/touch.c new file mode 100644 index 0000000000..b3185eb6ec --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/touch.c @@ -0,0 +1,26 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_CREAT, 0644); + if (fd < 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + } + return rc; +} diff --git a/curvefs/sdk/libcurvefs/examples/unlink.c b/curvefs/sdk/libcurvefs/examples/unlink.c new file mode 100644 index 0000000000..e20fbe5636 --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/unlink.c @@ -0,0 +1,25 @@ +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 1); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + rc = curvefs_unlink(instance, argv[1]); + if (rc != 0) { + fprintf(stderr, "unlink failed: retcode = %d\n", rc); + return rc; + } + + return 0; +} diff --git a/curvefs/sdk/libcurvefs/examples/write.c b/curvefs/sdk/libcurvefs/examples/write.c new file mode 100644 index 0000000000..d3f85c26fb --- /dev/null +++ b/curvefs/sdk/libcurvefs/examples/write.c @@ -0,0 +1,42 @@ +#include + +#include "common.h" + +int +main(int argc, char** argv) { + exact_args(argc, 2); + + uintptr_t instance = curvefs_create(); + load_cfg_from_environ(instance); + + char* fsname = get_filesystem_name(); + char* mountpoint = get_mountpoint(); + int rc = curvefs_mount(instance, fsname, mountpoint); + if (rc != 0) { + fprintf(stderr, "mount failed: retcode = %d\n", rc); + return rc; + } + + int fd = curvefs_open(instance, argv[1], O_WRONLY, 0777); + if (fd != 0) { + rc = fd; + fprintf(stderr, "open failed: retcode = %d\n", rc); + return rc; + } + + ssize_t n = curvefs_write(instance, fd, argv[2], strlen(argv[2])); + if (n < 0) { + rc = n; + fprintf(stderr, "write failed: retcode = %d\n", rc); + return rc; + } else if (n != strlen(argv[2])) { + fprintf(stderr, "write failed: %zd != %zu\n", n, strlen(argv[2])); + return -1; + } + + rc = curvefs_close(instance, fd); + if (rc != 0) { + fprintf(stderr, "close failed: retcode = %d\n", rc); + } + return 0; +} diff --git a/curvefs/sdk/libcurvefs/libcurvefs.cpp b/curvefs/sdk/libcurvefs/libcurvefs.cpp new file mode 100644 index 0000000000..d53c0b51b8 --- /dev/null +++ b/curvefs/sdk/libcurvefs/libcurvefs.cpp @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-12 + * Author: Jingli Chen (Wine93) + */ + +#include "curvefs/src/client/filesystem/error.h" +#include "curvefs/sdk/libcurvefs/libcurvefs.h" + +using ::curvefs::client::filesystem::CURVEFS_ERROR; +using ::curvefs::client::filesystem::SysErr; +using ::curvefs::client::vfs::DirStream; +using ::curvefs::client::vfs::DirEntry; + +static curvefs_mount_t* get_instance(uintptr_t instance_ptr) { + return reinterpret_cast(instance_ptr); +} + +static void stream_cast(DirStream* stream, dir_stream_t* dir_stream) { + dir_stream->fh = stream->fh; + dir_stream->ino = stream->ino; + dir_stream->offset = stream->offset; +} + +uintptr_t curvefs_create() { + auto mount = new curvefs_mount_t(); + mount->cfg = Configure::Default(); + return reinterpret_cast(mount); +} + +void curvefs_release(uintptr_t instance_ptr) { + auto mount = get_instance(instance_ptr); + delete mount; + mount = nullptr; +} + +void curvefs_conf_set(uintptr_t instance_ptr, + const char* key, + const char* value) { + auto mount = get_instance(instance_ptr); + return mount->cfg->Set(key, value); +} + +int curvefs_mount(uintptr_t instance_ptr, + const char* fsname, + const char* mountpoint) { + auto mount = get_instance(instance_ptr); + mount->vfs = std::make_shared(mount->cfg); + auto rc = mount->vfs->Mount(fsname, mountpoint); + return SysErr(rc); +} + +int curvefs_umonut(uintptr_t instance_ptr, + const char* fsname, + const char* mountpoint) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Umount(fsname, mountpoint); + return SysErr(rc); +} + +int curvefs_mkdir(uintptr_t instance_ptr, const char* path, uint16_t mode) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->MkDir(path, mode); + return SysErr(rc); +} + +int curvefs_mkdirs(uintptr_t instance_ptr, const char* path, uint16_t mode) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->MkDirs(path, mode); + return SysErr(rc); +} + +int curvefs_rmdir(uintptr_t instance_ptr, const char* path) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->RmDir(path); + return SysErr(rc); +} + +int curvefs_opendir(uintptr_t instance_ptr, + const char* path, + dir_stream_t* dir_stream) { + DirStream stream; + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->OpenDir(path, &stream); + if (rc == CURVEFS_ERROR::OK) { + stream_cast(&stream, dir_stream); + } + return SysErr(rc); +} + +ssize_t curvefs_readdir(uintptr_t instance_ptr, + dir_stream_t* dir_stream, + dirent_t* dirent) { + DirEntry dirEntry; + auto mount = get_instance(instance_ptr); + DirStream* stream = reinterpret_cast(dir_stream); + auto rc = mount->vfs->ReadDir(stream, &dirEntry); + if (rc == CURVEFS_ERROR::OK) { + std::string name = dirEntry.name; + name.copy(dirent->name, name.size() ); + dirent->name[name.size()] = '\0'; + mount->vfs->Attr2Stat(&dirEntry.attr, &dirent->stat); + return 1; + } else if (rc == CURVEFS_ERROR::END_OF_FILE) { + return 0; + } + return SysErr(rc); +} + +int curvefs_closedir(uintptr_t instance_ptr, dir_stream_t* dir_stream) { + DirStream* stream = reinterpret_cast(dir_stream); + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->CloseDir(stream); + return SysErr(rc); +} + +int curvefs_open(uintptr_t instance_ptr, + const char* path, + uint32_t flags, + uint16_t mode) { + CURVEFS_ERROR rc; + auto mount = get_instance(instance_ptr); + if (flags & O_CREAT) { + rc = mount->vfs->Create(path, mode); + if (rc != CURVEFS_ERROR::OK) { + return SysErr(rc); + } + } + + uint64_t fd; + rc = mount->vfs->Open(path, flags, mode, &fd); + if (rc != CURVEFS_ERROR::OK) { + return SysErr(rc); + } + return static_cast(fd); +} + +int curvefs_lseek(uintptr_t instance_ptr, + int fd, + uint64_t offset, + int whence) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->LSeek(fd, offset, whence); + return SysErr(rc); +} + +ssize_t curvefs_read(uintptr_t instance_ptr, + int fd, + char* buffer, + size_t count) { + size_t nread; + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Read(fd, buffer, count, &nread); + if (rc != CURVEFS_ERROR::OK) { + return SysErr(rc); + } + return nread; +} + +ssize_t curvefs_write(uintptr_t instance_ptr, + int fd, + char* buffer, + size_t count) { + size_t nwritten; + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Write(fd, buffer, count, &nwritten); + if (rc != CURVEFS_ERROR::OK) { + return SysErr(rc); + } + return nwritten; +} + +int curvefs_fsync(uintptr_t instance_ptr, int fd) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->FSync(fd); + return SysErr(rc); +} + +int curvefs_close(uintptr_t instance_ptr, int fd) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Close(fd); + return SysErr(rc); +} + +int curvefs_unlink(uintptr_t instance_ptr, const char* path) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Unlink(path); + return SysErr(rc); +} + +int curvefs_statfs(uintptr_t instance_ptr, + struct statvfs* statvfs) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->StatFs(statvfs); + return SysErr(rc); +} + +int curvefs_lstat(uintptr_t instance_ptr, const char* path, struct stat* stat) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->LStat(path, stat); + return SysErr(rc); +} + +int curvefs_fstat(uintptr_t instance_ptr, int fd, struct stat* stat) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->FStat(fd, stat); + return SysErr(rc); +} + +int curvefs_setattr(uintptr_t instance_ptr, + const char* path, + struct stat* stat, + int to_set) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->SetAttr(path, stat, to_set); + return SysErr(rc); +} + +int curvefs_chmod(uintptr_t instance_ptr, const char* path, uint16_t mode) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Chmod(path, mode); + return SysErr(rc); +} + +int curvefs_chown(uintptr_t instance_ptr, + const char* path, + uint32_t uid, + uint32_t gid) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Chown(path, uid, gid); + return SysErr(rc); +} + +int curvefs_rename(uintptr_t instance_ptr, + const char* oldpath, + const char* newpath) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Rename(oldpath, newpath); + return SysErr(rc); +} diff --git a/curvefs/sdk/libcurvefs/libcurvefs.h b/curvefs/sdk/libcurvefs/libcurvefs.h new file mode 100644 index 0000000000..35d2f743df --- /dev/null +++ b/curvefs/sdk/libcurvefs/libcurvefs.h @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-12 + * Author: Jingli Chen (Wine93) + */ + +#ifndef CURVEFS_SDK_LIBCURVEFS_LIBCURVEFS_H_ +#define CURVEFS_SDK_LIBCURVEFS_LIBCURVEFS_H_ + +#include +#include +#include + +#ifdef __cplusplus + +#include +#include +#include + +#include "curvefs/src/client/vfs/config.h" +#include "curvefs/src/client/vfs/vfs.h" + +using ::curvefs::client::vfs::Configure; +using ::curvefs::client::vfs::VFS; + +typedef struct { + std::shared_ptr cfg; + std::shared_ptr vfs; +} curvefs_mount_t; + +#endif // __cplusplus + +// Must be synchronized with DirStream if changed +typedef struct { + uint64_t ino; + uint64_t fh; + uint64_t offset; +} dir_stream_t; + +typedef struct { + struct stat stat; + char name[256]; +} dirent_t; + +#ifdef __cplusplus +extern "C" { +#endif + +uintptr_t curvefs_create(); + +void curvefs_release(uintptr_t instance_ptr); + +// NOTE: instance_ptr is the pointer of curvefs_mount_t instance. +void curvefs_conf_set(uintptr_t instance_ptr, + const char* key, + const char* value); + +int curvefs_mount(uintptr_t instance_ptr, + const char* fsname, + const char* mountpoint); + +int curvefs_umonut(uintptr_t instance_ptr, + const char* fsname, + const char* mountpoint); + +// directory +int curvefs_mkdir(uintptr_t instance_ptr, const char* path, uint16_t mode); + +int curvefs_mkdirs(uintptr_t instance_ptr, const char* path, uint16_t mode); + +int curvefs_rmdir(uintptr_t instance_ptr, const char* path); + +int curvefs_opendir(uintptr_t instance_ptr, + const char* path, + dir_stream_t* dir_stream); + +ssize_t curvefs_readdir(uintptr_t instance_ptr, + dir_stream_t* dir_stream, + dirent_t* dirent); + +int curvefs_closedir(uintptr_t instance_ptr, dir_stream_t* dir_stream); + +// file +int curvefs_open(uintptr_t instance_ptr, + const char* path, + uint32_t flags, + uint16_t mode); + +int curvefs_lseek(uintptr_t instance_ptr, + int fd, + uint64_t offset, + int whence); + +ssize_t curvefs_read(uintptr_t instance_ptr, + int fd, + char* buffer, + size_t count); + +ssize_t curvefs_write(uintptr_t instance_ptr, + int fd, + char* buffer, + size_t count); + +int curvefs_fsync(uintptr_t instance_ptr, int fd); + +int curvefs_close(uintptr_t instance_ptr, int fd); + +int curvefs_unlink(uintptr_t instance_ptr, const char* path); + +// others +int curvefs_statfs(uintptr_t instance_ptr, struct statvfs* statvfs); + +int curvefs_lstat(uintptr_t instance_ptr, const char* path, struct stat* stat); + +int curvefs_fstat(uintptr_t instance_ptr, int fd, struct stat* stat); + +int curvefs_setattr(uintptr_t instance_ptr, + const char* path, + struct stat* stat, + int to_set); + +int curvefs_chmod(uintptr_t instance_ptr, const char* path, uint16_t mode); + +int curvefs_chown(uintptr_t instance_ptr, + const char* path, + uint32_t uid, + uint32_t gid); + +int curvefs_rename(uintptr_t instance_ptr, + const char* oldpath, + const char* newpath); +#ifdef __cplusplus +} +#endif + +#endif // CURVEFS_SDK_LIBCURVEFS_LIBCURVEFS_H_ diff --git a/curvefs/src/client/BUILD b/curvefs/src/client/BUILD index 677e32b0b6..450e8b6ced 100644 --- a/curvefs/src/client/BUILD +++ b/curvefs/src/client/BUILD @@ -32,6 +32,19 @@ cc_binary( ], ) +cc_library( + name = "filesystem", + srcs = glob([ + "filesystem/*.cpp", + "filesystem/*.h", + ]), + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + ":fuse_client_lib", + ], +) + cc_library( name = "filesystem_xattr", hdrs = ["filesystem/xattr.h"], @@ -74,6 +87,7 @@ cc_library( "//curvefs/src/common:curvefs_common", "//curvefs/src/client/lease:curvefs_lease", "//curvefs/src/client/kvclient:memcached_client_lib", + "//curvefs/src/client/logger:logger", ":filesystem_xattr", "//external:brpc", "//external:gflags", diff --git a/curvefs/src/client/client_operator.cpp b/curvefs/src/client/client_operator.cpp index 8d44a5150f..720ce65ee3 100644 --- a/curvefs/src/client/client_operator.cpp +++ b/curvefs/src/client/client_operator.cpp @@ -153,7 +153,7 @@ CURVEFS_ERROR RenameOperator::CheckOverwrite() { if (rc == CURVEFS_ERROR::OK && !dentrys.empty()) { LOG(ERROR) << "The directory is not empty" << ", dentry = (" << dstDentry_.ShortDebugString() << ")"; - rc = CURVEFS_ERROR::NOTEMPTY; + rc = CURVEFS_ERROR::NOT_EMPTY; } return rc; @@ -170,7 +170,7 @@ CURVEFS_ERROR RenameOperator::Precheck() { } rc = dentryManager_->GetDentry(newParentId_, newname_, &dstDentry_); - if (rc == CURVEFS_ERROR::NOTEXIST) { + if (rc == CURVEFS_ERROR::NOT_EXIST) { return CURVEFS_ERROR::OK; } else if (rc == CURVEFS_ERROR::OK) { oldInodeId_ = dstDentry_.inodeid(); @@ -191,7 +191,7 @@ CURVEFS_ERROR RenameOperator::RecordOldInodeInfo() { oldInodeType_ = attr.type(); } else { LOG_ERROR("GetInode", rc); - return CURVEFS_ERROR::NOTEXIST; + return CURVEFS_ERROR::NOT_EXIST; } } diff --git a/curvefs/src/client/common/config.cpp b/curvefs/src/client/common/config.cpp index ad4b29b97c..dc80177ab3 100644 --- a/curvefs/src/client/common/config.cpp +++ b/curvefs/src/client/common/config.cpp @@ -48,7 +48,7 @@ DEFINE_validator(fs_disableXattr, [](const char*, bool value) { return true; }); namespace curvefs { namespace client { namespace common { -static bool pass_bool(const char *, bool) { return true; } +static bool pass_bool(const char*, bool) { return true; } DEFINE_bool(enableCto, true, "acheieve cto consistency"); DEFINE_bool(useFakeS3, false, "Use fake s3 to inject more metadata for testing metaserver"); @@ -314,6 +314,40 @@ void InitKVClientManagerOpt(Configuration *conf, &config->getThreadPooln); } +void GetGids( + Configuration* c, const std::string& key, std::vector* gids) { + std::string str; + std::vector ss; + c->GetValueFatalIfFail(key, &str); + curve::common::SplitString(str, ",", &ss); + uint32_t gid; + for (const auto& s : ss) { + LOG_IF(FATAL, !curve::common::StringToUl(s, &gid)) + << "Invalid `" << key << "`: <" << s << ">"; + gids->push_back(static_cast(gid)); + } +} + +void GetUmask(Configuration* c, const std::string& key, uint16_t* umask) { + std::string str; + c->GetValueFatalIfFail(key, &str); + *umask = stoi(str, 0, 8); +} + +void InitVFSOption(Configuration* c, VFSOption* option) { + { // vfs cache option + auto o = &option->vfsCacheOption; + c->GetValueFatalIfFail("vfs.entryCache.lruSize", &o->entryCacheLruSize); + c->GetValueFatalIfFail("vfs.attrCache.lruSize", &o->attrCacheLruSize); + } + { // user permission option + auto o = &option->userPermissionOption; + c->GetValueFatalIfFail("vfs.userPermission.uid", &o->uid); + GetGids(c, "vfs.userPermission.gids", &o->gids); + GetUmask(c, "vfs.userPermission.umask", &o->umask); + } +} + void InitFileSystemOption(Configuration* c, FileSystemOption* option) { c->GetValueFatalIfFail("fs.cto", &option->cto); c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto); @@ -383,6 +417,7 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) { InitLeaseOpt(conf, &clientOption->leaseOpt); InitRefreshDataOpt(conf, &clientOption->refreshDataOption); InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt); + InitVFSOption(conf, &clientOption->vfsOption); InitFileSystemOption(conf, &clientOption->fileSystemOption); conf->GetValueFatalIfFail("fuseClient.listDentryLimit", diff --git a/curvefs/src/client/common/config.h b/curvefs/src/client/common/config.h index 3d0c91dd70..464b1550c4 100644 --- a/curvefs/src/client/common/config.h +++ b/curvefs/src/client/common/config.h @@ -25,9 +25,10 @@ #include #include +#include -#include "curvefs/src/client/common/common.h" #include "curvefs/proto/common.pb.h" +#include "curvefs/src/client/common/common.h" #include "src/client/config_info.h" #include "src/common/configuration.h" #include "src/common/s3_adapter.h" @@ -180,6 +181,24 @@ struct RefreshDataOption { uint32_t refreshDataIntervalSec = 30; }; +// { vfs option +struct UserPermissionOption { + uint32_t uid; + std::vector gids; + uint16_t umask; +}; + +struct VFSCacheOption { + uint32_t entryCacheLruSize; + uint32_t attrCacheLruSize; +}; + +struct VFSOption { + VFSCacheOption vfsCacheOption; + UserPermissionOption userPermissionOption; +}; +// } + // { filesystem option struct KernelCacheOption { uint32_t entryTimeoutSec; @@ -244,6 +263,7 @@ struct FuseClientOption { LeaseOpt leaseOpt; RefreshDataOption refreshDataOption; KVClientManagerOpt kvClientManagerOpt; + VFSOption vfsOption; FileSystemOption fileSystemOption; uint32_t listDentryLimit; diff --git a/curvefs/src/client/curve_fuse_op.cpp b/curvefs/src/client/curve_fuse_op.cpp index 78c567f589..dd249c8892 100644 --- a/curvefs/src/client/curve_fuse_op.cpp +++ b/curvefs/src/client/curve_fuse_op.cpp @@ -32,12 +32,13 @@ #include "curvefs/src/client/common/common.h" #include "curvefs/src/client/common/config.h" -#include "curvefs/src/client/filesystem/access_log.h" #include "curvefs/src/client/filesystem/error.h" #include "curvefs/src/client/filesystem/meta.h" +#include "curvefs/src/client/filesystem/xattr.h" #include "curvefs/src/client/fuse_client.h" #include "curvefs/src/client/fuse_s3_client.h" #include "curvefs/src/client/fuse_volume_client.h" +#include "curvefs/src/client/logger/access_log.h" #include "curvefs/src/client/metric/client_metric.h" #include "curvefs/src/client/rpcclient/base_client.h" #include "curvefs/src/client/rpcclient/mds_client.h" @@ -47,7 +48,6 @@ #include "curvefs/src/common/metric_utils.h" #include "src/common/configuration.h" #include "src/common/gflags_helper.h" -#include "curvefs/src/client/filesystem/xattr.h" using ::curve::common::Configuration; using ::curvefs::client::CURVEFS_ERROR; @@ -62,24 +62,23 @@ using ::curvefs::client::common::kWarmupCacheStorageType; using ::curvefs::client::common::kWarmupDataType; using ::curvefs::client::common::kWarmupOpType; using ::curvefs::client::common::WarmupStorageType; -using ::curvefs::client::filesystem::AccessLogGuard; using ::curvefs::client::filesystem::AttrOut; using ::curvefs::client::filesystem::EntryOut; using ::curvefs::client::filesystem::FileOut; -using ::curvefs::client::filesystem::InitAccessLog; -using ::curvefs::client::filesystem::Logger; +using ::curvefs::client::filesystem::IsListWarmupXAttr; +using ::curvefs::client::filesystem::IsWarmupXAttr; using ::curvefs::client::filesystem::StrAttr; using ::curvefs::client::filesystem::StrEntry; -using ::curvefs::client::filesystem::StrFormat; using ::curvefs::client::filesystem::StrMode; +using ::curvefs::client::logger::AccessLogGuard; +using ::curvefs::client::logger::InitAccessLog; +using ::curvefs::client::logger::StrFormat; using ::curvefs::client::metric::ClientOpMetric; using ::curvefs::client::metric::InflightGuard; using ::curvefs::client::rpcclient::MDSBaseClient; using ::curvefs::client::rpcclient::MdsClientImpl; using ::curvefs::client::warmup::WarmupProgress; using ::curvefs::common::LatencyUpdater; -using ::curvefs::client::filesystem::IsWarmupXAttr; -using ::curvefs::client::filesystem::IsListWarmupXAttr; using ::curvefs::common::FLAGS_vlog_level; @@ -1006,5 +1005,5 @@ void FuseOpBmap(fuse_req_t req, auto client = Client(); auto fs = client->GetFileSystem(); - return fs->ReplyError(req, CURVEFS_ERROR::NOTSUPPORT); + return fs->ReplyError(req, CURVEFS_ERROR::NOT_SUPPORT); } diff --git a/curvefs/src/client/filesystem/.clang-format b/curvefs/src/client/filesystem/.clang-format new file mode 100644 index 0000000000..a5660a57d8 --- /dev/null +++ b/curvefs/src/client/filesystem/.clang-format @@ -0,0 +1,26 @@ +# I don't like current clang-formtter's specified style and +# and i think my style is more intuitive than it :), so I use +# this file to disable format, let us give some code snippet to prove it: +# +# my: +# std::vector tests { +# { " abc", "abc" }, +# { "abc ", "abc" }, +# { " abc ", "abc" }, +# { " a bc ", "a bc" }, +# { "a b c", "a b c" }, +# { " ", "" }, +# +# +# clang-formatter: +# std::vector tests{ +# {" abc", "abc"}, {"abc ", "abc"}, {" abc ", "abc"}, +# {" a bc ", "a bc"}, {"a b c", "a b c"}, {" ", ""}, +# }; +# +# Maybe you have different opinions, but please have a little mercy for me. +# +# by Wine93, 2023-10-16 + +DisableFormat: true +SortIncludes: Never diff --git a/curvefs/src/client/filesystem/defer_sync.cpp b/curvefs/src/client/filesystem/defer_sync.cpp index a69cdeb3ba..2b33e33ba4 100644 --- a/curvefs/src/client/filesystem/defer_sync.cpp +++ b/curvefs/src/client/filesystem/defer_sync.cpp @@ -81,6 +81,17 @@ void DeferSync::Push(const std::shared_ptr& inode) { inodes_.emplace_back(inode); } +bool DeferSync::IsDefered(Ino ino, InodeAttr* attr) { + LockGuard lk(mutex_); + for (const auto& inode : inodes_) { + if (inode->GetInodeId() == ino) { + inode->GetInodeAttr(attr); + return true; + } + } + return false; +} + } // namespace filesystem } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/filesystem/defer_sync.h b/curvefs/src/client/filesystem/defer_sync.h index 0bd59bb9bc..41264a7e00 100644 --- a/curvefs/src/client/filesystem/defer_sync.h +++ b/curvefs/src/client/filesystem/defer_sync.h @@ -51,6 +51,8 @@ class DeferSync { void Push(const std::shared_ptr& inode); + bool IsDefered(Ino ino, InodeAttr* attr); + private: void SyncTask(); diff --git a/curvefs/src/client/filesystem/dir_cache.cpp b/curvefs/src/client/filesystem/dir_cache.cpp index 64133d33ff..f71a1651b8 100644 --- a/curvefs/src/client/filesystem/dir_cache.cpp +++ b/curvefs/src/client/filesystem/dir_cache.cpp @@ -57,6 +57,15 @@ bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) { return true; } +bool DirEntryList::At(uint32_t index, DirEntry* dirEntry) { + ReadLockGuard lk(rwlock_); + if (index >= entries_.size()) { + return false; + } + *dirEntry = entries_[index]; + return true; +} + bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) { WriteLockGuard lk(rwlock_); auto iter = index_.find(ino); diff --git a/curvefs/src/client/filesystem/dir_cache.h b/curvefs/src/client/filesystem/dir_cache.h index 0fd0e6a7dd..6c07ae1221 100644 --- a/curvefs/src/client/filesystem/dir_cache.h +++ b/curvefs/src/client/filesystem/dir_cache.h @@ -61,6 +61,8 @@ class DirEntryList { bool Get(Ino ino, DirEntry* dirEntry); + bool At(uint32_t index, DirEntry* dirEntry); + bool UpdateAttr(Ino ino, const InodeAttr& attr); bool UpdateLength(Ino ino, const InodeAttr& open); diff --git a/curvefs/src/client/filesystem/error.cpp b/curvefs/src/client/filesystem/error.cpp index f8a677497b..1a77f467bd 100644 --- a/curvefs/src/client/filesystem/error.cpp +++ b/curvefs/src/client/filesystem/error.cpp @@ -35,15 +35,15 @@ static const std::map> errors = { { CURVEFS_ERROR::INTERNAL, { EIO, "internal error" } }, { CURVEFS_ERROR::UNKNOWN, { -1, "unknown" } }, { CURVEFS_ERROR::EXISTS, { EEXIST, "inode or dentry already exist" } }, - { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } }, + { CURVEFS_ERROR::NOT_EXIST, { ENOENT, "inode or dentry not exist" } }, { CURVEFS_ERROR::NO_SPACE, { ENOSPC, "no space to alloc" } }, { CURVEFS_ERROR::BAD_FD, { EBADF, "bad file number" } }, - { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } }, - { CURVEFS_ERROR::NOPERMISSION, { EACCES, "permission denied" } }, - { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } }, - { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } }, - { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } }, - { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } }, + { CURVEFS_ERROR::INVALID_PARAM , { EINVAL , "invalid argument" } }, + { CURVEFS_ERROR::NO_PERMISSION, { EACCES, "permission denied" } }, + { CURVEFS_ERROR::NOT_EMPTY, { ENOTEMPTY, "directory not empty" } }, + { CURVEFS_ERROR::NO_FLUSH, { -1, "no flush" } }, + { CURVEFS_ERROR::NOT_SUPPORT, { EOPNOTSUPP, "operation not supported" } }, + { CURVEFS_ERROR::NAME_TOO_LONG, { ENAMETOOLONG, "file name too long" } }, { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } }, { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } }, { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } }, @@ -51,6 +51,8 @@ static const std::map> errors = { { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } }, { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } }, { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } }, + { CURVEFS_ERROR::END_OF_FILE, { EOF, "end of file" } }, + { CURVEFS_ERROR::NOT_A_DIRECTORY, { ENOTDIR, "not a directory" } }, }; std::string StrErr(CURVEFS_ERROR code) { @@ -86,8 +88,8 @@ std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) { CURVEFS_ERROR ToFSError(MetaStatusCode code) { static std::map errs = { { MetaStatusCode::OK, CURVEFS_ERROR::OK }, - { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST }, - { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM }, + { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOT_EXIST }, + { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALID_PARAM }, { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS }, { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS }, { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL }, diff --git a/curvefs/src/client/filesystem/error.h b/curvefs/src/client/filesystem/error.h index 1f837ca9d6..75efe38283 100644 --- a/curvefs/src/client/filesystem/error.h +++ b/curvefs/src/client/filesystem/error.h @@ -40,23 +40,25 @@ enum class CURVEFS_ERROR { INTERNAL = -1, UNKNOWN = -2, EXISTS = -3, - NOTEXIST = -4, + NOT_EXIST = -4, NO_SPACE = -5, BAD_FD = -6, - INVALIDPARAM = -7, - NOPERMISSION = -8, - NOTEMPTY = -9, - NOFLUSH = -10, - NOTSUPPORT = -11, - NAMETOOLONG = -12, + INVALID_PARAM = -7, + NO_PERMISSION = -8, + NOT_EMPTY = -9, + NO_FLUSH = -10, + NOT_SUPPORT = -10, + NAME_TOO_LONG = -12, MOUNT_POINT_EXIST = -13, MOUNT_FAILED = -14, OUT_OF_RANGE = -15, NODATA = -16, IO_ERROR = -17, - CACHETOOSMALL = -18, + CACHE_TOO_SMALL = -18, STALE = -19, NOSYS = -20, + END_OF_FILE = -21, + NOT_A_DIRECTORY = -22, }; std::string StrErr(CURVEFS_ERROR code); diff --git a/curvefs/src/client/filesystem/filesystem.cpp b/curvefs/src/client/filesystem/filesystem.cpp index 795c896a5b..2aad563051 100644 --- a/curvefs/src/client/filesystem/filesystem.cpp +++ b/curvefs/src/client/filesystem/filesystem.cpp @@ -234,30 +234,34 @@ FileSystemMember FileSystem::BorrowMember() { } // fuse request* -CURVEFS_ERROR FileSystem::Lookup(Request req, - Ino parent, +CURVEFS_ERROR FileSystem::Lookup(Ino parent, const std::string& name, EntryOut* entryOut) { if (name.size() > option_.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } bool yes = negative_->Get(parent, name); if (yes) { - return CURVEFS_ERROR::NOTEXIST; + return CURVEFS_ERROR::NOT_EXIST; } auto rc = rpc_->Lookup(parent, name, entryOut); if (rc == CURVEFS_ERROR::OK) { negative_->Delete(parent, name); - } else if (rc == CURVEFS_ERROR::NOTEXIST) { + } else if (rc == CURVEFS_ERROR::NOT_EXIST) { negative_->Put(parent, name); } return rc; } -CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) { +CURVEFS_ERROR FileSystem::GetAttr(Ino ino, AttrOut* attrOut) { InodeAttr attr; + if (!option_.cto && deferSync_->IsDefered(ino, &attr)) { + *attrOut = AttrOut(attr); + return CURVEFS_ERROR::OK; + } + auto rc = rpc_->GetAttr(ino, &attr); if (rc == CURVEFS_ERROR::OK) { *attrOut = AttrOut(attr); @@ -265,7 +269,7 @@ CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) { return rc; } -CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) { +CURVEFS_ERROR FileSystem::OpenDir(Ino ino, FileInfo* fi) { InodeAttr attr; CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr); if (rc != CURVEFS_ERROR::OK) { @@ -287,8 +291,7 @@ CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) { return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FileSystem::ReadDir(Request req, - Ino ino, +CURVEFS_ERROR FileSystem::ReadDir(Ino ino, FileInfo* fi, std::shared_ptr* entries) { bool yes = dirCache_->Get(ino, entries); @@ -306,12 +309,12 @@ CURVEFS_ERROR FileSystem::ReadDir(Request req, return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) { +CURVEFS_ERROR FileSystem::ReleaseDir(Ino ino, FileInfo* fi) { ReleaseHandler(fi->fh); return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) { +CURVEFS_ERROR FileSystem::Open(Ino ino, FileInfo* fi) { std::shared_ptr inode; bool yes = openFiles_->IsOpened(ino, &inode); if (yes) { @@ -343,7 +346,7 @@ CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) { return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) { +CURVEFS_ERROR FileSystem::Release(Ino ino) { openFiles_->Close(ino); return CURVEFS_ERROR::OK; } diff --git a/curvefs/src/client/filesystem/filesystem.h b/curvefs/src/client/filesystem/filesystem.h index c2d48db708..eaa6cacc2a 100644 --- a/curvefs/src/client/filesystem/filesystem.h +++ b/curvefs/src/client/filesystem/filesystem.h @@ -67,25 +67,23 @@ class FileSystem { void Destory(); // fuse request - CURVEFS_ERROR Lookup(Request req, - Ino parent, + CURVEFS_ERROR Lookup(Ino parent, const std::string& name, EntryOut* entryOut); - CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut); + CURVEFS_ERROR GetAttr(Ino ino, AttrOut* attrOut); - CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi); + CURVEFS_ERROR OpenDir(Ino ino, FileInfo* fi); - CURVEFS_ERROR ReadDir(Request req, - Ino ino, + CURVEFS_ERROR ReadDir(Ino ino, FileInfo* fi, std::shared_ptr* entries); - CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi); + CURVEFS_ERROR ReleaseDir(Ino ino, FileInfo* fi); - CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi); + CURVEFS_ERROR Open(Ino ino, FileInfo* fi); - CURVEFS_ERROR Release(Request req, Ino ino); + CURVEFS_ERROR Release(Ino ino); // fuse reply: we control all replies to vfs layer in same entrance. void ReplyError(Request req, CURVEFS_ERROR code); @@ -129,15 +127,6 @@ class FileSystem { void ReleaseHandler(uint64_t fh); - // utility: others - FileSystemMember BorrowMember(); - - private: - FRIEND_TEST(FileSystemTest, Attr2Stat); - FRIEND_TEST(FileSystemTest, Entry2Param); - FRIEND_TEST(FileSystemTest, SetEntryTimeout); - FRIEND_TEST(FileSystemTest, SetAttrTimeout); - // utility: convert to system type. void Attr2Stat(InodeAttr* attr, struct stat* stat); @@ -148,6 +137,9 @@ class FileSystem { void SetAttrTimeout(AttrOut* attrOut); + // utility: others + FileSystemMember BorrowMember(); + private: FileSystemOption option_; ExternalMember member; diff --git a/curvefs/src/client/filesystem/lookup_cache.cpp b/curvefs/src/client/filesystem/lookup_cache.cpp index d5c473a114..7e49b0db1c 100644 --- a/curvefs/src/client/filesystem/lookup_cache.cpp +++ b/curvefs/src/client/filesystem/lookup_cache.cpp @@ -83,7 +83,7 @@ bool LookupCache::Put(Ino parent, const std::string& name) { if (yes) { entry.uses++; } else { - entry.uses = 0; + entry.uses = 1; } entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0); diff --git a/curvefs/src/client/filesystem/meta.cpp b/curvefs/src/client/filesystem/meta.cpp index 47bbeced83..4c835fd4d7 100644 --- a/curvefs/src/client/filesystem/meta.cpp +++ b/curvefs/src/client/filesystem/meta.cpp @@ -123,6 +123,10 @@ std::string StrAttr(AttrOut attrOut) { return Attr2Str(attrOut.attr); } +std::string StrAttr(InodeAttr attr) { + return Attr2Str(attr); +} + } // namespace filesystem } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/filesystem/meta.h b/curvefs/src/client/filesystem/meta.h index 1165a1fb4f..c10ec1b0d9 100644 --- a/curvefs/src/client/filesystem/meta.h +++ b/curvefs/src/client/filesystem/meta.h @@ -171,6 +171,8 @@ std::string StrEntry(EntryOut entryOut); std::string StrAttr(AttrOut attrOut); +std::string StrAttr(InodeAttr attr); + } // namespace filesystem } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/filesystem/rpc_client.cpp b/curvefs/src/client/filesystem/rpc_client.cpp index a7ec92764d..236c50687f 100644 --- a/curvefs/src/client/filesystem/rpc_client.cpp +++ b/curvefs/src/client/filesystem/rpc_client.cpp @@ -54,7 +54,7 @@ CURVEFS_ERROR RPCClient::Lookup(Ino parent, Dentry dentry; CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry); if (rc != CURVEFS_ERROR::OK) { - if (rc != CURVEFS_ERROR::NOTEXIST) { + if (rc != CURVEFS_ERROR::NOT_EXIST) { LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc << ", parent = " << parent << ", name = " << name; } diff --git a/curvefs/src/client/filesystem/utils.cpp b/curvefs/src/client/filesystem/utils.cpp index 0a6b6c38e0..dd4e2a5c6c 100644 --- a/curvefs/src/client/filesystem/utils.cpp +++ b/curvefs/src/client/filesystem/utils.cpp @@ -40,7 +40,7 @@ bool IsVolmeFile(const InodeAttr& attr) { return attr.type() == FsFileType::TYPE_FILE; } -bool IsSymLink(const InodeAttr& attr) { +bool IsSymlink(const InodeAttr& attr) { return attr.type() == FsFileType::TYPE_SYM_LINK; } diff --git a/curvefs/src/client/filesystem/utils.h b/curvefs/src/client/filesystem/utils.h index 8f37104e1d..706fa3d271 100644 --- a/curvefs/src/client/filesystem/utils.h +++ b/curvefs/src/client/filesystem/utils.h @@ -42,7 +42,7 @@ bool IsS3File(const InodeAttr& attr); bool IsVolmeFile(const InodeAttr& attr); // symbol link -bool IsSymLink(const InodeAttr& attr); +bool IsSymlink(const InodeAttr& attr); struct TimeSpec AttrMtime(const InodeAttr& attr); diff --git a/curvefs/src/client/fuse_client.cpp b/curvefs/src/client/fuse_client.cpp index b944b37f15..e3159fcc81 100644 --- a/curvefs/src/client/fuse_client.cpp +++ b/curvefs/src/client/fuse_client.cpp @@ -23,53 +23,55 @@ #include "curvefs/src/client/fuse_client.h" -#include #include #include +#include #include -#include -#include #include +#include #include #include +#include +#include "absl/memory/memory.h" #include "curvefs/proto/mds.pb.h" +#include "curvefs/src/client/client_operator.h" #include "curvefs/src/client/common/common.h" #include "curvefs/src/client/filesystem/error.h" #include "curvefs/src/client/filesystem/xattr.h" #include "curvefs/src/client/fuse_common.h" -#include "curvefs/src/client/client_operator.h" #include "curvefs/src/client/inode_wrapper.h" #include "curvefs/src/client/warmup/warmup_manager.h" #include "curvefs/src/client/xattr_manager.h" #include "curvefs/src/common/define.h" -#include "src/common/net_common.h" -#include "src/common/dummyserver.h" #include "src/client/client_common.h" -#include "absl/memory/memory.h" +#include "src/common/dummyserver.h" +#include "src/common/net_common.h" #define PORT_LIMIT 65535 -using ::curvefs::common::S3Info; -using ::curvefs::common::Volume; -using ::curvefs::mds::topology::PartitionTxId; -using ::curvefs::mds::FSStatusCode_Name; -using ::curvefs::client::filesystem::ExternalMember; +using ::curvefs::client::common::FuseClientOption; using ::curvefs::client::filesystem::DirEntry; using ::curvefs::client::filesystem::DirEntryList; +using ::curvefs::client::filesystem::ExternalMember; using ::curvefs::client::filesystem::FileOut; +using ::curvefs::client::filesystem::IsSpecialXAttr; using ::curvefs::client::filesystem::MAX_XATTR_NAME_LENGTH; using ::curvefs::client::filesystem::MAX_XATTR_VALUE_LENGTH; -using ::curvefs::client::filesystem::XATTR_DIR_FILES; -using ::curvefs::client::filesystem::XATTR_DIR_SUBDIRS; using ::curvefs::client::filesystem::XATTR_DIR_ENTRIES; using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; -using ::curvefs::client::filesystem::XATTR_DIR_RFILES; +using ::curvefs::client::filesystem::XATTR_DIR_FILES; +using ::curvefs::client::filesystem::XATTR_DIR_PREFIX; +using ::curvefs::client::filesystem::XATTR_DIR_RENTRIES; using ::curvefs::client::filesystem::XATTR_DIR_RFBYTES; +using ::curvefs::client::filesystem::XATTR_DIR_RFILES; using ::curvefs::client::filesystem::XATTR_DIR_RSUBDIRS; -using ::curvefs::client::filesystem::XATTR_DIR_RENTRIES; -using ::curvefs::client::filesystem::XATTR_DIR_PREFIX; -using ::curvefs::client::filesystem::IsSpecialXAttr; +using ::curvefs::client::filesystem::XATTR_DIR_SUBDIRS; +using ::curvefs::common::FSType; +using ::curvefs::common::S3Info; +using ::curvefs::common::Volume; +using ::curvefs::mds::FSStatusCode_Name; +using ::curvefs::mds::topology::PartitionTxId; #define RETURN_IF_UNSUCCESS(action) \ do { \ @@ -279,8 +281,8 @@ CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req, fuse_ino_t parent, const char* name, EntryOut* entryOut) { - CURVEFS_ERROR rc = fs_->Lookup(req, parent, name, entryOut); - if (rc != CURVEFS_ERROR::OK && rc != CURVEFS_ERROR::NOTEXIST) { + CURVEFS_ERROR rc = fs_->Lookup(parent, name, entryOut); + if (rc != CURVEFS_ERROR::OK && rc != CURVEFS_ERROR::NOT_EXIST) { LOG(ERROR) << "Lookup() failed, retCode = " << rc << ", parent = " << parent << ", name = " << name; } @@ -329,8 +331,8 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req, // update parent summary info const Inode *inode = inodeWrapper->GetInodeLocked(); XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, - std::to_string(length)}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(length)}); for (const auto &it : inode->parent()) { auto tret = xattrManager_->UpdateParentInodeXattr( it, xattr, false); @@ -343,7 +345,7 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req, } inodeWrapper->GetInodeAttrLocked(&fileOut->attr); } else { - return CURVEFS_ERROR::NOPERMISSION; + return CURVEFS_ERROR::NO_PERMISSION; } } return CURVEFS_ERROR::OK; @@ -353,7 +355,7 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi, FileOut* fileOut) { - CURVEFS_ERROR rc = fs_->Open(req, ino, fi); + CURVEFS_ERROR rc = fs_->Open(ino, fi); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "open(" << ino << ") failed, retCode = " << rc; return rc; @@ -400,19 +402,19 @@ CURVEFS_ERROR FuseClient::MakeNode( bool internal, std::shared_ptr& inodeWrapper) { if (strlen(name) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } // check if node is recycle or under recycle if (!internal && strcmp(name, RECYCLENAME) == 0 && parent == ROOTINODEID) { LOG(WARNING) << "Can not make node " << RECYCLENAME << " under root dir."; - return CURVEFS_ERROR::NOPERMISSION; + return CURVEFS_ERROR::NO_PERMISSION; } if (!internal && parent == RECYCLEINODEID) { LOG(WARNING) << "Can not make node under recycle."; - return CURVEFS_ERROR::NOPERMISSION; + return CURVEFS_ERROR::NO_PERMISSION; } const struct fuse_ctx *ctx = fuse_req_ctx(req); @@ -423,6 +425,7 @@ CURVEFS_ERROR FuseClient::MakeNode( } else { param.length = 0; } + param.uid = ctx->uid; param.gid = ctx->gid; param.mode = mode; @@ -480,8 +483,8 @@ CURVEFS_ERROR FuseClient::MakeNode( } else { xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true); if (tret != CURVEFS_ERROR::OK) { LOG(ERROR) << "UpdateParentInodeXattr failed," @@ -559,8 +562,8 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent, } else { xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, false); if (tret != CURVEFS_ERROR::OK) { LOG(WARNING) << "UpdateParentInodeXattr failed," @@ -587,7 +590,7 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, ManageInodeType manageType, EntryOut* entryOut) { if (strlen(name) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } InodeParam param; @@ -650,8 +653,8 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, } else { xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); } - xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true); if (tret != CURVEFS_ERROR::OK) { LOG(ERROR) << "UpdateParentInodeXattr failed," @@ -666,12 +669,12 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, CURVEFS_ERROR FuseClient::GetOrCreateRecycleDir(fuse_req_t req, Dentry *out) { auto ret = dentryManager_->GetDentry(ROOTINODEID, RECYCLENAME, out); - if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOTEXIST) { + if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOT_EXIST) { LOG(ERROR) << "dentryManager_ GetDentry fail, ret = " << ret << ", inode = " << ROOTINODEID << ", name = " << RECYCLENAME; return ret; - } else if (ret == CURVEFS_ERROR::NOTEXIST) { + } else if (ret == CURVEFS_ERROR::NOT_EXIST) { LOG(INFO) << "recycle dir is not exist, create " << RECYCLENAME << ", parentid = " << ROOTINODEID; EntryOut entryOut; @@ -708,12 +711,12 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino, uint64_t recycleTimeDirIno; ret = dentryManager_->GetDentry(RECYCLEINODEID, recycleTimeDirName.c_str(), &dentry); - if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOTEXIST) { + if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOT_EXIST) { LOG(ERROR) << "dentryManager_ GetDentry fail, ret = " << ret << ", inode = " << RECYCLEINODEID << ", name = " << recycleTimeDirName; return ret; - } else if (ret == CURVEFS_ERROR::NOTEXIST) { + } else if (ret == CURVEFS_ERROR::NOT_EXIST) { std::shared_ptr inode; bool internal = true; ret = MakeNode(req, RECYCLEINODEID, recycleTimeDirName.c_str(), @@ -772,13 +775,13 @@ bool FuseClient::ShouldMoveToRecycle(fuse_ino_t parent) { CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent, const char *name, FsFileType type) { if (strlen(name) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } // check if node is recycle or recycle time dir if ((strcmp(name, RECYCLENAME) == 0 && parent == ROOTINODEID) || parent == RECYCLEINODEID) { - return CURVEFS_ERROR::NOPERMISSION; + return CURVEFS_ERROR::NO_PERMISSION; } Dentry dentry; @@ -803,7 +806,7 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent, } if (!dentryList.empty()) { LOG(ERROR) << "rmdir not empty"; - return CURVEFS_ERROR::NOTEMPTY; + return CURVEFS_ERROR::NOT_EMPTY; } } @@ -830,7 +833,7 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent, CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - CURVEFS_ERROR rc = fs_->OpenDir(req, ino, fi); + CURVEFS_ERROR rc = fs_->OpenDir(ino, fi); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "opendir() failed, retCode = " << rc << ", ino = " << ino; @@ -850,7 +853,7 @@ CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req, DirBufferHead* buffer = handler->buffer; if (!handler->padding) { auto entries = std::make_shared(); - CURVEFS_ERROR rc = fs_->ReadDir(req, ino, fi, &entries); + CURVEFS_ERROR rc = fs_->ReadDir(ino, fi, &entries); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "readdir() failed, retCode = " << rc << ", ino = " << ino << ", fh = " << fi->fh; @@ -880,7 +883,7 @@ CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req, CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) { - CURVEFS_ERROR rc = fs_->ReleaseDir(req, ino, fi); + CURVEFS_ERROR rc = fs_->ReleaseDir(ino, fi); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "releasedir() failed, retCode = " << rc << ", ino = " << ino; @@ -899,7 +902,7 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent, // is only used in linux interface renameat(), not required by posix, // we can ignore it now if (flags != 0) { - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } uint64_t maxNameLength = option_.fileSystemOption.maxNameLength; @@ -908,7 +911,7 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent, << ", name len = " << strlen(name) << ", new name = " << newname << ", new name len = " << strlen(newname) << ", maxNameLength = " << maxNameLength; - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } auto renameOp = @@ -950,7 +953,7 @@ CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi, AttrOut* attrOut) { - CURVEFS_ERROR rc = fs_->GetAttr(req, ino, attrOut); + CURVEFS_ERROR rc = fs_->GetAttr(ino, attrOut); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "getattr() fail, retCode = " << rc << ", ino = " << ino; @@ -1026,8 +1029,8 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, // update parent summary info const Inode* inode = inodeWrapper->GetInodeLocked(); XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, - std::to_string(std::abs(changeSize))}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(std::abs(changeSize))}); bool direction = changeSize > 0; for (const auto &it : inode->parent()) { auto tret = xattrManager_->UpdateParentInodeXattr( @@ -1146,36 +1149,26 @@ CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino, // get xattr key for (const auto &it : inodeAttr.xattr()) { + if (option_.fileSystemOption.disableXAttr && + !IsSpecialXAttr(it.first)) { + continue; + } // +1 because, the format is key\0key\0 *realSize += it.first.length() + 1; } - // add summary xattr key - if (inodeAttr.type() == FsFileType::TYPE_DIRECTORY) { - *realSize += strlen(XATTR_DIR_RFILES) + 1; - *realSize += strlen(XATTR_DIR_RSUBDIRS) + 1; - *realSize += strlen(XATTR_DIR_RENTRIES) + 1; - *realSize += strlen(XATTR_DIR_RFBYTES) + 1; - } - if (size == 0) { return CURVEFS_ERROR::OK; } else if (size >= *realSize) { for (const auto &it : inodeAttr.xattr()) { + if (option_.fileSystemOption.disableXAttr && + !IsSpecialXAttr(it.first)) { + continue; + } auto tsize = it.first.length() + 1; memcpy(value, it.first.c_str(), tsize); value += tsize; } - if (inodeAttr.type() == FsFileType::TYPE_DIRECTORY) { - memcpy(value, XATTR_DIR_RFILES, strlen(XATTR_DIR_RFILES) + 1); - value += strlen(XATTR_DIR_RFILES) + 1; - memcpy(value, XATTR_DIR_RSUBDIRS, strlen(XATTR_DIR_RSUBDIRS) + 1); - value += strlen(XATTR_DIR_RSUBDIRS) + 1; - memcpy(value, XATTR_DIR_RENTRIES, strlen(XATTR_DIR_RENTRIES) + 1); - value += strlen(XATTR_DIR_RENTRIES) + 1; - memcpy(value, XATTR_DIR_RFBYTES, strlen(XATTR_DIR_RFBYTES) + 1); - value += strlen(XATTR_DIR_RFBYTES) + 1; - } return CURVEFS_ERROR::OK; } return CURVEFS_ERROR::OUT_OF_RANGE; @@ -1187,7 +1180,7 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char* name, EntryOut* entryOut) { if (strlen(name) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } const struct fuse_ctx *ctx = fuse_req_ctx(req); InodeParam param; @@ -1234,8 +1227,8 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, XAttr xattr; xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true); if (tret != CURVEFS_ERROR::OK) { LOG(ERROR) << "UpdateParentInodeXattr failed," @@ -1255,7 +1248,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, FsFileType type, EntryOut* entryOut) { if (strlen(newname) > option_.fileSystemOption.maxNameLength) { - return CURVEFS_ERROR::NAMETOOLONG; + return CURVEFS_ERROR::NAME_TOO_LONG; } std::shared_ptr inodeWrapper; CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper); @@ -1295,8 +1288,8 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, XAttr xattr; xattr.mutable_xattrinfos()->insert({XATTR_DIR_ENTRIES, "1"}); xattr.mutable_xattrinfos()->insert({XATTR_DIR_FILES, "1"}); - xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, - std::to_string(inodeWrapper->GetLength())}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(inodeWrapper->GetLength())}); auto tret = xattrManager_->UpdateParentInodeXattr( newparent, xattr, true); if (tret != CURVEFS_ERROR::OK) { @@ -1328,7 +1321,7 @@ CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino, CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) { - CURVEFS_ERROR rc = fs_->Release(req, ino); + CURVEFS_ERROR rc = fs_->Release(ino); if (rc != CURVEFS_ERROR::OK) { LOG(ERROR) << "release() failed, ino = " << ino; } diff --git a/curvefs/src/client/fuse_client.h b/curvefs/src/client/fuse_client.h index ea8971f9ba..13adc3c591 100644 --- a/curvefs/src/client/fuse_client.h +++ b/curvefs/src/client/fuse_client.h @@ -43,6 +43,7 @@ #include "curvefs/src/client/common/config.h" #include "curvefs/src/client/dentry_cache_manager.h" #include "curvefs/src/client/dir_buffer.h" +#include "curvefs/src/client/filesystem/error.h" #include "curvefs/src/client/filesystem/filesystem.h" #include "curvefs/src/client/filesystem/meta.h" #include "curvefs/src/client/fuse_common.h" @@ -58,6 +59,7 @@ #include "curvefs/src/common/fast_align.h" #include "curvefs/src/common/s3util.h" #include "src/common/concurrent/concurrent.h" +#include "src/common/configuration.h" #include "src/common/throttle.h" #define DirectIOAlignment 512 @@ -65,10 +67,11 @@ using ::curve::common::Atomic; using ::curve::common::InterruptibleSleeper; using ::curve::common::Thread; +using ::curvefs::client::filesystem::CURVEFS_ERROR; +using ::curvefs::client::metric::FSMetric; using ::curvefs::common::FSType; using ::curvefs::metaserver::DentryFlag; using ::curvefs::metaserver::ManageInodeType; -using ::curvefs::client::metric::FSMetric; namespace curvefs { namespace client { @@ -79,15 +82,16 @@ class WarmupManager; } using common::FuseClientOption; +using ::curve::common::Configuration; +using ::curvefs::client::filesystem::AttrOut; +using ::curvefs::client::filesystem::EntryOut; +using ::curvefs::client::filesystem::FileOut; +using ::curvefs::client::filesystem::FileSystem; using rpcclient::MDSBaseClient; using rpcclient::MdsClient; using rpcclient::MdsClientImpl; using rpcclient::MetaServerClient; using rpcclient::MetaServerClientImpl; -using ::curvefs::client::filesystem::FileSystem; -using ::curvefs::client::filesystem::EntryOut; -using ::curvefs::client::filesystem::AttrOut; -using ::curvefs::client::filesystem::FileOut; using curvefs::common::is_aligned; @@ -294,13 +298,9 @@ class FuseClient { } } - std::shared_ptr GetFsInfo() { - return fsInfo_; - } + std::shared_ptr GetFsInfo() { return fsInfo_; } - std::shared_ptr GetFileSystem() { - return fs_; - } + virtual std::shared_ptr GetFileSystem() { return fs_; } virtual void FlushAll(); diff --git a/curvefs/src/client/fuse_common.h b/curvefs/src/client/fuse_common.h index 816bc7b5fb..d033afc6bc 100644 --- a/curvefs/src/client/fuse_common.h +++ b/curvefs/src/client/fuse_common.h @@ -26,9 +26,9 @@ #define FUSE_USE_VERSION 34 -#include -#include #include +#include +#include #ifdef __cplusplus extern "C" { diff --git a/curvefs/src/client/fuse_s3_client.cpp b/curvefs/src/client/fuse_s3_client.cpp index 216a65bf0b..508f1a745e 100644 --- a/curvefs/src/client/fuse_s3_client.cpp +++ b/curvefs/src/client/fuse_s3_client.cpp @@ -26,8 +26,8 @@ #include #include -#include "curvefs/src/client/kvclient/memcache_client.h" #include "curvefs/src/client/filesystem/xattr.h" +#include "curvefs/src/client/kvclient/memcache_client.h" namespace curvefs { namespace client { @@ -43,11 +43,11 @@ DECLARE_bool(supportKVcache); namespace curvefs { namespace client { -using curvefs::client::common::FLAGS_supportKVcache; using curvefs::client::common::FLAGS_enableCto; +using curvefs::client::common::FLAGS_supportKVcache; +using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; using curvefs::mds::topology::MemcacheClusterInfo; using curvefs::mds::topology::MemcacheServerInfo; -using ::curvefs::client::filesystem::XATTR_DIR_FBYTES; CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) { FuseClientOption opt(option); @@ -77,7 +77,7 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) { LOG(ERROR) << "writeCacheMaxByte is too small" << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)" ", writeCacheMaxByte = " << writeCacheMaxByte; - return CURVEFS_ERROR::CACHETOOSMALL; + return CURVEFS_ERROR::CACHE_TOO_SMALL; } auto fsCacheManager = std::make_shared( @@ -166,7 +166,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, if (fi->flags & O_DIRECT) { if (!(is_aligned(off, DirectIOAlignment) && is_aligned(size, DirectIOAlignment))) - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } uint64_t start = butil::cpuwide_time_us(); int wRet = s3Adaptor_->Write(ino, off, size, buf); @@ -206,8 +206,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, if (enableSumInDir_ && changeSize != 0) { const Inode* inode = inodeWrapper->GetInodeLocked(); XAttr xattr; - xattr.mutable_xattrinfos()->insert({XATTR_DIR_FBYTES, - std::to_string(changeSize)}); + xattr.mutable_xattrinfos()->insert( + {XATTR_DIR_FBYTES, std::to_string(changeSize)}); for (const auto &it : inode->parent()) { auto tret = xattrManager_->UpdateParentInodeXattr(it, xattr, true); if (tret != CURVEFS_ERROR::OK) { @@ -240,7 +240,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino, if (fi->flags & O_DIRECT) { if (!(is_aligned(off, DirectIOAlignment) && is_aligned(size, DirectIOAlignment))) - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } uint64_t start = butil::cpuwide_time_us(); diff --git a/curvefs/src/client/fuse_volume_client.cpp b/curvefs/src/client/fuse_volume_client.cpp index fc270a0282..26bcecdaa7 100644 --- a/curvefs/src/client/fuse_volume_client.cpp +++ b/curvefs/src/client/fuse_volume_client.cpp @@ -150,7 +150,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req, if (!(is_aligned(off, DirectIOAlignment) && is_aligned(size, DirectIOAlignment))) { fsMetric_->userWrite.eps.count << 1; - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } } @@ -208,7 +208,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req, is_aligned(size, DirectIOAlignment))) { fsMetric_->userRead.eps.count << 1; - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } } diff --git a/curvefs/src/client/inode_cache_manager.cpp b/curvefs/src/client/inode_cache_manager.cpp index b71ac2ffa7..e8601c1526 100644 --- a/curvefs/src/client/inode_cache_manager.cpp +++ b/curvefs/src/client/inode_cache_manager.cpp @@ -147,7 +147,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync( // split inodeIds by partitionId and batch limit std::vector> inodeGroups; if (!metaClient_->SplitRequestInodes(fsId_, *inodeIds, &inodeGroups)) { - return CURVEFS_ERROR::NOTEXIST; + return CURVEFS_ERROR::NOT_EXIST; } ::curve::common::Mutex mutex; diff --git a/curvefs/src/client/logger/BUILD b/curvefs/src/client/logger/BUILD new file mode 100644 index 0000000000..e5a115c166 --- /dev/null +++ b/curvefs/src/client/logger/BUILD @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_DEFAULT_COPTS") + +cc_library( + name = "logger", + srcs = glob(["*.cpp"]), + hdrs = glob(["*.h"]), + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//external:glog", + "@spdlog//:spdlog", + "@com_google_absl//absl/strings:str_format", + "//curvefs/src/common:dynamic_vlog", + "//curvefs/src/client/common:common", + ], +) diff --git a/curvefs/src/client/logger/access_log.cpp b/curvefs/src/client/logger/access_log.cpp new file mode 100644 index 0000000000..28c9255b9f --- /dev/null +++ b/curvefs/src/client/logger/access_log.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-14 + * Author: Jingli Chen (Wine93) + */ + +#include "curvefs/src/client/logger/access_log.h" + +namespace curvefs { +namespace client { +namespace logger { + +std::shared_ptr Logger; +bool inited = false; + +bool InitAccessLog(const std::string& prefix) { + if (inited) { + return true; + } + + std::string filename = StrFormat("%s/access.%d.log", prefix, getpid()); + Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0); + spdlog::flush_every(std::chrono::seconds(1)); + inited = true; + return true; +} + +} // namespace logger +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/filesystem/access_log.h b/curvefs/src/client/logger/access_log.h similarity index 75% rename from curvefs/src/client/filesystem/access_log.h rename to curvefs/src/client/logger/access_log.h index 608b564f65..5c9e075087 100644 --- a/curvefs/src/client/filesystem/access_log.h +++ b/curvefs/src/client/logger/access_log.h @@ -20,6 +20,8 @@ * Author: Jingli Chen (Wine93) */ +// clang-format off + #include #include #include @@ -32,8 +34,8 @@ #include "absl/strings/str_format.h" #include "curvefs/src/client/common/config.h" -#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_ -#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_ +#ifndef CURVEFS_SRC_CLIENT_LOGGER_ACCESS_LOG_H_ +#define CURVEFS_SRC_CLIENT_LOGGER_ACCESS_LOG_H_ namespace curvefs { namespace client { @@ -42,24 +44,20 @@ namespace common { DECLARE_bool(access_logging); } -namespace filesystem { +namespace logger { using ::absl::StrFormat; using ::curvefs::client::common::FLAGS_access_logging; using MessageHandler = std::function; -static std::shared_ptr Logger; +extern std::shared_ptr Logger; +extern bool inited; -bool InitAccessLog(const std::string& prefix) { - std::string filename = StrFormat("%s/access_%d.log", prefix, getpid()); - Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0); - spdlog::flush_every(std::chrono::seconds(1)); - return true; -} +bool InitAccessLog(const std::string& prefix); struct AccessLogGuard { explicit AccessLogGuard(MessageHandler handler) - : enable(FLAGS_access_logging), + : enable(FLAGS_access_logging && inited), handler(handler) { if (!enable) { return; @@ -82,8 +80,8 @@ struct AccessLogGuard { butil::Timer timer; }; -} // namespace filesystem +} // namespace logger } // namespace client } // namespace curvefs -#endif // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_ +#endif // CURVEFS_SRC_CLIENT_LOGGER_ACCESS_LOG_H_ diff --git a/curvefs/src/client/logger/error_log.h b/curvefs/src/client/logger/error_log.h new file mode 100644 index 0000000000..a259ce755a --- /dev/null +++ b/curvefs/src/client/logger/error_log.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-03 + * Author: Jingli Chen (Wine93) + */ + +// clang-format off + +#include + +#include +#include +#include + +#include "curvefs/src/common/dynamic_vlog.h" + +#ifndef CURVEFS_SRC_CLIENT_LOGGER_ERROR_LOG_H_ +#define CURVEFS_SRC_CLIENT_LOGGER_ERROR_LOG_H_ + +namespace curvefs { +namespace client { +namespace logger { + +using ::curvefs::common::FLAGS_vlog_level; + +inline bool InitErrorLog(const std::string& prefix, + const std::string& name, + int32_t loglevel) { + FLAGS_log_dir = prefix; + FLAGS_v = loglevel; + FLAGS_vlog_level = loglevel; + FLAGS_minloglevel = 0; + FLAGS_logtostderr = 0; + FLAGS_alsologtostderr = 0; + google::InitGoogleLogging(name.c_str()); + return true; +} + +inline void ShutdownErrorLog() { google::ShutdownGoogleLogging(); } + +} // namespace logger +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_LOGGER_ERROR_LOG_H_ diff --git a/curvefs/src/client/metric/client_metric.h b/curvefs/src/client/metric/client_metric.h index 58a23fa84b..bfbf0f3373 100644 --- a/curvefs/src/client/metric/client_metric.h +++ b/curvefs/src/client/metric/client_metric.h @@ -28,6 +28,7 @@ #include #include +#include #include #include "src/client/client_metric.h" diff --git a/curvefs/src/client/s3/client_s3_adaptor.cpp b/curvefs/src/client/s3/client_s3_adaptor.cpp index 89fb0257b6..81a0148e01 100644 --- a/curvefs/src/client/s3/client_s3_adaptor.cpp +++ b/curvefs/src/client/s3/client_s3_adaptor.cpp @@ -72,7 +72,7 @@ S3ClientAdaptorImpl::Init( LOG(ERROR) << "chunkSize:" << chunkSize_ << " is not integral multiple for the blockSize:" << blockSize_; - return CURVEFS_ERROR::INVALIDPARAM; + return CURVEFS_ERROR::INVALID_PARAM; } prefetchBlocks_ = option.prefetchBlocks; prefetchExecQueueNum_ = option.prefetchExecQueueNum; diff --git a/curvefs/src/client/s3/client_s3_cache_manager.cpp b/curvefs/src/client/s3/client_s3_cache_manager.cpp index c169c45675..1bd098eb71 100644 --- a/curvefs/src/client/s3/client_s3_cache_manager.cpp +++ b/curvefs/src/client/s3/client_s3_cache_manager.cpp @@ -232,7 +232,7 @@ CURVEFS_ERROR FsCacheManager::FsSync(bool force) { g_s3MultiManagerMetric->fileManagerNum << -1; } } - } else if (ret == CURVEFS_ERROR::NOTEXIST) { + } else if (ret == CURVEFS_ERROR::NOT_EXIST) { iter->second->ReleaseCache(); WriteLockGuard writeLockGuard(rwLock_); auto iter1 = fileCacheManagerMap_.find(iter->first); @@ -1808,7 +1808,7 @@ CURVEFS_ERROR ChunkCacheManager::Flush(uint64_t inodeId, bool force, assert(flushingDataCache_->IsDirty()); do { ret = flushingDataCache_->Flush(inodeId, toS3); - if (ret == CURVEFS_ERROR::NOTEXIST) { + if (ret == CURVEFS_ERROR::NOT_EXIST) { LOG(WARNING) << "dataCache flush failed. ret:" << ret << ",index:" << index_ << ",data chunkpos:" << flushingDataCache_->GetChunkPos(); diff --git a/curvefs/src/client/sdk_helper.cpp b/curvefs/src/client/sdk_helper.cpp new file mode 100644 index 0000000000..e75ae21978 --- /dev/null +++ b/curvefs/src/client/sdk_helper.cpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-08 + * Author: Jingli Chen (Wine93) + */ + +// clang-format off + +#include "curvefs/src/client/logger/access_log.h" +#include "curvefs/src/client/logger/error_log.h" +#include "src/common/gflags_helper.h" +#include "curvefs/src/client/sdk_helper.h" + +namespace curvefs { +namespace client { + +using ::curve::common::GflagsLoadValueFromConfIfCmdNotSet; +using ::curvefs::client::logger::InitAccessLog; +using ::curvefs::client::logger::InitErrorLog; +using ::curvefs::client::logger::ShutdownErrorLog; + +void SDKHelper::InitLog(Configuration* cfg) { + // prefix + std::string prefix; + cfg->GetStringValue("client.common.logDir", &prefix); + + // name + static std::string name = "/curvefs-sdk"; + + // log level + int32_t loglevel = 0; + curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy; + dummy.Load(cfg, "v", "client.loglevel", &loglevel); + + InitErrorLog(prefix, name, loglevel); + InitAccessLog(prefix); +} + +void SDKHelper::InitOption(Configuration* cfg, FuseClientOption* option) { + cfg->PrintConfig(); + InitFuseClientOption(cfg, option); +} + +FSType SDKHelper::Str2Type(const std::string& s) { + if (s == "s3") { + return FSType::TYPE_S3; + } else if (s == "volume") { + return FSType::TYPE_VOLUME; + } + return FSType(); +} + +std::string SDKHelper::Type2Str(FSType t) { + if (t == FSType::TYPE_S3) { + return "s3"; + } else if (t == FSType::TYPE_VOLUME) { + return "volume"; + } + return "unknown"; +} + +bool SDKHelper::GetFsInfoFromMDS(const MdsOption& option, + const std::string& fsname, + FsInfo* info) { + MdsClientImpl client; + MDSBaseClient base; + client.Init(option, &base); + auto rc = client.GetFsInfo(fsname, info); + if (rc != FSStatusCode::OK) { + LOG(ERROR) << "Get fsinfo from mds failed, fsname = " << fsname + << ", retCode = " << FSStatusCode_Name(rc); + return false; + } + return true; +} + +bool SDKHelper::CheckFsType(FSType real, FSType arg) { + if (real != arg) { + LOG(ERROR) << "The fstype obtained from the mds is " + << FSType_Name(real) << ", but user specified is " + << FSType_Name(arg); + return false; + } else if (real != FSType::TYPE_VOLUME && real != FSType::TYPE_S3) { + LOG(ERROR) << "The fstype obtained from the mds is " + << FSType_Name(real) << ", which is not supported"; + return false; + } + return true; +} + +MountOption SDKHelper::GetMountOption(const std::string& fsname, + const std::string& mountpoint) { + auto mountOption = MountOption(); + mountOption.mountPoint = mountpoint.c_str(); + mountOption.fsName = fsname.c_str(); + mountOption.fsType = Type2Str(FSType::TYPE_S3).c_str(); + return mountOption; +} + +CURVEFS_ERROR SDKHelper::CheckMountOption(std::shared_ptr client, + const MdsOption& mdsOption, + MountOption* mountOption) { + FsInfo info; + bool yes = GetFsInfoFromMDS(mdsOption, mountOption->fsName, &info) && + CheckFsType(info.fstype(), Str2Type(mountOption->fsType)); + if (!yes) { + return CURVEFS_ERROR::INTERNAL; + } + client->SetFsInfo(std::make_shared(info)); + return CURVEFS_ERROR::OK; +} + +CURVEFS_ERROR SDKHelper::Mount(std::shared_ptr client, + const std::string& fsname, + const std::string& mountpoint, + FuseClientOption option) { + auto mountOption = GetMountOption(fsname, mountpoint); + auto rc = CheckMountOption(client, option.mdsOpt, &mountOption); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + rc = client->Init(option); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + rc = client->Run(); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + rc = client->SetMountStatus(&mountOption); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + return client->FuseOpInit(nullptr, nullptr); +} + +CURVEFS_ERROR SDKHelper::Umount(std::shared_ptr client, + const std::string& fsname, + const std::string& mountpoint) { + auto mountOption = GetMountOption(fsname, mountpoint); + client->FuseOpDestroy(&mountOption); + client->Fini(); + client->UnInit(); + ShutdownErrorLog(); + return CURVEFS_ERROR::OK; +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/sdk_helper.h b/curvefs/src/client/sdk_helper.h new file mode 100644 index 0000000000..0a5b3c4fce --- /dev/null +++ b/curvefs/src/client/sdk_helper.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-08-08 + * Author: Jingli Chen (Wine93) + */ + +// clang-format off + +#ifndef CURVEFS_SRC_CLIENT_SDK_HELPER_H_ +#define CURVEFS_SRC_CLIENT_SDK_HELPER_H_ + +#include +#include + +#include "curvefs/proto/mds.pb.h" +#include "curvefs/proto/metaserver.pb.h" +#include "curvefs/src/client/common/config.h" +#include "curvefs/src/client/fuse_client.h" +#include "curvefs/src/client/fuse_common.h" +#include "curvefs/src/client/fuse_s3_client.h" +#include "curvefs/src/client/fuse_volume_client.h" +#include "src/common/configuration.h" + +namespace curvefs { +namespace client { + +using ::curve::common::Configuration; +using ::curvefs::client::common::MdsOption; +using ::curvefs::common::FSType; +using ::curvefs::mds::FsInfo; + +class SDKHelper { + public: + SDKHelper() = default; + + void InitLog(Configuration* cfg); + + void InitOption(Configuration* cfg, FuseClientOption* option); + + CURVEFS_ERROR Mount(std::shared_ptr client, + const std::string& fsname, + const std::string& mountpoint, + FuseClientOption option); + + CURVEFS_ERROR Umount(std::shared_ptr client, + const std::string& fsname, + const std::string& mountpoint); + + private: + FSType Str2Type(const std::string& s); + + std::string Type2Str(FSType t); + + bool GetFsInfoFromMDS(const MdsOption& option, + const std::string& fsname, + FsInfo* info); + + bool CheckFsType(FSType real, FSType arg); + + MountOption GetMountOption(const std::string& fsname, + const std::string& mountpoint); + + CURVEFS_ERROR CheckMountOption(std::shared_ptr client, + const MdsOption& mdsOption, + MountOption* mountOption); +}; + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_SDK_HELPER_H_ diff --git a/curvefs/src/client/vfs/.clang-format b/curvefs/src/client/vfs/.clang-format new file mode 100644 index 0000000000..a5660a57d8 --- /dev/null +++ b/curvefs/src/client/vfs/.clang-format @@ -0,0 +1,26 @@ +# I don't like current clang-formtter's specified style and +# and i think my style is more intuitive than it :), so I use +# this file to disable format, let us give some code snippet to prove it: +# +# my: +# std::vector tests { +# { " abc", "abc" }, +# { "abc ", "abc" }, +# { " abc ", "abc" }, +# { " a bc ", "a bc" }, +# { "a b c", "a b c" }, +# { " ", "" }, +# +# +# clang-formatter: +# std::vector tests{ +# {" abc", "abc"}, {"abc ", "abc"}, {" abc ", "abc"}, +# {" a bc ", "a bc"}, {"a b c", "a b c"}, {" ", ""}, +# }; +# +# Maybe you have different opinions, but please have a little mercy for me. +# +# by Wine93, 2023-10-16 + +DisableFormat: true +SortIncludes: Never diff --git a/curvefs/src/client/vfs/BUILD b/curvefs/src/client/vfs/BUILD new file mode 100644 index 0000000000..ba2926ff6f --- /dev/null +++ b/curvefs/src/client/vfs/BUILD @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_DEFAULT_COPTS") + +cc_library( + name = "vfs", + srcs = glob(["*.cpp"]), + hdrs = glob(["*.h"]), + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "@incbin//:incbin", + "//src/common:curve_common", + "//curvefs/proto:metaserver_cc_proto", + "//curvefs/src/client:filesystem", + "//curvefs/src/client/logger:logger", + ], +) diff --git a/curvefs/src/client/vfs/README.md b/curvefs/src/client/vfs/README.md new file mode 100644 index 0000000000..b074365365 --- /dev/null +++ b/curvefs/src/client/vfs/README.md @@ -0,0 +1,21 @@ + +CurveFS VFS Design +=== + +Permission +--- + +`uid/gid` AND `euid/egid` AND `suid/sgid` + +```C++ +euid = uid +egid = gid + +if (suid != NULL) { + euid = suid; +} + +if (sgid != NULL) { + egid = sgid; +} +``` diff --git a/curvefs/src/client/vfs/cache.cpp b/curvefs/src/client/vfs/cache.cpp new file mode 100644 index 0000000000..c10fe93aff --- /dev/null +++ b/curvefs/src/client/vfs/cache.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-04 + * Author: Jingli Chen (Wine93) + */ + +#include + +#include "absl/strings/str_format.h" +#include "curvefs/src/client/filesystem/utils.h" +#include "curvefs/src/client/vfs/cache.h" + +namespace curvefs { +namespace client { +namespace vfs { + +#define RETURN_FALSE_IF_DISABLED() \ + do { \ + if (!enable_) { \ + return false; \ + } \ + } while (0) + +using ::curvefs::client::filesystem::Now; + +EntryCache::EntryCache(size_t lruSize) + : enable_(lruSize > 0), + lru_(std::make_shared(lruSize)) {} + +std::string EntryCache::EntryKey(Ino parent, const std::string& name) { + return absl::StrFormat("%lu:%s", parent, name); +} + +bool EntryCache::Get(Ino parent, const std::string& name, Ino* ino) { + RETURN_FALSE_IF_DISABLED(); + Entry value; + auto key = EntryKey(parent, name); + bool yes = lru_->Get(key, &value); + if (!yes) { + return false; + } else if (value.expire < Now()) { + return false; + } + *ino = value.ino; + return true; +} + +bool EntryCache::Put(Ino parent, + const std::string& name, + Ino ino, + uint64_t timeoutSec) { + RETURN_FALSE_IF_DISABLED(); + auto key = EntryKey(parent, name); + Entry value; + value.ino = ino; + value.expire = Now() + TimeSpec(timeoutSec, 0); + lru_->Put(key, value); + return true; +} + +bool EntryCache::Delete(Ino parent, const std::string& name) { + RETURN_FALSE_IF_DISABLED(); + auto key = EntryKey(parent, name); + lru_->Remove(key); + return true; +} + +size_t EntryCache::Size() { + return lru_->Size(); +} + +AttrCache::AttrCache(size_t lruSize) + : enable_(lruSize > 0), + lru_(std::make_shared(lruSize)) {} + +bool AttrCache::Get(Ino ino, InodeAttr* attr) { + RETURN_FALSE_IF_DISABLED(); + Attr value; + bool yes = lru_->Get(ino, &value); + if (!yes) { + return false; + } else if (value.expire < Now()) { + return false; + } + *attr = value.attr; + return true; +} + +bool AttrCache::Put(Ino ino, const InodeAttr& attr, uint64_t timeoutSec) { + RETURN_FALSE_IF_DISABLED(); + Attr value; + value.attr = std::move(attr); + value.expire = Now() + TimeSpec(timeoutSec, 0); + lru_->Put(ino, value); + return true; +} + +bool AttrCache::Delete(Ino ino) { + RETURN_FALSE_IF_DISABLED(); + lru_->Remove(ino); + return true; +} + +size_t AttrCache::Size() { + return lru_->Size(); +} + +} // namespace vfs +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/vfs/cache.h b/curvefs/src/client/vfs/cache.h new file mode 100644 index 0000000000..f1653444a8 --- /dev/null +++ b/curvefs/src/client/vfs/cache.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-04 + * Author: Jingli Chen (Wine93) + */ + +#ifndef CURVEFS_SRC_CLIENT_VFS_CACHE_H_ +#define CURVEFS_SRC_CLIENT_VFS_CACHE_H_ + +#include +#include + +#include "src/common/lru_cache.h" +#include "curvefs/src/client/vfs/meta.h" + +namespace curvefs { +namespace client { +namespace vfs { + +using ::curve::common::LRUCache; + +class EntryCache { + public: + struct Entry { + Ino ino; + TimeSpec expire; + }; + + // TODO(Wine93): is there a more effective type for entry cache? maybe + // absl::btree> is a choise, + // but it is a bit complex to implement lru evit strategy. + using LRUType = LRUCache; + + public: + EntryCache() = delete; + + explicit EntryCache(size_t lruSize); + + bool Get(Ino parent, const std::string& name, Ino* ino); + + bool Put(Ino parent, const std::string& name, Ino ino, uint64_t timeoutSec); + + bool Delete(Ino parent, const std::string& name); + + size_t Size(); + + private: + std::string EntryKey(Ino parent, const std::string& name); + + private: + bool enable_; + std::shared_ptr lru_; +}; + +class AttrCache { + public: + struct Attr { + InodeAttr attr; + TimeSpec expire; + }; + + using LRUType = LRUCache; + + public: + AttrCache() = delete; + + explicit AttrCache(size_t lruSize); + + bool Get(Ino ino, InodeAttr* attr); + + bool Put(Ino ino, const InodeAttr& attr, uint64_t timeoutSec); + + bool Delete(Ino ino); + + size_t Size(); + + private: + bool enable_; + std::shared_ptr lru_; +}; + +} // namespace vfs +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_VFS_CACHE_H_ diff --git a/curvefs/src/client/vfs/config.cpp b/curvefs/src/client/vfs/config.cpp new file mode 100644 index 0000000000..a75f01eb80 --- /dev/null +++ b/curvefs/src/client/vfs/config.cpp @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2023 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2023-07-11 + * Author: Jingli Chen (Wine93) + */ + +#define INCBIN_STYLE INCBIN_STYLE_SNAKE +#define INCBIN_PREFIX g_ + +#include + +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/strings/str_split.h" +#include "curvefs/src/client/vfs/utils.h" +#include "curvefs/src/client/vfs/config.h" + +/* Usage: INCBIN(<>, <>) + * + * Symbols defined by INCBIN + * ------------------------------------------ + * const unsigned char g_client_conf_data[] // g_<>_data + * const unsigned char* const g_client_conf_end; // g_<