diff --git a/.gitignore b/.gitignore index 92dc59bcf5..353d3f019c 100755 --- a/.gitignore +++ b/.gitignore @@ -177,3 +177,4 @@ curvefs/sdk/libcurvefs/examples/bin curvefs/sdk/output/ hadoop-test/ nnbench-test/ +dependency-reduced-pom.xml diff --git a/.obm.cfg b/.obm.cfg index 1803905eb0..80f12479ca 100644 --- a/.obm.cfg +++ b/.obm.cfg @@ -1,2 +1,3 @@ -container_name: curve-build-playground.master -container_image: opencurvedocker/curve-build:ubuntu22 +container_name: curve-build-playground.debian11 +container_image: opencurvedocker/curve-base:build-debian11 + diff --git a/Makefile b/Makefile index aefeeb374d..03c2187ae1 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,7 @@ ci-list: @bash util/build_in_image.sh --stor=$(stor) --list ci-build: - @bash util/build_in_image.sh --stor=$(stor) --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os) --sanitizer=$(sanitizer) + @bash util/build_in_image.sh --stor=$(stor) --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os) ci-dep: @bash util/build_in_image.sh --stor=$(stor) --only="" --dep=1 @@ -127,3 +127,6 @@ init-hadoop: sdk: @bash util/sdk.sh + +clean: + @bash util/clean.sh diff --git a/curvefs/docker/debian11/Dockerfile b/curvefs/docker/debian11/Dockerfile deleted file mode 100644 index 9ea3ca5c1c..0000000000 --- a/curvefs/docker/debian11/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM opencurvedocker/curve-base:debian11 -COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 /usr/lib/ -COPY curvefs /curvefs -RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \ - && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \ - && cp /curvefs/tools-v2/sbin/curve /usr/bin/ diff --git a/curvefs/docker/debian11/entrypoint.sh b/curvefs/docker/debian11/entrypoint.sh deleted file mode 100755 index 35faec3cad..0000000000 --- a/curvefs/docker/debian11/entrypoint.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. - -############################ GLOBAL VARIABLES -g_role="" -g_args="" -g_prefix="" -g_binary="" -g_start_args="" -g_preexec="/curvefs/tools-v2/sbin/daemon" - -############################ BASIC FUNCTIONS -function msg() { - printf '%b' "$1" >&2 -} - -function success() { - msg "\33[32m[✔]\33[0m ${1}${2}" -} - -function die() { - msg "\33[31m[✘]\33[0m ${1}${2}" - exit 1 -} - -############################ FUNCTIONS -function usage () { - cat << _EOC_ -Usage: - entrypoint.sh --role=ROLE - entrypoint.sh --role=ROLE --args=ARGS - -Examples: - entrypoint.sh --role=etcd - entrypoint.sh --role=client --args="-o default_permissions" -_EOC_ -} - -function get_options() { - local long_opts="role:,args:,help" - local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"` - eval set -- "${args}" - while true - do - case "$1" in - -r|--role) - g_role=$2 - shift 2 - ;; - -a|--args) - g_args=$2 - shift 2 - ;; - -h) - usage - exit 1 - ;; - --) - shift - break - ;; - *) - exit 1 - ;; - esac - done -} - -function prepare() { - g_prefix="/curvefs/$g_role" - conf_path="$g_prefix/conf/$g_role.conf" - - case $g_role in - etcd) - g_binary="$g_prefix/sbin/etcd" - g_start_args="--config-file $conf_path" - ;; - mds) - g_binary="$g_prefix/sbin/curvefs-mds" - g_start_args="--confPath $conf_path" - ;; - metaserver) - g_binary="$g_prefix/sbin/curvefs-metaserver" - g_start_args="--confPath $conf_path" - ;; - client) - g_binary="$g_prefix/sbin/curve-fuse" - g_start_args="--confPath $conf_path" - ;; - monitor) - g_binary="python3" - g_start_args="target_json.py" - ;; - *) - usage - exit 1 - ;; - esac - - if [ "$g_args" != "" ]; then - g_start_args=$g_args - fi -} - -function create_directory() { - if [ "$g_role" != "monitor" ]; then - chmod 700 "$g_prefix/data" - fi - - if [ "$g_role" == "etcd" ]; then - mkdir -p "$g_prefix/data/wal" - elif [ "$g_role" == "metaserver" ]; then - mkdir -p "$g_prefix/data/storage" - elif [ "$g_role" == "client" ]; then - mkdir -p "$g_prefix/mnt" - fi -} - -function main() { - get_options "$@" - - prepare - create_directory - [[ $(command -v crontab) ]] && cron - [[ ! -z $g_preexec ]] && $g_preexec & - if [ $g_role == "etcd" ]; then - exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1 - elif [ $g_role == "monitor" ]; then - cd $g_prefix - exec $g_binary $g_start_args - else - exec $g_binary $g_start_args - fi - -} - -############################ MAIN() -main "$@" diff --git a/curvefs/docker/openeuler/Dockerfile b/curvefs/docker/openeuler/Dockerfile deleted file mode 100644 index 4aa04b8f56..0000000000 --- a/curvefs/docker/openeuler/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM opencurvedocker/curve-base:openeuler -COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 libfuse3.so.3.10.5 libsnappy.so.1.1.9 libetcdclient.so /usr/lib/ -COPY curvefs /curvefs -RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \ - && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \ - && cp /curvefs/tools-v2/sbin/curve /usr/bin/ \ - && ln -s /usr/lib/libfuse3.so.3.10.5 /usr/lib/libfuse3.so.3 \ - && ln -s /usr/lib64/libsnappy.so.1.1.9 /usr/lib64/libsnappy.so.1 \ - && ldconfig diff --git a/curvefs/docker/openeuler/entrypoint.sh b/curvefs/docker/openeuler/entrypoint.sh deleted file mode 100644 index 35faec3cad..0000000000 --- a/curvefs/docker/openeuler/entrypoint.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc. - -############################ GLOBAL VARIABLES -g_role="" -g_args="" -g_prefix="" -g_binary="" -g_start_args="" -g_preexec="/curvefs/tools-v2/sbin/daemon" - -############################ BASIC FUNCTIONS -function msg() { - printf '%b' "$1" >&2 -} - -function success() { - msg "\33[32m[✔]\33[0m ${1}${2}" -} - -function die() { - msg "\33[31m[✘]\33[0m ${1}${2}" - exit 1 -} - -############################ FUNCTIONS -function usage () { - cat << _EOC_ -Usage: - entrypoint.sh --role=ROLE - entrypoint.sh --role=ROLE --args=ARGS - -Examples: - entrypoint.sh --role=etcd - entrypoint.sh --role=client --args="-o default_permissions" -_EOC_ -} - -function get_options() { - local long_opts="role:,args:,help" - local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"` - eval set -- "${args}" - while true - do - case "$1" in - -r|--role) - g_role=$2 - shift 2 - ;; - -a|--args) - g_args=$2 - shift 2 - ;; - -h) - usage - exit 1 - ;; - --) - shift - break - ;; - *) - exit 1 - ;; - esac - done -} - -function prepare() { - g_prefix="/curvefs/$g_role" - conf_path="$g_prefix/conf/$g_role.conf" - - case $g_role in - etcd) - g_binary="$g_prefix/sbin/etcd" - g_start_args="--config-file $conf_path" - ;; - mds) - g_binary="$g_prefix/sbin/curvefs-mds" - g_start_args="--confPath $conf_path" - ;; - metaserver) - g_binary="$g_prefix/sbin/curvefs-metaserver" - g_start_args="--confPath $conf_path" - ;; - client) - g_binary="$g_prefix/sbin/curve-fuse" - g_start_args="--confPath $conf_path" - ;; - monitor) - g_binary="python3" - g_start_args="target_json.py" - ;; - *) - usage - exit 1 - ;; - esac - - if [ "$g_args" != "" ]; then - g_start_args=$g_args - fi -} - -function create_directory() { - if [ "$g_role" != "monitor" ]; then - chmod 700 "$g_prefix/data" - fi - - if [ "$g_role" == "etcd" ]; then - mkdir -p "$g_prefix/data/wal" - elif [ "$g_role" == "metaserver" ]; then - mkdir -p "$g_prefix/data/storage" - elif [ "$g_role" == "client" ]; then - mkdir -p "$g_prefix/mnt" - fi -} - -function main() { - get_options "$@" - - prepare - create_directory - [[ $(command -v crontab) ]] && cron - [[ ! -z $g_preexec ]] && $g_preexec & - if [ $g_role == "etcd" ]; then - exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1 - elif [ $g_role == "monitor" ]; then - cd $g_prefix - exec $g_binary $g_start_args - else - exec $g_binary $g_start_args - fi - -} - -############################ MAIN() -main "$@" diff --git a/curvefs/proto/metaserver.proto b/curvefs/proto/metaserver.proto index f0ab8167a2..54e92972a7 100644 --- a/curvefs/proto/metaserver.proto +++ b/curvefs/proto/metaserver.proto @@ -115,11 +115,13 @@ message CreateDentryRequest { required uint32 partitionId = 3; required Dentry dentry = 4; optional Time create = 5; + optional CreateInodeRequest createInodeRequest = 6; } message CreateDentryResponse { required MetaStatusCode statusCode = 1; optional uint64 appliedIndex = 2; + optional Inode inode = 3; } message DeleteDentryRequest { @@ -213,7 +215,7 @@ message UpdateDeallocatableBlockGroupRequest { } message UpdateDeallocatableBlockGroupResponse { - required MetaStatusCode statusCode = 1; + required MetaStatusCode statusCode = 1; optional uint64 appliedIndex = 2; } diff --git a/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.h b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.h deleted file mode 100644 index 05d0a21e3f..0000000000 --- a/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.h +++ /dev/null @@ -1,221 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class io_opencurve_curve_fs_libfs_CurveFSMount */ - -#ifndef _Included_io_opencurve_curve_fs_libfs_CurveFSMount -#define _Included_io_opencurve_curve_fs_libfs_CurveFSMount -#ifdef __cplusplus -extern "C" { -#endif -#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_RDONLY -#define io_opencurve_curve_fs_libfs_CurveFSMount_O_RDONLY 1L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_RDWR -#define io_opencurve_curve_fs_libfs_CurveFSMount_O_RDWR 2L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_APPEND -#define io_opencurve_curve_fs_libfs_CurveFSMount_O_APPEND 4L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_CREAT -#define io_opencurve_curve_fs_libfs_CurveFSMount_O_CREAT 8L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_TRUNC -#define io_opencurve_curve_fs_libfs_CurveFSMount_O_TRUNC 16L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_EXCL -#define io_opencurve_curve_fs_libfs_CurveFSMount_O_EXCL 32L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_WRONLY -#define io_opencurve_curve_fs_libfs_CurveFSMount_O_WRONLY 64L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_O_DIRECTORY -#define io_opencurve_curve_fs_libfs_CurveFSMount_O_DIRECTORY 128L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_SET -#define io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_SET 0L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_CUR -#define io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_CUR 1L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_END -#define io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_END 2L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_MODE -#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_MODE 1L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_UID -#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_UID 2L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_GID -#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_GID 4L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_MTIME -#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_MTIME 8L -#undef io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_ATIME -#define io_opencurve_curve_fs_libfs_CurveFSMount_SETATTR_ATIME 16L -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSCreate - * Signature: ()J - */ -JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSCreate - (JNIEnv *, jobject); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSRelease - * Signature: (J)V - */ -JNIEXPORT void JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRelease - (JNIEnv *, jobject, jlong); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSConfSet - * Signature: (JLjava/lang/String;Ljava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSConfSet - (JNIEnv *, jclass, jlong, jstring, jstring); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSMount - * Signature: (JLjava/lang/String;Ljava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMount - (JNIEnv *, jclass, jlong, jstring, jstring); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSUmount - * Signature: (JLjava/lang/String;Ljava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUmount - (JNIEnv *, jclass, jlong, jstring, jstring); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSMkDirs - * Signature: (JLjava/lang/String;I)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMkDirs - (JNIEnv *, jclass, jlong, jstring, jint); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSRmDir - * Signature: (JLjava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRmDir - (JNIEnv *, jclass, jlong, jstring); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSListDir - * Signature: (JLjava/lang/String;)[Ljava/lang/String; - */ -JNIEXPORT jobjectArray JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSListDir - (JNIEnv *, jclass, jlong, jstring); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSOpen - * Signature: (JLjava/lang/String;II)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSOpen - (JNIEnv *, jclass, jlong, jstring, jint, jint); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSLSeek - * Signature: (JIJI)J - */ -JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLSeek - (JNIEnv *, jclass, jlong, jint, jlong, jint); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativieCurveFSRead - * Signature: (JI[BJJ)J - */ -JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSRead - (JNIEnv *, jclass, jlong, jint, jbyteArray, jlong, jlong); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativieCurveFSWrite - * Signature: (JI[BJJ)J - */ -JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSWrite - (JNIEnv *, jclass, jlong, jint, jbyteArray, jlong, jlong); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSFSync - * Signature: (JI)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFSync - (JNIEnv *, jclass, jlong, jint); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSClose - * Signature: (JI)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSClose - (JNIEnv *, jclass, jlong, jint); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSUnlink - * Signature: (JLjava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUnlink - (JNIEnv *, jclass, jlong, jstring); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSStatFs - * Signature: (JLio/opencurve/curve/fs/libfs/CurveFSStatVFS;)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSStatFs - (JNIEnv *, jclass, jlong, jobject); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSLstat - * Signature: (JLjava/lang/String;Lio/opencurve/curve/fs/libfs/CurveFSStat;)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLstat - (JNIEnv *, jclass, jlong, jstring, jobject); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSFStat - * Signature: (JILio/opencurve/curve/fs/libfs/CurveFSStat;)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFStat - (JNIEnv *, jclass, jlong, jint, jobject); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSSetAttr - * Signature: (JLjava/lang/String;Lio/opencurve/curve/fs/libfs/CurveFSStat;I)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSSetAttr - (JNIEnv *, jclass, jlong, jstring, jobject, jint); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSChmod - * Signature: (JLjava/lang/String;I)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChmod - (JNIEnv *, jclass, jlong, jstring, jint); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSChown - * Signature: (JLjava/lang/String;II)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChown - (JNIEnv *, jclass, jlong, jstring, jint, jint); - -/* - * Class: io_opencurve_curve_fs_libfs_CurveFSMount - * Method: nativeCurveFSRename - * Signature: (JLjava/lang/String;Ljava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRename - (JNIEnv *, jclass, jlong, jstring, jstring); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.cpp b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.cpp similarity index 54% rename from curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.cpp rename to curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.cpp index d9c0abed81..997f88bc5f 100644 --- a/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.cpp +++ b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 NetEase Inc. + * Copyright (c) 2024 NetEase Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,46 +16,67 @@ /* * Project: Curve - * Created Date: 2023-08-08 + * Created Date: 2024-03-08 * Author: Jingli Chen (Wine93) */ #include #include +#include +#include #include "absl/cleanup/cleanup.h" #include "curvefs/sdk/libcurvefs/libcurvefs.h" -#include "curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFSMount.h" - -/* Cached field IDs for io.opencurve.curve.fs.CurveStat */ -static jfieldID curvestat_mode_fid; -static jfieldID curvestat_uid_fid; -static jfieldID curvestat_gid_fid; -static jfieldID curvestat_size_fid; -static jfieldID curvestat_blksize_fid; -static jfieldID curvestat_blocks_fid; -static jfieldID curvestat_a_time_fid; -static jfieldID curvestat_m_time_fid; -static jfieldID curvestat_is_file_fid; -static jfieldID curvestat_is_directory_fid; -static jfieldID curvestat_is_symlink_fid; - -/* Cached field IDs for io.opencurve.curve.fs.CurveStatVFS */ -static jfieldID curvestatvfs_bsize_fid; -static jfieldID curvestatvfs_frsize_fid; -static jfieldID curvestatvfs_blocks_fid; -static jfieldID curvestatvfs_bavail_fid; -static jfieldID curvestatvfs_files_fid; -static jfieldID curvestatvfs_fsid_fid; -static jfieldID curvestatvfs_namemax_fid; - -/* - * Setup cached field IDs - */ +#include "curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.h" + +const char* statvfs_cls_name = + "io/opencurve/curve/fs/libfs/CurveFsMount$StatVfs"; +const char* stat_cls_name = + "io/opencurve/curve/fs/libfs/CurveFsMount$Stat"; +const char* file_cls_name = + "io/opencurve/curve/fs/libfs/CurveFsMount$File"; +const char* dirent_cls_name = + "io/opencurve/curve/fs/libfs/CurveFsMount$Dirent"; + +// Cached for class +static jclass statvfs_cls; +static jclass stat_cls; +static jclass file_cls; +static jclass dirent_cls; + +// Cached field IDs for io.opencurve.curve.fs.CurveFsMount.StatVfs +static jfieldID statvfs_bsize_fid; +static jfieldID statvfs_frsize_fid; +static jfieldID statvfs_blocks_fid; +static jfieldID statvfs_bavail_fid; +static jfieldID statvfs_files_fid; +static jfieldID statvfs_fsid_fid; +static jfieldID statvfs_namemax_fid; + +// Cached field IDs for io.opencurve.curve.fs.CurveFsMount.Stat +static jfieldID stat_mode_fid; +static jfieldID stat_uid_fid; +static jfieldID stat_gid_fid; +static jfieldID stat_size_fid; +static jfieldID stat_blksize_fid; +static jfieldID stat_blocks_fid; +static jfieldID stat_atime_fid; +static jfieldID stat_mtime_fid; +static jfieldID stat_isFile_fid; +static jfieldID stat_isDirectory_fid; +static jfieldID stat_isSymlink_fid; + +// Cached field IDs for io.opencurve.curve.fs.CurveFsMount.File +static jfieldID file_fd_fid; +static jfieldID file_length_fid; + +// Cached field IDs for io.opencurve.curve.fs.CurveFsMount.Dirent +static jfieldID dirent_name_fid; +static jfieldID dirent_stat_fid; + + +// Setup cached field IDs static void setup_field_ids(JNIEnv* env) { - jclass curvestat_cls; - jclass curvestatvfs_cls; - /* * Get a fieldID from a class with a specific type * @@ -75,94 +96,122 @@ static void setup_field_ids(JNIEnv* env) { return; \ } while (0) - /* Cache CurveStat fields */ + // Cache StatVfs fields + statvfs_cls = env->FindClass(statvfs_cls_name); + if (!statvfs_cls) { + return; + } + + GETFID(statvfs, bsize, J); + GETFID(statvfs, frsize, J); + GETFID(statvfs, blocks, J); + GETFID(statvfs, bavail, J); + GETFID(statvfs, files, J); + GETFID(statvfs, fsid, J); + GETFID(statvfs, namemax, J); + + // Cache Stat fields + stat_cls = env->FindClass(stat_cls_name); + if (!stat_cls) { + return; + } - curvestat_cls = env->FindClass("io/opencurve/curve/fs/libfs/CurveFSStat"); - if (!curvestat_cls) { + GETFID(stat, mode, I); + GETFID(stat, uid, I); + GETFID(stat, gid, I); + GETFID(stat, size, J); + GETFID(stat, blksize, J); + GETFID(stat, blocks, J); + GETFID(stat, atime, J); + GETFID(stat, mtime, J); + GETFID(stat, isFile, Z); + GETFID(stat, isDirectory, Z); + GETFID(stat, isSymlink, Z); + + // Cache File fields + file_cls = env->FindClass(file_cls_name); + if (!file_cls) { return; } - GETFID(curvestat, mode, I); - GETFID(curvestat, uid, I); - GETFID(curvestat, gid, I); - GETFID(curvestat, size, J); - GETFID(curvestat, blksize, J); - GETFID(curvestat, blocks, J); - GETFID(curvestat, a_time, J); - GETFID(curvestat, m_time, J); - GETFID(curvestat, is_file, Z); - GETFID(curvestat, is_directory, Z); - GETFID(curvestat, is_symlink, Z); - - /* Cache CurveStatVFS fields */ - - curvestatvfs_cls = - env->FindClass("io/opencurve/curve/fs/libfs/CurveFSStatVFS"); - if (!curvestatvfs_cls) { + GETFID(file, fd, I); + GETFID(file, length, J); + + // Cache Dirent fields + dirent_cls = env->FindClass(dirent_cls_name); + if (!dirent_cls) { return; } - GETFID(curvestatvfs, bsize, J); - GETFID(curvestatvfs, frsize, J); - GETFID(curvestatvfs, blocks, J); - GETFID(curvestatvfs, bavail, J); - GETFID(curvestatvfs, files, J); - GETFID(curvestatvfs, fsid, J); - GETFID(curvestatvfs, namemax, J); + GETFID(dirent, name, Ljava/lang/String;); + GETFID(dirent, stat, Lio/opencurve/curve/fs/libfs/CurveFsMount$Stat;); #undef GETFID } -static void fill_curvestat(JNIEnv* env, - jobject j_curvestat, - struct stat* stat) { - env->SetIntField(j_curvestat, curvestat_mode_fid, stat->st_mode); - env->SetIntField(j_curvestat, curvestat_uid_fid, stat->st_uid); - env->SetIntField(j_curvestat, curvestat_gid_fid, stat->st_gid); - env->SetLongField(j_curvestat, curvestat_size_fid, stat->st_size); - env->SetLongField(j_curvestat, curvestat_blksize_fid, stat->st_blksize); - env->SetLongField(j_curvestat, curvestat_blocks_fid, stat->st_blocks); +static void fill_statvfs(JNIEnv* env, jobject j_statvfs, struct statvfs* st) { + env->SetLongField(j_statvfs, statvfs_bsize_fid, st->f_bsize); + env->SetLongField(j_statvfs, statvfs_frsize_fid, st->f_frsize); + env->SetLongField(j_statvfs, statvfs_blocks_fid, st->f_blocks); + env->SetLongField(j_statvfs, statvfs_bavail_fid, st->f_bavail); + env->SetLongField(j_statvfs, statvfs_files_fid, st->f_files); + env->SetLongField(j_statvfs, statvfs_fsid_fid, st->f_fsid); + env->SetLongField(j_statvfs, statvfs_namemax_fid, st->f_namemax); +} + +static void fill_stat(JNIEnv* env, jobject j_stat, struct stat* stat) { + env->SetIntField(j_stat, stat_mode_fid, stat->st_mode); + env->SetIntField(j_stat, stat_uid_fid, stat->st_uid); + env->SetIntField(j_stat, stat_gid_fid, stat->st_gid); + env->SetLongField(j_stat, stat_size_fid, stat->st_size); + env->SetLongField(j_stat, stat_blksize_fid, stat->st_blksize); + env->SetLongField(j_stat, stat_blocks_fid, stat->st_blocks); // mtime uint64_t time = stat->st_mtim.tv_sec; time *= 1000; time += stat->st_mtim.tv_nsec / 1000000; - env->SetLongField(j_curvestat, curvestat_m_time_fid, time); + env->SetLongField(j_stat, stat_mtime_fid, time); // atime time = stat->st_atim.tv_sec; time *= 1000; time += stat->st_atim.tv_nsec / 1000000; - env->SetLongField(j_curvestat, curvestat_a_time_fid, time); + env->SetLongField(j_stat, stat_atime_fid, time); - env->SetBooleanField(j_curvestat, curvestat_is_file_fid, + env->SetBooleanField(j_stat, stat_isFile_fid, S_ISREG(stat->st_mode) ? JNI_TRUE : JNI_FALSE); - env->SetBooleanField(j_curvestat, curvestat_is_directory_fid, + env->SetBooleanField(j_stat, stat_isDirectory_fid, S_ISDIR(stat->st_mode) ? JNI_TRUE : JNI_FALSE); - env->SetBooleanField(j_curvestat, curvestat_is_symlink_fid, + env->SetBooleanField(j_stat, stat_isSymlink_fid, S_ISLNK(stat->st_mode) ? JNI_TRUE : JNI_FALSE); } -static void fill_curvestatvfs(JNIEnv* env, - jobject j_curvestatvfs, - struct statvfs st) { - env->SetLongField(j_curvestatvfs, curvestatvfs_bsize_fid, st.f_bsize); - env->SetLongField(j_curvestatvfs, curvestatvfs_frsize_fid, st.f_frsize); - env->SetLongField(j_curvestatvfs, curvestatvfs_blocks_fid, st.f_blocks); - env->SetLongField(j_curvestatvfs, curvestatvfs_bavail_fid, st.f_bavail); - env->SetLongField(j_curvestatvfs, curvestatvfs_files_fid, st.f_files); - env->SetLongField(j_curvestatvfs, curvestatvfs_fsid_fid, st.f_fsid); - env->SetLongField(j_curvestatvfs, curvestatvfs_namemax_fid, st.f_namemax); +static void fill_file(JNIEnv* env, jobject j_file, file_t* file) { + env->SetIntField(j_file, file_fd_fid, file->fd); + env->SetLongField(j_file, file_length_fid, file->length); } -/* Map io_opencurve_curve_fs_libfs_CurveFSMount_O_* open flags to values in libc */ +static void fill_dirent(JNIEnv* env, jobject j_dirent, dirent_t* dirent) { + jstring j_name = env->NewStringUTF(dirent->name); + jobject j_stat = env->AllocObject(env->FindClass(stat_cls_name)); + fill_stat(env, j_stat, &dirent->stat); + + env->SetObjectField(j_dirent, dirent_name_fid, j_name); + env->SetObjectField(j_dirent, dirent_stat_fid, j_stat); + + env->DeleteLocalRef(j_name); + env->DeleteLocalRef(j_stat); +} + +// Map io_opencurve_curve_fs_libfs_CurveFsMount_O_* open flags to values in libc static inline uint32_t fixup_open_flags(jint jflags) { uint32_t flags = 0; #define FIXUP_OPEN_FLAG(name) \ - if (jflags & io_opencurve_curve_fs_libfs_CurveFSMount_##name) \ + if (jflags & io_opencurve_curve_fs_libfs_CurveFsMount_##name) \ flags |= name; FIXUP_OPEN_FLAG(O_RDONLY) @@ -189,12 +238,12 @@ static inline uint32_t fixup_open_flags(jint jflags) { #define CURVEFS_SETATTR_MTIME_NOW (1 << 8) #define CURVEFS_SETATTR_CTIME (1 << 10) -/* Map JAVA_SETATTR_* to values in curve lib */ +// Map JAVA_SETATTR_* to values in curve lib static inline int fixup_attr_mask(jint jmask) { int mask = 0; #define FIXUP_ATTR_MASK(name) \ - if (jmask & io_opencurve_curve_fs_libfs_CurveFSMount_##name) \ + if (jmask & io_opencurve_curve_fs_libfs_CurveFsMount_##name) \ mask |= CURVEFS_##name; FIXUP_ATTR_MASK(SETATTR_MODE) @@ -207,17 +256,15 @@ static inline int fixup_attr_mask(jint jmask) { return mask; } -/* - * Exception throwing helper. Adapted from Apache Hadoop header - * org_apache_hadoop.h by adding the do {} while (0) construct. - */ +// Exception throwing helper. Adapted from Apache Hadoop header +// org_apache_hadoop.h by adding the do {} while (0) construct. #define THROW(env, exception_name, message) \ do { \ jclass ecls = env->FindClass(exception_name); \ if (ecls) { \ int ret = env->ThrowNew(ecls, message); \ if (ret < 0) { \ - printf("(CurveFS) Fatal Error\n"); \ + printf("(CurveFs) Fatal Error\n"); \ } \ env->DeleteLocalRef(ecls); \ } \ @@ -232,7 +279,7 @@ static void handle_error(JNIEnv* env, int rc) { THROW(env, "org/apache/hadoop/fs/FileAlreadyExistsException", ""); return; case ENOTDIR: - THROW(env, "org/apache/hadoop/fs/ParentNotDirectoryException", ""); + THROW(env, "io/opencurve/curve/fs/libfs/CurveFsException$NotADirectoryException", ""); // NOLINT return; default: break; @@ -241,26 +288,26 @@ static void handle_error(JNIEnv* env, int rc) { THROW(env, "java/io/IOException", strerror(rc)); } -// nativeCurveFSCreate: curvefs_create +// nativeCurveFsCreate: curvefs_new JNIEXPORT jlong -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSCreate - (JNIEnv* env, jobject) { +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsNew + (JNIEnv* env, jobject) { setup_field_ids(env); - uintptr_t instance = curvefs_create(); + uintptr_t instance = curvefs_new(); return reinterpret_cast(instance); } -// nativeCurveFSRelease: curvefs_release +// nativeCurveFsRelease: curvefs_delete JNIEXPORT void -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRelease +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsDelete (JNIEnv* env, jobject, jlong j_instance) { uintptr_t instance = static_cast(j_instance); - return curvefs_release(instance); + return curvefs_delete(instance); } -// nativeCurveFSConfSet: curvefs_conf_set +// nativeCurveFsConfSet: curvefs_conf_set JNIEXPORT void -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSConfSet +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsConfSet (JNIEnv* env, jclass, jlong j_instance, jstring j_key, jstring j_value) { uintptr_t instance = static_cast(j_instance); const char* key = env->GetStringUTFChars(j_key, NULL); @@ -273,9 +320,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSConfSet return curvefs_conf_set(instance, key, value); } -// nativeCurveFSMount: curvefs_mount +// nativeCurveFsMount: curvefs_mount JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMount +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsMount (JNIEnv* env, jclass, jlong j_instance, jstring j_fsname, jstring j_mountpoint) { uintptr_t instance = static_cast(j_instance); @@ -293,9 +340,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMount return rc; } -// nativeCurveFSUmount: curvefs_umount +// nativeCurveFsUmount: curvefs_umount JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUmount +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsUmount (JNIEnv* env, jclass, jlong j_instance, jstring j_fsname, jstring j_mountpoint) { uintptr_t instance = static_cast(j_instance); @@ -313,9 +360,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUmount return rc; } -// nativeCurveFSMkDirs: curvefs_mkdirs +// nativeCurveFsMkDirs: curvefs_mkdirs JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMkDirs +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsMkDirs (JNIEnv* env, jclass, jlong j_instance, jstring j_path, jint j_mode) { uintptr_t instance = static_cast(j_instance); const char* path = env->GetStringUTFChars(j_path, NULL); @@ -333,9 +380,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSMkDirs return rc; } -// nativeCurveFSRmDir: curvefs_rmdir +// nativeCurveFsRmDir: curvefs_rmdir JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRmDir +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRmDir (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { uintptr_t instance = static_cast(j_instance); const char* path = env->GetStringUTFChars(j_path, NULL); @@ -350,9 +397,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRmDir return rc; } -// nativeCurveFSListDir: curvefs_opendir/curvefs_readdir/curvefs_closedir +// nativeCurveFsListDir: curvefs_opendir/curvefs_readdir/curvefs_closedir JNIEXPORT jobjectArray -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSListDir +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsListDir (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { uintptr_t instance = static_cast(j_instance); const char* path = env->GetStringUTFChars(j_path, NULL); @@ -361,8 +408,8 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSListDir }); // curvefs_opendir - dir_stream_t dir_stream; - auto rc = curvefs_opendir(instance, path, &dir_stream); + uint64_t fd; + auto rc = curvefs_opendir(instance, path, &fd); if (rc != 0) { handle_error(env, rc); return NULL; @@ -370,60 +417,89 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSListDir // curvefs_readdir std::vector dirents; - dirent_t dirent; - for ( ;; ) { - ssize_t n = curvefs_readdir(instance, &dir_stream, &dirent); + std::vector buffer(8192); + for ( ; ; ) { + ssize_t n = curvefs_readdir(instance, fd, buffer.data(), 8192); if (n < 0) { handle_error(env, rc); return NULL; } else if (n == 0) { break; } - dirents.push_back(dirent); + + // TODO(Wine93): less memory copy + dirents.insert(dirents.end(), buffer.begin(), buffer.begin() + n); } // closedir - rc = curvefs_closedir(instance, &dir_stream); + rc = curvefs_closedir(instance, fd); if (rc != 0) { handle_error(env, rc); return NULL; } - // extract entry name - jobjectArray j_names = env->NewObjectArray( - dirents.size(), env->FindClass("java/lang/String"), NULL); + jobjectArray j_dirents = env->NewObjectArray(dirents.size(), + env->FindClass(dirent_cls_name), + NULL); for (int i = 0; i < dirents.size(); i++) { - jstring j_name = env->NewStringUTF(dirents[i].name); - env->SetObjectArrayElement(j_names, i, j_name); - env->DeleteLocalRef(j_name); + // NOTE!!!: don't use static class + jobject j_dirent = env->AllocObject(env->FindClass(dirent_cls_name)); + fill_dirent(env, j_dirent, &dirents[i]); + env->SetObjectArrayElement(j_dirents, i, j_dirent); + env->DeleteLocalRef(j_dirent); } - return j_names; + + return j_dirents; } -// nativeCurveFSOpen: curvefs_open +// nativeCurveFsOpen: curvefs_create JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSOpen +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsCreate (JNIEnv* env, jclass, - jlong j_instance, jstring j_path, jint j_flags, jint j_mode) { + jlong j_instance, jstring j_path, jint j_mode, jobject j_file) { uintptr_t instance = static_cast(j_instance); const char* path = env->GetStringUTFChars(j_path, NULL); - uint32_t flags = fixup_open_flags(j_flags); uint16_t mode = static_cast(j_mode); auto defer = absl::MakeCleanup([&]() { env->ReleaseStringUTFChars(j_path, path); }); - int fd = curvefs_open(instance, path, flags, mode); - if (fd < 0) { - handle_error(env, fd); + file_t file; + int rc = curvefs_create(instance, path, mode, &file); + if (rc < 0) { + handle_error(env, rc); } - return fd; + + fill_file(env, j_file, &file); + return rc; } -// nativeCurveFSLSeek: curvefs_lseek +// nativeCurveFsOpen: curvefs_open +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsOpen + (JNIEnv* env, jclass, + jlong j_instance, jstring j_path, jint j_flags, jobject j_file) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + uint32_t flags = fixup_open_flags(j_flags); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + file_t file; + int rc = curvefs_open(instance, path, flags, &file); + if (rc < 0) { + handle_error(env, rc); + } + + fill_file(env, j_file, &file); + return rc; +} + +// nativeCurveFsLSeek: curvefs_lseek JNIEXPORT jlong -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLSeek +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsLSeek (JNIEnv* env, jclass, jlong j_instance, jint j_fd, jlong j_offset, jint j_whence) { uintptr_t instance = static_cast(j_instance); @@ -432,13 +508,13 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLSeek int whence; switch (j_whence) { - case io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_SET: + case io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_SET: whence = SEEK_SET; break; - case io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_CUR: + case io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_CUR: whence = SEEK_CUR; break; - case io_opencurve_curve_fs_libfs_CurveFSMount_SEEK_END: + case io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_END: whence = SEEK_END; break; default: @@ -452,11 +528,11 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLSeek return rc; } -// nativieCurveFSRead: curvefs_read -JNIEXPORT jlong -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSRead +// nativieCurveFsRead: curvefs_read +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativieCurveFsRead (JNIEnv* env, jclass, jlong j_instance, jint j_fd, - jbyteArray j_buffer, jlong j_size, jlong j_offset) { + jlong offset, jbyteArray j_buffer, jlong j_size) { uintptr_t instance = static_cast(j_instance); int fd = static_cast(j_fd); jbyte* c_buffer = env->GetByteArrayElements(j_buffer, NULL); @@ -470,14 +546,14 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSRead if (n < 0) { handle_error(env, n); } - return static_cast(n); + return static_cast(n); } -// nativieCurveFSWrite: curvefs_write -JNIEXPORT jlong -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSWrite +// nativieCurveFsWrite: curvefs_write +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativieCurveFsWrite (JNIEnv* env, jclass, jlong j_instance, jint j_fd, - jbyteArray j_buffer, jlong j_size, jlong j_offset) { + jlong offset, jbyteArray j_buffer, jlong j_size) { uintptr_t instance = static_cast(j_instance); int fd = static_cast(j_fd); jbyte* c_buffer = env->GetByteArrayElements(j_buffer, NULL); @@ -491,12 +567,12 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativieCurveFSWrite if (n < 0) { handle_error(env, n); } - return static_cast(n); + return static_cast(n); } -// nativeCurveFSFSync: curvefs_fsync +// nativeCurveFsFSync: curvefs_fsync JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFSync +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsFSync (JNIEnv* env, jclass, jlong j_instance, jint j_fd) { uintptr_t instance = static_cast(j_instance); int fd = static_cast(j_fd); @@ -508,9 +584,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFSync return rc; } -// nativeCurveFSClose: curvefs_close +// nativeCurveFsClose: curvefs_close JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSClose +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsClose (JNIEnv* env, jclass, jlong j_instance, jint j_fd) { uintptr_t instance = static_cast(j_instance); int fd = static_cast(j_fd); @@ -522,9 +598,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSClose return rc; } -// nativeCurveFSUnlink: curvefs_unlink +// nativeCurveFsUnlink: curvefs_unlink JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUnlink +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsUnlink (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { uintptr_t instance = static_cast(j_instance); const char* path = env->GetStringUTFChars(j_path, NULL); @@ -539,11 +615,11 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSUnlink return rc; } -// nativeCurveFSStatFs: curvefs_statfs +// nativeCurveFsStatFs: curvefs_statfs JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSStatFs +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsStatFs (JNIEnv* env, jclass, - jlong j_instance, jobject j_curvestatvfs) { + jlong j_instance, jobject j_statvfs) { uintptr_t instance = static_cast(j_instance); struct statvfs statvfs; @@ -553,15 +629,15 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSStatFs return rc; } - fill_curvestatvfs(env, j_curvestatvfs, statvfs); + fill_statvfs(env, j_statvfs, &statvfs); return rc; } -// nativeCurveFSLstat: curvefs_lstat +// nativeCurveFsLstat: curvefs_lstat JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLstat +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsLStat (JNIEnv* env, jclass, - jlong j_instance, jstring j_path, jobject j_curvestat) { + jlong j_instance, jstring j_path, jobject j_stat) { uintptr_t instance = static_cast(j_instance); const char* path = env->GetStringUTFChars(j_path, NULL); auto defer = absl::MakeCleanup([&]() { @@ -576,14 +652,14 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSLstat return rc; } - fill_curvestat(env, j_curvestat, &stat); + fill_stat(env, j_stat, &stat); return rc; } -// nativeCurveFSFStat: curvefs_fstat +// nativeCurveFsFStat: curvefs_fstat JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFStat - (JNIEnv* env, jclass, jlong j_instance, jint j_fd, jobject j_curvestat) { +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsFStat + (JNIEnv* env, jclass, jlong j_instance, jint j_fd, jobject j_stat) { uintptr_t instance = static_cast(j_instance); int fd = static_cast(j_fd); @@ -595,15 +671,15 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSFStat return rc; } - fill_curvestat(env, j_curvestat, &stat); + fill_stat(env, j_stat, &stat); return rc; } -// nativeCurveFSSetAttr: curvefs_setattr +// nativeCurveFsSetAttr: curvefs_setattr JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSSetAttr +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsSetAttr (JNIEnv* env, jclass, - jlong j_instance, jstring j_path, jobject j_curvestat, jint j_mask) { + jlong j_instance, jstring j_path, jobject j_stat, jint j_mask) { uintptr_t instance = static_cast(j_instance); const char* path = env->GetStringUTFChars(j_path, NULL); int to_set = fixup_attr_mask(j_mask); @@ -613,11 +689,11 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSSetAttr struct stat stat; memset(&stat, 0, sizeof(stat)); - stat.st_mode = env->GetIntField(j_curvestat, curvestat_mode_fid); - stat.st_uid = env->GetIntField(j_curvestat, curvestat_uid_fid); - stat.st_gid = env->GetIntField(j_curvestat, curvestat_gid_fid); - uint64_t mtime_msec = env->GetLongField(j_curvestat, curvestat_m_time_fid); - uint64_t atime_msec = env->GetLongField(j_curvestat, curvestat_a_time_fid); + stat.st_mode = env->GetIntField(j_stat, stat_mode_fid); + stat.st_uid = env->GetIntField(j_stat, stat_uid_fid); + stat.st_gid = env->GetIntField(j_stat, stat_gid_fid); + uint64_t mtime_msec = env->GetLongField(j_stat, stat_mtime_fid); + uint64_t atime_msec = env->GetLongField(j_stat, stat_atime_fid); stat.st_mtim.tv_sec = mtime_msec / 1000; stat.st_mtim.tv_nsec = (mtime_msec % 1000) * 1000000; stat.st_atim.tv_sec = atime_msec / 1000; @@ -630,9 +706,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSSetAttr return rc; } -// nativeCurveFSChmod: curvefs_chmod +// nativeCurveFsChmod: curvefs_chmod JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChmod +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsChmod (JNIEnv* env, jclass, jlong j_instance, jstring j_path, jint j_mode) { uintptr_t instance = static_cast(j_instance); uint16_t mode = static_cast(j_mode); @@ -648,9 +724,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChmod return rc; } -// nativeCurveFSChown: curvefs_chown +// nativeCurveFsChown: curvefs_chown JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChown +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsChown (JNIEnv* env, jclass, jlong j_instance, jstring j_path, jint j_uid, jint j_gid) { uintptr_t instance = static_cast(j_instance); @@ -668,9 +744,9 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSChown return rc; } -// nativeCurveFSRename: curvefs_rename +// nativeCurveFsRename: curvefs_rename JNIEXPORT jint -JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRename +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRename (JNIEnv* env, jclass, jlong j_instance, jstring j_src, jstring j_dst) { uintptr_t instance = static_cast(j_instance); const char* src = env->GetStringUTFChars(j_src, NULL); @@ -686,3 +762,37 @@ JNICALL Java_io_opencurve_curve_fs_libfs_CurveFSMount_nativeCurveFSRename } return rc; } + +// nativeCurveFsRemove: curvefs_remove +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRemove + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_remove(instance, path); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} + +// nativeCurveFsRemoveAll: curvefs_removeall +JNIEXPORT jint +JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRemoveAll + (JNIEnv* env, jclass, jlong j_instance, jstring j_path) { + uintptr_t instance = static_cast(j_instance); + const char* path = env->GetStringUTFChars(j_path, NULL); + auto defer = absl::MakeCleanup([&]() { + env->ReleaseStringUTFChars(j_path, path); + }); + + int rc = curvefs_removeall(instance, path); + if (rc != 0) { + handle_error(env, rc); + } + return rc; +} diff --git a/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.h b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.h new file mode 100644 index 0000000000..0e75e1b4a5 --- /dev/null +++ b/curvefs/sdk/java/native/io_opencurve_curve_fs_libfs_CurveFsMount.h @@ -0,0 +1,245 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class io_opencurve_curve_fs_libfs_CurveFsMount */ + +#ifndef _Included_io_opencurve_curve_fs_libfs_CurveFsMount +#define _Included_io_opencurve_curve_fs_libfs_CurveFsMount +#ifdef __cplusplus +extern "C" { +#endif +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_RDONLY +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_RDONLY 1L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_RDWR +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_RDWR 2L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_APPEND +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_APPEND 4L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_CREAT +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_CREAT 8L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_TRUNC +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_TRUNC 16L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_EXCL +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_EXCL 32L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_WRONLY +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_WRONLY 64L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_O_DIRECTORY +#define io_opencurve_curve_fs_libfs_CurveFsMount_O_DIRECTORY 128L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_SET +#define io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_SET 0L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_CUR +#define io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_CUR 1L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_END +#define io_opencurve_curve_fs_libfs_CurveFsMount_SEEK_END 2L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_MODE +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_MODE 1L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_UID +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_UID 2L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_GID +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_GID 4L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_MTIME +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_MTIME 8L +#undef io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_ATIME +#define io_opencurve_curve_fs_libfs_CurveFsMount_SETATTR_ATIME 16L +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsNew + * Signature: ()J + */ +JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsNew + (JNIEnv *, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsDelete + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsDelete + (JNIEnv *, jobject, jlong); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsConfSet + * Signature: (JLjava/lang/String;Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsConfSet + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsMount + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsMount + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsUmount + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsUmount + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsMkDirs + * Signature: (JLjava/lang/String;I)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsMkDirs + (JNIEnv *, jclass, jlong, jstring, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsRmDir + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRmDir + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsListDir + * Signature: (JLjava/lang/String;)[Lio/opencurve/curve/fs/libfs/CurveFsMount/Dirent; + */ +JNIEXPORT jobjectArray JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsListDir + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsCreate + * Signature: (JLjava/lang/String;ILio/opencurve/curve/fs/libfs/CurveFsMount/File;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsCreate + (JNIEnv *, jclass, jlong, jstring, jint, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsOpen + * Signature: (JLjava/lang/String;ILio/opencurve/curve/fs/libfs/CurveFsMount/File;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsOpen + (JNIEnv *, jclass, jlong, jstring, jint, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsLSeek + * Signature: (JIJI)J + */ +JNIEXPORT jlong JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsLSeek + (JNIEnv *, jclass, jlong, jint, jlong, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativieCurveFsRead + * Signature: (JIJ[BJ)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativieCurveFsRead + (JNIEnv *, jclass, jlong, jint, jlong, jbyteArray, jlong); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativieCurveFsWrite + * Signature: (JIJ[BJ)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativieCurveFsWrite + (JNIEnv *, jclass, jlong, jint, jlong, jbyteArray, jlong); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsFSync + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsFSync + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsClose + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsClose + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsUnlink + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsUnlink + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsStatFs + * Signature: (JLio/opencurve/curve/fs/libfs/CurveFsMount/StatVfs;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsStatFs + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsLStat + * Signature: (JLjava/lang/String;Lio/opencurve/curve/fs/libfs/CurveFsMount/Stat;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsLStat + (JNIEnv *, jclass, jlong, jstring, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsFStat + * Signature: (JILio/opencurve/curve/fs/libfs/CurveFsMount/Stat;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsFStat + (JNIEnv *, jclass, jlong, jint, jobject); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsSetAttr + * Signature: (JLjava/lang/String;Lio/opencurve/curve/fs/libfs/CurveFsMount/Stat;I)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsSetAttr + (JNIEnv *, jclass, jlong, jstring, jobject, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsChmod + * Signature: (JLjava/lang/String;I)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsChmod + (JNIEnv *, jclass, jlong, jstring, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsChown + * Signature: (JLjava/lang/String;II)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsChown + (JNIEnv *, jclass, jlong, jstring, jint, jint); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsRename + * Signature: (JLjava/lang/String;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRename + (JNIEnv *, jclass, jlong, jstring, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsRemove + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRemove + (JNIEnv *, jclass, jlong, jstring); + +/* + * Class: io_opencurve_curve_fs_libfs_CurveFsMount + * Method: nativeCurveFsRemoveAll + * Signature: (JLjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_io_opencurve_curve_fs_libfs_CurveFsMount_nativeCurveFsRemoveAll + (JNIEnv *, jclass, jlong, jstring); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSProto.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSProto.java deleted file mode 100644 index df8ab69b21..0000000000 --- a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSProto.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2023 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: Curve - * Created Date: 2023-08-01 - * Author: NetEase Media Bigdata - */ - -package io.opencurve.curve.fs.hadoop; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import io.opencurve.curve.fs.libfs.CurveFSStat; -import io.opencurve.curve.fs.libfs.CurveFSStatVFS; - -import java.io.IOException; -import java.net.URI; - -abstract class CurveFSProto { - // init* - abstract void initialize(URI uri, Configuration conf) throws IOException; - abstract void shutdown() throws IOException; - // directory* - abstract void mkdirs(Path path, int mode) throws IOException; - abstract void rmdir(Path path) throws IOException; - abstract String[] listdir(Path path) throws IOException; - // file* - abstract int open(Path path, int flags, int mode) throws IOException; - abstract long lseek(int fd, long offset, int whence) throws IOException; - abstract int write(int fd, byte[] buf, long size, long offset) throws IOException; - abstract int read(int fd, byte[] buf, long size, long offset) throws IOException; - abstract void fsync(int fd) throws IOException; - abstract void close(int fd) throws IOException; - abstract void unlink(Path path) throws IOException; - // others - abstract void statfs(Path path, CurveFSStatVFS stat) throws IOException; - abstract void lstat(Path path, CurveFSStat stat) throws IOException; - abstract void fstat(int fd, CurveFSStat stat) throws IOException; - abstract void setattr(Path path, CurveFSStat stat, int mask) throws IOException; - abstract void chmod(Path path, int mode) throws IOException; - abstract void chown(Path path, int uid, int gid) throws IOException; - abstract void rename(Path src, Path dst) throws IOException; -} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSTalker.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSTalker.java deleted file mode 100644 index 26d0492142..0000000000 --- a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSTalker.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (c) 2023 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: Curve - * Created Date: 2023-08-01 - * Author: NetEase Media Bigdata - */ - -package io.opencurve.curve.fs.hadoop; - -import org.apache.commons.logging.Log; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import io.opencurve.curve.fs.libfs.CurveFSMount; -import io.opencurve.curve.fs.libfs.CurveFSStat; -import io.opencurve.curve.fs.libfs.CurveFSStatVFS; - -import java.util.UUID; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.util.Map; - -class CurveFSTalker extends CurveFSProto { - private CurveFSMount mount; - private String fsname = null; - private String mountpoint = null; - private boolean inited = false; - - private static final String PREFIX_KEY = "curvefs"; - - CurveFSTalker(Configuration conf, Log log) { - mount = null; - } - - private String tostr(Path path) { - if (null == path) { - return "/"; - } - return path.toUri().getPath(); - } - - private void loadCfg(Configuration conf) { - Map m = conf.getValByRegex("^" + PREFIX_KEY + "\\..*"); - for (Map.Entry entry : m.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - if (key.equals(PREFIX_KEY + ".name")) { - fsname = value; - } else { - mount.confSet(key.substring(PREFIX_KEY.length() + 1), value); - } - } - } - - @Override - void initialize(URI uri, Configuration conf) throws IOException { - mount = new CurveFSMount(); - loadCfg(conf); - if (null == fsname || fsname.isEmpty()) { - throw new IOException("curvefs.name is not set"); - } - mountpoint = UUID.randomUUID().toString(); - mount.mount(fsname, mountpoint); - inited = true; - } - - @Override - void shutdown() throws IOException { - if (inited) { - mount.umount(fsname, mountpoint); - mount = null; - inited = false; - } - } - - @Override - void mkdirs(Path path, int mode) throws IOException { - mount.mkdirs(tostr(path), mode); - } - - @Override - void rmdir(Path path) throws IOException { - mount.rmdir(tostr(path)); - } - - @Override - String[] listdir(Path path) throws IOException { - CurveFSStat stat = new CurveFSStat(); - try { - mount.lstat(tostr(path), stat); - } catch (FileNotFoundException e) { - return null; - } - if (!stat.isDir()) { - return null; - } - - return mount.listdir(tostr(path)); - } - - @Override - int open(Path path, int flags, int mode) throws IOException { - return mount.open(tostr(path), flags, mode); - } - - @Override - long lseek(int fd, long offset, int whence) throws IOException { - return mount.lseek(fd, offset, whence); - } - - @Override - int write(int fd, byte[] buf, long size, long offset) throws IOException { - return mount.write(fd, buf, size, offset); - } - - @Override - int read(int fd, byte[] buf, long size, long offset) throws IOException { - return mount.read(fd, buf, size, offset); - } - - @Override - void fsync(int fd) throws IOException { - mount.fsync(fd); - } - - @Override - void close(int fd) throws IOException { - mount.close(fd); - } - - @Override - void unlink(Path path) throws IOException { - mount.unlink(tostr(path)); - } - - @Override - void statfs(Path path, CurveFSStatVFS stat) throws IOException { - mount.statfs(tostr(path), stat); - } - - @Override - void lstat(Path path, CurveFSStat stat) throws IOException { - mount.lstat(tostr(path), stat); - } - - @Override - void fstat(int fd, CurveFSStat stat) throws IOException { - mount.fstat(fd, stat); - } - - @Override - void setattr(Path path, CurveFSStat stat, int mask) throws IOException { - mount.setattr(tostr(path), stat, mask); - } - - @Override - void chmod(Path path, int mode) throws IOException { - mount.chmod(tostr(path), mode); - } - - @Override - void chown(Path path, int uid, int gid) throws IOException { - mount.chown(tostr(path), uid, gid); - } - - @Override - void rename(Path src, Path dst) throws IOException { - mount.rename(tostr(src), tostr(dst)); - } -} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java index fc031d38d8..8f1239afcf 100644 --- a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFileSystem.java @@ -22,368 +22,446 @@ package io.opencurve.curve.fs.hadoop; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import java.net.URI; +import java.io.IOException; +import java.io.OutputStream; +import java.io.FileNotFoundException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; -import io.opencurve.curve.fs.libfs.CurveFSMount; -import io.opencurve.curve.fs.libfs.CurveFSStat; -import io.opencurve.curve.fs.libfs.CurveFSStatVFS; -import io.opencurve.curve.fs.hadoop.permission.Permission; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; +import io.opencurve.curve.fs.hadoop.permission.Permission; +import io.opencurve.curve.fs.libfs.CurveFsProto; +import io.opencurve.curve.fs.libfs.CurveFsMount; +import io.opencurve.curve.fs.libfs.CurveFsMount.StatVfs; +import io.opencurve.curve.fs.libfs.CurveFsMount.Stat; +import io.opencurve.curve.fs.libfs.CurveFsMount.File; +import io.opencurve.curve.fs.libfs.CurveFsMount.Dirent; +import io.opencurve.curve.fs.libfs.CurveFsException.NotADirectoryException; public class CurveFileSystem extends FileSystem { - private static final Log LOG = LogFactory.getLog(CurveFileSystem.class); + private static final int O_RDONLY = CurveFsMount.O_RDONLY; + private static final int O_WRONLY = CurveFsMount.O_WRONLY; + private static final int O_APPEND = CurveFsMount.O_APPEND; + private static final int SETATTR_MTIME = CurveFsMount.SETATTR_MTIME; + private static final int SETATTR_ATIME = CurveFsMount.SETATTR_ATIME; private URI uri; private Path workingDir; - private CurveFSProto curve = null; - private Permission perm = null; + private CurveFsProto curvefs = null; + private Permission permission = null; public CurveFileSystem() {} - public CurveFileSystem(Configuration conf) { - setConf(conf); - } + public CurveFileSystem(Configuration conf) { setConf(conf); } - private Path makeAbsolute(Path path) { - if (path.isAbsolute()) { - return path; - } - return new Path(workingDir, path); + // e.g. /my/dir1 + private String makeAbsolute(Path path) { + return makeQualified(path).toUri().getPath(); } + /** + * Get the current working directory for the given FileSystem + * @return the directory pathname + */ @Override - public URI getUri() { - return uri; + public Path getWorkingDirectory() { + return workingDir; } + /** + * Set the current working directory for the given FileSystem. All relative + * paths will be resolved relative to it. + * + * @param new_dir Path of new working directory + */ @Override - public String getScheme() { - return "hdfs"; + public void setWorkingDirectory(Path new_dir) { + workingDir = fixRelativePart(new_dir); + checkPath(workingDir); } + /** Called after a new FileSystem instance is constructed. + * @param name a uri whose authority section names the host, port, etc. + * for this FileSystem + * @param conf the configuration + */ @Override public void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); - if (curve == null) { - curve = new CurveFSTalker(conf, LOG); + if (curvefs == null) { + curvefs = new CurveFsMount(); } - if (perm == null) { - perm = new Permission(); + if (permission == null) { + permission = new Permission(); } - perm.initialize(conf); - curve.initialize(uri, conf); + curvefs.initialize(uri, conf); + permission.initialize(conf); setConf(conf); this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); this.workingDir = getHomeDirectory(); } - @Override - public FSDataInputStream open(Path path, int bufferSize) throws IOException { - path = makeAbsolute(path); - - int fd = curve.open(path, CurveFSMount.O_RDONLY, 0); - - CurveFSStat stat = new CurveFSStat(); - curve.fstat(fd, stat); - - CurveFSInputStream istream = new CurveFSInputStream(getConf(), curve, fd, stat.size, bufferSize); - return new FSDataInputStream(istream); - } - + /** + * No more filesystem operations are needed. Will + * release any held locks. + */ @Override public void close() throws IOException { super.close(); - curve.shutdown(); + curvefs.shutdown(); } + /** + * Return the protocol scheme for the FileSystem. + *

+ * This implementation throws an UnsupportedOperationException. + * + * @return the protocol scheme for the FileSystem. + */ @Override - public FSDataOutputStream append(Path path, int bufferSize, Progressable progress) throws IOException { - path = makeAbsolute(path); - - if (progress != null) { - progress.progress(); - } - - int fd = curve.open(path, CurveFSMount.O_WRONLY | CurveFSMount.O_APPEND, 0); - if (progress != null) { - progress.progress(); - } - - CurveFSOutputStream ostream = new CurveFSOutputStream(getConf(), curve, fd, bufferSize); - return new FSDataOutputStream(ostream, statistics); + public String getScheme() { + return "hdfs"; } + /** Returns a URI whose scheme and authority identify this FileSystem.*/ @Override - public Path getWorkingDirectory() { - return workingDir; + public URI getUri() { + return uri; } + /** + * Call {@link #mkdirs(Path, FsPermission)} with default permission. + */ @Override - public void setWorkingDirectory(Path dir) { - workingDir = makeAbsolute(dir); + public boolean mkdirs(Path f) throws IOException { + FsPermission perms = FsPermission.getDirDefault().applyUMask(FsPermission.getUMask(getConf())); + return mkdirs(f, perms); } + /** + * Make the given file and all non-existent parents into + * directories. Has the semantics of Unix 'mkdir -p'. + * Existence of the directory hierarchy is not an error. + * @param f path to create + * @param permission to apply to f + */ @Override - public boolean mkdirs(Path path, FsPermission perms) throws IOException { - path = makeAbsolute(path); - curve.mkdirs(path, (int) perms.toShort()); + public boolean mkdirs(Path f, FsPermission perms) throws IOException { + try { + curvefs.mkdirs(makeAbsolute(f), perms.toShort()); + } catch (IOException e) { + return false; + } return true; } - @Override - public boolean mkdirs(Path f) throws IOException { - FsPermission perms = FsPermission.getDirDefault().applyUMask(FsPermission.getUMask(getConf()));; - return mkdirs(f, perms); + private FileStatus newFileStatus(Path f, Stat stat) { + return new FileStatus(stat.size, + stat.isDirectory, + 1, + stat.blksize, + stat.mtime, + stat.atime, + new FsPermission((short) stat.mode), + permission.getUsername(stat.uid), + permission.getGroupname(stat.gid), + makeQualified(f)); // e.g. curvefs://my/dir1 } + /** + * Return a file status object that represents the path. + * @param f The path we want information from + * @return a FileStatus object + * @throws FileNotFoundException when the path does not exist; + * IOException see specific implementation + */ @Override - public FileStatus getFileStatus(Path path) throws IOException { - path = makeAbsolute(path); - - CurveFSStat stat = new CurveFSStat(); - curve.lstat(path, stat); - String owner = perm.getUsername(stat.uid);; - String group = perm.getGroupname(stat.gid);; - - FileStatus status = new FileStatus( - stat.size, stat.isDir(), 1, stat.blksize, - stat.m_time, stat.a_time, - new FsPermission((short) stat.mode), owner, group, - path.makeQualified(this)); - return status; + public FileStatus getFileStatus(Path f) throws IOException { + Stat stat = new Stat(); + curvefs.lstat(makeAbsolute(f), stat); + return newFileStatus(f, stat); } + /** + * List the statuses of the files/directories in the given path if the path is + * a directory. + * + * @param f given path + * @return the statuses of the files/directories in the given patch + * @throws FileNotFoundException when the path does not exist; + * IOException see specific implementation + */ @Override - public FileStatus[] listStatus(Path path) throws IOException { - path = makeAbsolute(path); - - if (isFile(path)) { - return new FileStatus[]{getFileStatus(path)}; + public FileStatus[] listStatus(Path f) throws IOException { + Dirent[] dirents; + try { + dirents = curvefs.listdir(makeAbsolute(f)); + } catch(NotADirectoryException e) { + return new FileStatus[]{ getFileStatus(f) }; } - String[] dirlist = curve.listdir(path); - if (dirlist != null) { - FileStatus[] status = new FileStatus[dirlist.length]; - for (int i = 0; i < status.length; i++) { - status[i] = getFileStatus(new Path(path, dirlist[i])); - } - return status; - } else { - throw new FileNotFoundException("File " + path + " does not exist."); + FileStatus[] statuses = new FileStatus[dirents.length]; + for (int i = 0; i < dirents.length; i++) { + Path p = makeQualified(new Path(f, new String(dirents[i].name))); + statuses[i] = newFileStatus(p, dirents[i].stat); } + return statuses; } - @Override - public void setPermission(Path path, FsPermission permission) throws IOException { - path = makeAbsolute(path); - curve.chmod(path, permission.toShort()); + private FSDataInputStream createFsDataInputStream(File file, + int bufferSize) throws IOException { + CurveFsInputStream istream = + new CurveFsInputStream(getConf(), curvefs, file.fd, file.length, bufferSize); + return new FSDataInputStream(istream); } - @Override - public void setTimes(Path path, long mtime, long atime) throws IOException { - path = makeAbsolute(path); - - CurveFSStat stat = new CurveFSStat(); - - int mask = 0; - if (mtime != -1) { - stat.m_time = mtime; - mask |= CurveFSMount.SETATTR_MTIME; - } - - if (atime != -1) { - stat.a_time = atime; - mask |= CurveFSMount.SETATTR_ATIME; - } - - curve.setattr(path, stat, mask); + private FSDataOutputStream createFsDataOutputStream(File file, + int bufferSize) throws IOException { + OutputStream ostream = + new CurveFsOutputStream(getConf(), curvefs, file.fd, bufferSize); + return new FSDataOutputStream(ostream, statistics); } + /** + * Opens an FSDataInputStream at the indicated Path. + * @param f the file name to open + * @param bufferSize the size of the buffer to be used. + */ @Override - public FSDataOutputStream create(Path path, FsPermission permission, boolean overwrite, int bufferSize, - short replication, long blockSize, Progressable progress) throws IOException { - path = makeAbsolute(path); - - boolean exists = exists(path); - - if (progress != null) { - progress.progress(); - } - - int flags = CurveFSMount.O_WRONLY | CurveFSMount.O_CREAT; - - if (exists) { - if (overwrite) { - flags |= CurveFSMount.O_TRUNC; - } else { - throw new FileAlreadyExistsException(); - } - } else { - Path parent = path.getParent(); - if (parent != null) { - if (!mkdirs(parent)) { - throw new IOException("mkdirs failed for " + parent.toString()); - } - } - } - - if (progress != null) { - progress.progress(); - } - - int fd = curve.open(path, flags, (int) permission.toShort()); - - if (progress != null) { - progress.progress(); - } - - OutputStream ostream = new CurveFSOutputStream(getConf(), curve, fd, bufferSize); - return new FSDataOutputStream(ostream, statistics); + public FSDataInputStream open(Path f, int bufferSize) throws IOException { + File file = new File(); + curvefs.open(makeAbsolute(f), O_RDONLY, file); + return createFsDataInputStream(file, bufferSize); } + /** + * Create an FSDataOutputStream at the indicated Path with write-progress + * reporting. + * @param f the file name to open + * @param permission + * @param overwrite if a file with this name already exists, then if true, + * the file will be overwritten, and if false an error will be thrown. + * @param bufferSize the size of the buffer to be used. + * @param replication required block replication for the file. + * @param blockSize + * @param progress + * @throws IOException + * @see #setPermission(Path, FsPermission) + */ @Override - public void setOwner(Path path, String username, String groupname) throws IOException { - CurveFSStat stat = new CurveFSStat(); - curve.lstat(path, stat); - - int uid = stat.uid; - int gid = stat.gid; - if (username != null) { - uid = perm.getUid(username); - } - if (groupname != null) { - gid = perm.getGid(groupname); + public FSDataOutputStream create(Path f, + FsPermission permission, + boolean overwrite, + int bufferSize, + short replication, + long blockSize, + Progressable progress) throws IOException { + File file = new File(); + for ( ; ; ) { + try { + curvefs.create(makeAbsolute(f), permission.toShort(), file); + } catch(FileNotFoundException e) { // parent directorty not exist + Path parent = makeQualified(f).getParent(); + try { + mkdirs(parent, FsPermission.getDirDefault()); + } catch (FileAlreadyExistsException ignored) {} + continue; // create file again + } catch(FileAlreadyExistsException e) { + if (!overwrite || isDirectory(f)) { + throw new FileAlreadyExistsException("File already exists: " + f); + } + delete(f, false); + continue; // create file again + } + break; } - - curve.chown(path, uid, gid); + return createFsDataOutputStream(file, bufferSize); } + /** + * Opens an FSDataOutputStream at the indicated Path with write-progress + * reporting. Same as create(), except fails if parent directory doesn't + * already exist. + * @param f the file name to open + * @param permission + * @param overwrite if a file with this name already exists, then if true, + * the file will be overwritten, and if false an error will be thrown. + * @param bufferSize the size of the buffer to be used. + * @param replication required block replication for the file. + * @param blockSize + * @param progress + * @throws IOException + * @see #setPermission(Path, FsPermission) + * @deprecated API only for 0.20-append + */ @Deprecated @Override - public FSDataOutputStream createNonRecursive(Path path, FsPermission permission, + public FSDataOutputStream createNonRecursive(Path f, + FsPermission permission, boolean overwrite, - int bufferSize, short replication, long blockSize, + int bufferSize, + short replication, + long blockSize, Progressable progress) throws IOException { - path = makeAbsolute(path); - - Path parent = path.getParent(); - - if (parent != null) { - CurveFSStat stat = new CurveFSStat(); - curve.lstat(parent, stat); - if (stat.isFile()) { - throw new FileAlreadyExistsException(parent.toString()); + File file = new File(); + for ( ; ; ) { + try { + curvefs.create(makeAbsolute(f), permission.toShort(), file); + } catch(FileAlreadyExistsException e) { + if (!overwrite || isDirectory(f)) { + throw new FileAlreadyExistsException("File already exists: " + f); + } + delete(f, false); + continue; // create file again } + break; } + return createFsDataOutputStream(file, bufferSize); + } - return this.create(path, permission, overwrite, - bufferSize, replication, blockSize, progress); + /** + * Append to an existing file (optional operation). + * Same as append(f, getConf().getInt("io.file.buffer.size", 4096), null) + * @param f the existing file to be appended. + * @throws IOException + */ + @Override + public FSDataOutputStream append(Path f, + int bufferSize, + Progressable progress) throws IOException { + File file = new File(); + curvefs.open(makeAbsolute(f), O_WRONLY | O_APPEND, file); + return createFsDataOutputStream(file, bufferSize); } + /** + * Renames Path src to Path dst. Can take place on local fs + * or remote DFS. + * @param src path to be renamed + * @param dst new path after rename + * @throws IOException on failure + * @return true if rename is successful + */ @Override public boolean rename(Path src, Path dst) throws IOException { - src = makeAbsolute(src); - dst = makeAbsolute(dst); - try { - CurveFSStat stat = new CurveFSStat(); - curve.lstat(dst, stat); - if (stat.isDir()) { - return rename(src, new Path(dst, src.getName())); - } + curvefs.rename(makeAbsolute(src), makeAbsolute(dst)); + } catch(FileNotFoundException e) { // src path not exist return false; - } catch (FileNotFoundException e) { + } catch (FileAlreadyExistsException e) { + FileStatus status = getFileStatus(dst); + if (!status.isDirectory()) { + return false; + } + // FIXME: } - try { - curve.rename(src, dst); - } catch (FileNotFoundException e) { - throw e; - } catch (Exception e) { - return false; - } return true; } - @Deprecated + /** Delete a file. + * + * @param f the path to delete. + * @param recursive if path is a directory and set to + * true, the directory is deleted else throws an exception. In + * case of a file the recursive can be set to either true or false. + * @return true if delete is successful else false. + * @throws IOException + */ @Override - public boolean delete(Path path) throws IOException { - return delete(path, false); - } - - @Override - public boolean delete(Path path, boolean recursive) throws IOException { - path = makeAbsolute(path); - - FileStatus status; + public boolean delete(Path f, boolean recursive) throws IOException { try { - status = getFileStatus(path); - } catch (FileNotFoundException e) { - return false; - } - - if (status.isFile()) { - curve.unlink(path); - return true; - } - - FileStatus[] dirlist = listStatus(path); - if (dirlist == null) { - return false; - } - - if (!recursive && dirlist.length > 0) { - throw new IOException("Directory " + path.toString() + "is not empty."); - } - - for (FileStatus fs : dirlist) { - if (!delete(fs.getPath(), recursive)) { - return false; + if (recursive) { + curvefs.removeall(makeAbsolute(f)); + } else { + curvefs.remove(makeAbsolute(f)); } + } catch (IOException e) { + return false; } - - curve.rmdir(path); return true; } + /** + * Returns a status object describing the use and capacity of the + * file system. If the file system has multiple partitions, the + * use and capacity of the partition pointed to by the specified + * path is reflected. + * @param p Path for which status should be obtained. null means + * the default partition. + * @return a FsStatus object + * @throws IOException + * see specific implementation + */ @Override public FsStatus getStatus(Path p) throws IOException { - CurveFSStatVFS stat = new CurveFSStatVFS(); - curve.statfs(p, stat); - - FsStatus status = new FsStatus(stat.bsize * stat.blocks, - stat.bsize * (stat.blocks - stat.bavail), - stat.bsize * stat.bavail); - return status; + StatVfs statvfs = new StatVfs(); + curvefs.statfs(makeAbsolute(p), statvfs); + return new FsStatus(statvfs.bsize * statvfs.blocks, // capacity + statvfs.bsize * (statvfs.blocks - statvfs.bavail), // used + statvfs.bsize * statvfs.bavail); // remaining } + /** + * Set permission of a path. + * @param p + * @param permission + */ @Override - public short getDefaultReplication() { - return 1; + public void setPermission(Path p, FsPermission permission) throws IOException { + curvefs.chmod(makeAbsolute(p), permission.toShort()); } + /** + * Set owner of a path (i.e. a file or a directory). + * The parameters username and groupname cannot both be null. + * @param p The path + * @param username If it is null, the original username remains unchanged. + * @param groupname If it is null, the original groupname remains unchanged. + */ @Override - public long getDefaultBlockSize() { - return super.getDefaultBlockSize(); - } + public void setOwner(Path p, String username, String groupname) throws IOException { + Stat stat = new Stat(); + curvefs.lstat(makeAbsolute(p), stat); // TODO(Wine93): remove this operation - @Override - protected int getDefaultPort() { - return super.getDefaultPort(); + int uid = stat.uid; + int gid = stat.gid; + if (username != null) { + uid = permission.getUid(username); + } + if (groupname != null) { + gid = permission.getGid(groupname); + } + curvefs.chown(makeAbsolute(p), uid, gid); } + /** + * Set access time of a file + * @param p The path + * @param mtime Set the modification time of this file. + * The number of milliseconds since Jan 1, 1970. + * A value of -1 means that this call should not set modification time. + * @param atime Set the access time of this file. + * The number of milliseconds since Jan 1, 1970. + * A value of -1 means that this call should not set access time. + */ @Override - public String getCanonicalServiceName() { - return null; + public void setTimes(Path p, long mtime, long atime) throws IOException { + Stat stat = new Stat(); + int mask = 0; + if (mtime != -1) { + stat.mtime = mtime; + mask |= SETATTR_MTIME; + } + if (atime != -1) { + stat.atime = atime; + mask |= SETATTR_ATIME; + } + curvefs.setattr(makeAbsolute(p), stat, mask); } -} +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSInputStream.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsInputStream.java similarity index 92% rename from curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSInputStream.java rename to curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsInputStream.java index 25ad56564f..eb40992bf7 100644 --- a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSInputStream.java +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsInputStream.java @@ -26,7 +26,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSInputStream; -import io.opencurve.curve.fs.libfs.CurveFSMount; +import io.opencurve.curve.fs.libfs.CurveFsMount; +import io.opencurve.curve.fs.libfs.CurveFsProto; import java.io.IOException; @@ -35,15 +36,15 @@ * An {@link FSInputStream} for a CurveFileSystem and corresponding * Curve instance. */ -public class CurveFSInputStream extends FSInputStream { - private static final Log LOG = LogFactory.getLog(CurveFSInputStream.class); +public class CurveFsInputStream extends FSInputStream { + private static final Log LOG = LogFactory.getLog(CurveFsInputStream.class); private boolean closed; private int fileHandle; private long fileLength; - private CurveFSProto curve; + private CurveFsProto curvefs; private byte[] buffer; private int bufPos = 0; @@ -57,12 +58,12 @@ public class CurveFSInputStream extends FSInputStream { * @param flength The current length of the file. If the length changes * you will need to close and re-open it to access the new data. */ - public CurveFSInputStream(Configuration conf, CurveFSProto curvefs, + public CurveFsInputStream(Configuration conf, CurveFsProto curve, int fh, long flength, int bufferSize) { fileLength = flength; fileHandle = fh; closed = false; - curve = curvefs; + curvefs = curve; buffer = new byte[1<<21]; LOG.debug("CurveInputStream constructor: initializing stream with fh " + fh + " and file length " + flength); @@ -83,14 +84,14 @@ protected void finalize() throws Throwable { } private synchronized boolean fillBuffer() throws IOException { - bufValid = curve.read(fileHandle, buffer, buffer.length, -1); + bufValid = curvefs.read(fileHandle, -1, buffer, buffer.length); bufPos = 0; if (bufValid < 0) { int err = bufValid; bufValid = 0; - curve.lseek(fileHandle, curvePos, CurveFSMount.SEEK_SET); + curvefs.lseek(fileHandle, curvePos, CurveFsMount.SEEK_SET); throw new IOException("Failed to fill read buffer! Error code:" + err); } curvePos += bufValid; @@ -127,7 +128,7 @@ public synchronized void seek(long targetPos) throws IOException { } long oldPos = curvePos; - curvePos = curve.lseek(fileHandle, targetPos, CurveFSMount.SEEK_SET); + curvePos = curvefs.lseek(fileHandle, targetPos, CurveFsMount.SEEK_SET); bufValid = 0; bufPos = 0; if (curvePos < 0) { @@ -244,9 +245,9 @@ public synchronized int read(byte buf[], int off, int len) throws IOException { public void close() throws IOException { LOG.trace("CurveOutputStream.close:enter"); if (!closed) { - curve.close(fileHandle); + curvefs.close(fileHandle); closed = true; LOG.trace("CurveOutputStream.close:exit"); } } -} +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSOutputStream.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsOutputStream.java similarity index 87% rename from curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSOutputStream.java rename to curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsOutputStream.java index 06855bcd7d..8f24b3c7ca 100644 --- a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFSOutputStream.java +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/hadoop/CurveFsOutputStream.java @@ -25,7 +25,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import io.opencurve.curve.fs.libfs.CurveFSMount; +import io.opencurve.curve.fs.libfs.CurveFsMount; +import io.opencurve.curve.fs.libfs.CurveFsProto; import java.io.IOException; import java.io.OutputStream; @@ -41,13 +42,10 @@ * libcurvefs. Currently it might be useful to reduce JNI crossings, but not * much more. */ -public class CurveFSOutputStream extends OutputStream { +public class CurveFsOutputStream extends OutputStream { private boolean closed; - - private CurveFSProto curve; - + private CurveFsProto curvefs; private int fileHandle; - private byte[] buffer; private int bufUsed = 0; @@ -56,9 +54,11 @@ public class CurveFSOutputStream extends OutputStream { * @param conf The FileSystem configuration. * @param fh The Curve filehandle to connect to. */ - public CurveFSOutputStream(Configuration conf, CurveFSProto curvefs, - int fh, int bufferSize) { - curve = curvefs; + public CurveFsOutputStream(Configuration conf, + CurveFsProto curve, + int fh, + int bufferSize) { + curvefs = curve; fileHandle = fh; closed = false; buffer = new byte[1<<21]; @@ -92,7 +92,7 @@ private synchronized void checkOpen() throws IOException { */ public synchronized long getPos() throws IOException { checkOpen(); - return curve.lseek(fileHandle, 0, CurveFSMount.SEEK_CUR); + return curvefs.lseek(fileHandle, 0, CurveFsMount.SEEK_CUR); } @Override @@ -129,7 +129,7 @@ private synchronized void flushBuffer() throws IOException { } while (bufUsed > 0) { - int ret = curve.write(fileHandle, buffer, bufUsed, -1); + int ret = curvefs.write(fileHandle, -1, buffer, bufUsed); if (ret < 0) { throw new IOException("curve.write: ret=" + ret); } @@ -161,14 +161,14 @@ private synchronized void flushBuffer() throws IOException { public synchronized void flush() throws IOException { checkOpen(); flushBuffer(); // buffer -> libcurvefs - curve.fsync(fileHandle); // libcurvefs -> cluster + curvefs.fsync(fileHandle); // libcurvefs -> cluster } @Override public synchronized void close() throws IOException { checkOpen(); flush(); - curve.close(fileHandle); + curvefs.close(fileHandle); closed = true; } -} +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSMount.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSMount.java deleted file mode 100644 index 7b423dcc0f..0000000000 --- a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSMount.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright (c) 2023 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: Curve - * Created Date: 2023-07-07 - * Author: Jingli Chen (Wine93) - */ -package io.opencurve.curve.fs.libfs; - -import java.io.IOException; - -public class CurveFSMount { - // init - private native long nativeCurveFSCreate(); - private native void nativeCurveFSRelease(long instancePtr); - private static native void nativeCurveFSConfSet(long instancePtr, String key, String value); - private static native int nativeCurveFSMount(long instancePtr, String fsname, String mountpoint); - private static native int nativeCurveFSUmount(long instancePtr, String fsname, String mountpoint); - // dir* - private static native int nativeCurveFSMkDirs(long instancePtr, String path, int mode); - private static native int nativeCurveFSRmDir(long instancePtr, String path); - private static native String[] nativeCurveFSListDir(long instancePtr, String path); - // file* - private static native int nativeCurveFSOpen(long instancePtr, String path, int flags, int mode); - private static native long nativeCurveFSLSeek(long instancePtr, int fd, long offset, int whence); - private static native long nativieCurveFSRead(long instancePtr, int fd, byte[] buffer, long size, long offset); - private static native long nativieCurveFSWrite(long instancePtr, int fd, byte[] buffer, long size, long offset); - private static native int nativeCurveFSFSync(long instancePtr, int fd); - private static native int nativeCurveFSClose(long instancePtr, int fd); - private static native int nativeCurveFSUnlink(long instancePtr, String path); - // others - private static native int nativeCurveFSStatFs(long instancePtr, CurveFSStatVFS statvfs); - private static native int nativeCurveFSLstat(long instancePtr, String path, CurveFSStat stat); - private static native int nativeCurveFSFStat(long instancePtr, int fd, CurveFSStat stat); - private static native int nativeCurveFSSetAttr(long instancePtr, String path, CurveFSStat stat, int mask); - private static native int nativeCurveFSChmod(long instancePtr, String path, int mode); - private static native int nativeCurveFSChown(long instancePtr, String path, int uid, int gid); - private static native int nativeCurveFSRename(long instancePtr, String src, String dst); - - /* - * Flags for open(). - * - * Must be synchronized with JNI if changed. - */ - public static final int O_RDONLY = 1; - public static final int O_RDWR = 2; - public static final int O_APPEND = 4; - public static final int O_CREAT = 8; - public static final int O_TRUNC = 16; - public static final int O_EXCL = 32; - public static final int O_WRONLY = 64; - public static final int O_DIRECTORY = 128; - - /* - * Whence flags for seek(). - * - * Must be synchronized with JNI if changed. - */ - public static final int SEEK_SET = 0; - public static final int SEEK_CUR = 1; - public static final int SEEK_END = 2; - - /* - * Attribute flags for setattr(). - * - * Must be synchronized with JNI if changed. - */ - public static final int SETATTR_MODE = 1; - public static final int SETATTR_UID = 2; - public static final int SETATTR_GID = 4; - public static final int SETATTR_MTIME = 8; - public static final int SETATTR_ATIME = 16; - - private static final String CURVEFS_DEBUG_ENV_VAR = "CURVEFS_DEBUG"; - private static final String CLASS_NAME = "io.opencurve.curve.fs.CurveFSMount"; - - private long instancePtr; - - private static void accessLog(String name, String... args) { - String value = System.getenv(CURVEFS_DEBUG_ENV_VAR); - if (!Boolean.valueOf(value)) { - return; - } - - String params = String.join(",", args); - String message = String.format("%s.%s(%s)", CLASS_NAME, name, params); - System.out.println(message); - } - - static { - accessLog("loadLibrary"); - try { - CurveFSNativeLoader.getInstance().loadLibrary(); - } catch(Exception e) {} - } - - protected void finalize() throws Throwable { - accessLog("finalize"); - } - - public CurveFSMount() { - accessLog("CurveMount"); - instancePtr = nativeCurveFSCreate(); - } - - public void confSet(String key, String value) { - accessLog("confSet", key, value); - nativeCurveFSConfSet(instancePtr, key, value); - } - - public void mount(String fsname, String mountpoint) throws IOException { - accessLog("mount"); - nativeCurveFSMount(instancePtr, fsname, mountpoint); - } - - public void umount(String fsname, String mountpoint) throws IOException { - accessLog("umount"); - nativeCurveFSUmount(instancePtr, fsname, mountpoint); - } - - public void shutdown() throws IOException { - accessLog("shutdown"); - } - - // directory* - public void mkdirs(String path, int mode) throws IOException { - accessLog("mkdirs", path.toString()); - nativeCurveFSMkDirs(instancePtr, path, mode); - } - - public void rmdir(String path) throws IOException { - accessLog("rmdir", path.toString()); - nativeCurveFSRmDir(instancePtr, path); - } - - public String[] listdir(String path) throws IOException { - accessLog("listdir", path.toString()); - return nativeCurveFSListDir(instancePtr, path); - } - - // file* - public int open(String path, int flags, int mode) throws IOException { - accessLog("open", path.toString()); - return nativeCurveFSOpen(instancePtr, path, flags, mode); - } - - public long lseek(int fd, long offset, int whence) throws IOException { - accessLog("lseek", String.valueOf(fd), String.valueOf(offset), String.valueOf(whence)); - return nativeCurveFSLSeek(instancePtr, fd, offset, whence); - } - - public int read(int fd, byte[] buf, long size, long offset) throws IOException { - accessLog("read", String.valueOf(fd), String.valueOf(size), String.valueOf(size)); - long rc = nativieCurveFSRead(instancePtr, fd, buf, size, offset); - return (int) rc; - } - - public int write(int fd, byte[] buf, long size, long offset) throws IOException { - accessLog("write", String.valueOf(fd), String.valueOf(size), String.valueOf(size)); - long rc = nativieCurveFSWrite(instancePtr, fd, buf, size, offset); - return (int) rc; - } - - public void fsync(int fd) throws IOException { - accessLog("fsync", String.valueOf(fd)); - nativeCurveFSFSync(instancePtr, fd); - } - - public void close(int fd) throws IOException { - accessLog("close", String.valueOf(fd)); - nativeCurveFSClose(instancePtr, fd); - } - - public void unlink(String path) throws IOException { - accessLog("unlink", path.toString()); - nativeCurveFSUnlink(instancePtr, path); - } - - // others - public void statfs(String path, CurveFSStatVFS statvfs) throws IOException { - accessLog("statfs", path.toString()); - nativeCurveFSStatFs(instancePtr, statvfs); - } - - public void lstat(String path, CurveFSStat stat) throws IOException { - accessLog("lstat", path.toString()); - nativeCurveFSLstat(instancePtr, path, stat); - } - - public void fstat(int fd, CurveFSStat stat) throws IOException { - accessLog("fstat", String.valueOf(fd)); - nativeCurveFSFStat(instancePtr, fd, stat); - } - - public void setattr(String path, CurveFSStat stat, int mask) throws IOException { - accessLog("setattr", path.toString()); - nativeCurveFSSetAttr(instancePtr, path, stat, mask); - } - - public void chmod(String path, int mode) throws IOException { - accessLog("chmod", path.toString()); - nativeCurveFSChmod(instancePtr, path, mode); - } - - public void chown(String path, int uid, int gid) throws IOException { - accessLog("chown", path.toString(), String.valueOf(uid), String.valueOf(gid)); - nativeCurveFSChown(instancePtr, path, uid, gid); - } - - public void rename(String src, String dst) throws IOException { - accessLog("rename", src.toString(), dst.toString()); - nativeCurveFSRename(instancePtr, src, dst); - } -} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStat.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStat.java deleted file mode 100644 index d188ca93f5..0000000000 --- a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStat.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2023 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: Curve - * Created Date: 2023-07-07 - * Author: Jingli Chen (Wine93) - */ - -package io.opencurve.curve.fs.libfs; - -public class CurveFSStat { - public boolean is_file; /* S_ISREG */ - public boolean is_directory; /* S_ISDIR */ - public boolean is_symlink; /* S_ISLNK */ - - public int mode; - public int uid; - public int gid; - public long size; - public long blksize; - public long blocks; - public long a_time; - public long m_time; - - public boolean isFile() { - return is_file; - } - - public boolean isDir() { - return is_directory; - } - - public boolean isSymlink() { - return is_symlink; - } -} diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStatVFS.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsException.java similarity index 67% rename from curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStatVFS.java rename to curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsException.java index 5ee2c1b8b0..c37adc7cf7 100644 --- a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSStatVFS.java +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsException.java @@ -16,18 +16,22 @@ /* * Project: Curve - * Created Date: 2023-07-07 + * Created Date: 2024-03-01 * Author: Jingli Chen (Wine93) */ package io.opencurve.curve.fs.libfs; -public class CurveFSStatVFS { - public long bsize; - public long frsize; - public long blocks; - public long bavail; - public long files; - public long fsid; - public long namemax; -} +import java.io.IOException; + +public class CurveFsException extends IOException { + public static class NotADirectoryException extends IOException { + public NotADirectoryException() { + super(); + } + + public NotADirectoryException(String s) { + super(s); + } + } +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsMount.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsMount.java new file mode 100644 index 0000000000..e9efb28b7d --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsMount.java @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2024 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2024-03-01 + * Author: Jingli Chen (Wine93) + */ + +package io.opencurve.curve.fs.libfs; + +import java.net.URI; +import java.util.Map; +import java.util.UUID; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; + +public class CurveFsMount extends CurveFsProto { + // init + private native long nativeCurveFsNew(); + private native void nativeCurveFsDelete(long instancePtr); + private static native void nativeCurveFsConfSet(long instancePtr, String key, String value); + private static native int nativeCurveFsMount(long instancePtr, String fsname, String mountpoint); + private static native int nativeCurveFsUmount(long instancePtr, String fsname, String mountpoint); + // dir* + private static native int nativeCurveFsMkDirs(long instancePtr, String path, int mode); + private static native int nativeCurveFsRmDir(long instancePtr, String path); + private static native Dirent[] nativeCurveFsListDir(long instancePtr, String path); + // file* + private static native int nativeCurveFsCreate(long instancePtr, String path, int mode, File file); + private static native int nativeCurveFsOpen(long instancePtr, String path, int flags, File file); + private static native long nativeCurveFsLSeek(long instancePtr, int fd, long offset, int whence); + private static native int nativieCurveFsRead(long instancePtr, int fd, long offset, byte[] buffer, long size); + private static native int nativieCurveFsWrite(long instancePtr, int fd, long offset, byte[] buffer, long size); + private static native int nativeCurveFsFSync(long instancePtr, int fd); + private static native int nativeCurveFsClose(long instancePtr, int fd); + private static native int nativeCurveFsUnlink(long instancePtr, String path); + // others + private static native int nativeCurveFsStatFs(long instancePtr, StatVfs statvfs); + private static native int nativeCurveFsLStat(long instancePtr, String path, Stat stat); + private static native int nativeCurveFsFStat(long instancePtr, int fd, Stat stat); + private static native int nativeCurveFsSetAttr(long instancePtr, String path, Stat stat, int mask); + private static native int nativeCurveFsChmod(long instancePtr, String path, int mode); + private static native int nativeCurveFsChown(long instancePtr, String path, int uid, int gid); + private static native int nativeCurveFsRename(long instancePtr, String src, String dst); + private static native int nativeCurveFsRemove(long instancePtr, String path); + private static native int nativeCurveFsRemoveAll(long instancePtr, String path); + + public static class StatVfs { + public long bsize; + public long frsize; + public long blocks; + public long bavail; + public long files; + public long fsid; + public long namemax; + } + + public static class Stat { + public int mode; + public int uid; + public int gid; + public long size; + public long blksize; + public long blocks; + public long atime; + public long mtime; + + public boolean isFile; + public boolean isDirectory; + public boolean isSymlink; + } + + public static class File { + public int fd; + public long length; + }; + + public static class Dirent { + public String name; + public Stat stat; + }; + + // Flags for open(): must be synchronized with JNI if changed. + public static final int O_RDONLY = 1; + public static final int O_RDWR = 2; + public static final int O_APPEND = 4; + public static final int O_CREAT = 8; + public static final int O_TRUNC = 16; + public static final int O_EXCL = 32; + public static final int O_WRONLY = 64; + public static final int O_DIRECTORY = 128; + + // Whence flags for seek(): must be synchronized with JNI if changed. + public static final int SEEK_SET = 0; + public static final int SEEK_CUR = 1; + public static final int SEEK_END = 2; + + // Attribute flags for setattr(): must be synchronized with JNI if changed. + public static final int SETATTR_MODE = 1; + public static final int SETATTR_UID = 2; + public static final int SETATTR_GID = 4; + public static final int SETATTR_MTIME = 8; + public static final int SETATTR_ATIME = 16; + + private static final String PREFIX_KEY = "curvefs"; + public static final String REGEX_CURVEFS_CONF = "^" + PREFIX_KEY + "\\..*"; + public static final String KEY_FSNAME = "curvefs.name"; + + private static long instancePtr; + private static String fsname; + private static String mountpoint; + private static boolean initialized = false; + + static { + try { + CurveFsNativeLoader.getInstance().loadLibrary(); + } catch(Exception e) { + System.out.println("Load curvefs native library failed: " + e.getMessage()); + } + } + + public CurveFsMount() {} + + private void setConf(Configuration conf) throws IOException { + Map m = conf.getValByRegex(REGEX_CURVEFS_CONF); + for (Map.Entry entry : m.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.equals(KEY_FSNAME)) { + fsname = value; + continue; + } + nativeCurveFsConfSet(instancePtr, key.substring(PREFIX_KEY.length() + 1), value); + } + + if (null == fsname || fsname.isEmpty()) { + throw new IOException(KEY_FSNAME + " is not set"); + } + mountpoint = UUID.randomUUID().toString(); + } + + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + if (initialized) { + return; + } + instancePtr = nativeCurveFsNew(); + setConf(conf); + nativeCurveFsMount(instancePtr, fsname, mountpoint); + initialized = true; + } + + @Override + public void shutdown() throws IOException { + if (!initialized) { + return; + } + nativeCurveFsUmount(instancePtr, fsname, mountpoint); + nativeCurveFsDelete(instancePtr); + initialized = false; + } + + @Override + public void mkdirs(String path, int mode) throws IOException { + nativeCurveFsMkDirs(instancePtr, path, mode); + } + + @Override + public void rmdir(String path) throws IOException { + nativeCurveFsRmDir(instancePtr, path); + } + + @Override + public Dirent[] listdir(String path) throws IOException { + return nativeCurveFsListDir(instancePtr, path); + } + + @Override + public void create(String path, int mode, File file) throws IOException { + nativeCurveFsCreate(instancePtr, path, mode, file); + } + + public void open(String path, int flags, File file) throws IOException { + nativeCurveFsOpen(instancePtr, path, flags, file); + } + + @Override + public long lseek(int fd, long offset, int whence) throws IOException { + return nativeCurveFsLSeek(instancePtr, fd, offset, whence); + } + + @Override + public int read(int fd, long offset, byte[] buffer, long size) throws IOException { + return nativieCurveFsRead(instancePtr, fd, offset, buffer, size); + } + + @Override + public int write(int fd, long offset, byte[] buffer, long size) throws IOException { + return nativieCurveFsWrite(instancePtr, fd, offset, buffer, size); + } + + @Override + public void fsync(int fd) throws IOException { + nativeCurveFsFSync(instancePtr, fd); + } + + @Override + public void close(int fd) throws IOException { + nativeCurveFsClose(instancePtr, fd); + } + + @Override + public void unlink(String path) throws IOException { + nativeCurveFsUnlink(instancePtr, path); + } + + @Override + public void statfs(String path, StatVfs statvfs) throws IOException { + nativeCurveFsStatFs(instancePtr, statvfs); + } + + @Override + public void lstat(String path, Stat stat) throws IOException { + nativeCurveFsLStat(instancePtr, path, stat); + } + + @Override + public void fstat(int fd, Stat stat) throws IOException { + nativeCurveFsFStat(instancePtr, fd, stat); + } + + @Override + public void setattr(String path, Stat stat, int mask) throws IOException { + nativeCurveFsSetAttr(instancePtr, path, stat, mask); + } + + @Override + public void chmod(String path, int mode) throws IOException { + nativeCurveFsChmod(instancePtr, path, mode); + } + + @Override + public void chown(String path, int uid, int gid) throws IOException { + nativeCurveFsChown(instancePtr, path, uid, gid); + } + + @Override + public void rename(String src, String dst) throws IOException { + nativeCurveFsRename(instancePtr, src, dst); + } + + @Override + public void remove(String path) throws IOException { + nativeCurveFsRemove(instancePtr, path); + } + + @Override + public void removeall(String path) throws IOException { + nativeCurveFsRemoveAll(instancePtr, path); + } +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSNativeLoader.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsNativeLoader.java similarity index 86% rename from curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSNativeLoader.java rename to curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsNativeLoader.java index 7ba7a535e8..95df921e51 100644 --- a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFSNativeLoader.java +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsNativeLoader.java @@ -13,60 +13,53 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - /* * Project: Curve * Created Date: 2023-07-07 * Author: Jingli Chen (Wine93) */ - package io.opencurve.curve.fs.libfs; - import java.net.URL; import java.net.URLConnection; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; - import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.utils.IOUtils; -public class CurveFSNativeLoader { +public class CurveFsNativeLoader { boolean initialized = false; - private static final CurveFSNativeLoader instance = new CurveFSNativeLoader(); + private static final CurveFsNativeLoader instance = new CurveFsNativeLoader(); private static final String TMP_DIR = "/tmp"; private static final String CURVEFS_LIBRARY_PATH = "/tmp/libcurvefs"; private static final String RESOURCE_TAR_NAME = "libcurvefs.tar"; private static final String JNI_LIBRARY_NAME = "libcurvefs_jni.so"; - private CurveFSNativeLoader() {} + private CurveFsNativeLoader() {} - public static CurveFSNativeLoader getInstance() { + public static CurveFsNativeLoader getInstance() { return instance; } public long getJarModifiedTime() throws IOException { - URL location = CurveFSNativeLoader.class.getProtectionDomain().getCodeSource().getLocation(); + URL location = CurveFsNativeLoader.class.getProtectionDomain().getCodeSource().getLocation(); URLConnection conn = location.openConnection(); return conn.getLastModified(); } - public void descompress(InputStream in, String dest) throws IOException { File dir = new File(dest); if (!dir.exists()) { dir.mkdirs(); } - ArchiveEntry entry; TarArchiveInputStream reader = new TarArchiveInputStream(in); while ((entry = reader.getNextTarEntry()) != null) { if (entry.isDirectory()) { continue; } - String path = TMP_DIR + File.separator + entry.getName(); File file = new File(path); IOUtils.copy(reader, new FileOutputStream(file)); @@ -74,25 +67,23 @@ public void descompress(InputStream in, String dest) throws IOException { reader.close(); } - public void loadJniLibrary() throws IOException { + public void loadJNILibrary() throws IOException { File libFile = new File(CURVEFS_LIBRARY_PATH, JNI_LIBRARY_NAME); System.load(libFile.getAbsolutePath()); } - public synchronized void loadLibrary() throws IOException { if (initialized) { return; } - long jarModifiedTime = getJarModifiedTime(); File libDir = new File(CURVEFS_LIBRARY_PATH); if (libDir.exists() && libDir.lastModified() == jarModifiedTime) { - loadJniLibrary(); + loadJNILibrary(); initialized = true; return; } - InputStream reader = CurveFSNativeLoader.class.getResourceAsStream("/" + RESOURCE_TAR_NAME); + InputStream reader = CurveFsNativeLoader.class.getResourceAsStream("/" + RESOURCE_TAR_NAME); if (reader == null) { throw new IOException("Cannot get resource " + RESOURCE_TAR_NAME + " from Jar file."); } @@ -100,7 +91,7 @@ public synchronized void loadLibrary() throws IOException { reader.close(); libDir.setLastModified(jarModifiedTime); - loadJniLibrary(); + loadJNILibrary(); initialized = true; } -} +} \ No newline at end of file diff --git a/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsProto.java b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsProto.java new file mode 100644 index 0000000000..427de2d811 --- /dev/null +++ b/curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/CurveFsProto.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: Curve + * Created Date: 2024-03-01 + * Author: Jingli Chen (Wine93) + */ + +package io.opencurve.curve.fs.libfs; + +import java.net.URI; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; + +import io.opencurve.curve.fs.libfs.CurveFsMount.StatVfs; +import io.opencurve.curve.fs.libfs.CurveFsMount.Stat; +import io.opencurve.curve.fs.libfs.CurveFsMount.File; +import io.opencurve.curve.fs.libfs.CurveFsMount.Dirent; + +public abstract class CurveFsProto { + public abstract void initialize(URI uri, Configuration conf) throws IOException; + public abstract void shutdown() throws IOException; + + public abstract void mkdirs(String path, int mode) throws IOException; + public abstract void rmdir(String path) throws IOException; + public abstract Dirent[] listdir(String path) throws IOException; + + public abstract void create(String path, int mode, File file) throws IOException; + public abstract void open(String path, int flags, File file) throws IOException; + public abstract long lseek(int fd, long offset, int whence) throws IOException; + public abstract int write(int fd, long offset, byte[] buffer, long size) throws IOException; + public abstract int read(int fd, long offset, byte[] buffer, long size) throws IOException; + public abstract void fsync(int fd) throws IOException; + public abstract void close(int fd) throws IOException; + public abstract void unlink(String path) throws IOException; + + public abstract void statfs(String path, StatVfs statvfs) throws IOException; + public abstract void lstat(String path, Stat stat) throws IOException; + public abstract void fstat(int fd, Stat stat) throws IOException; + public abstract void setattr(String path, Stat stat, int mask) throws IOException; + public abstract void chmod(String path, int mode) throws IOException; + public abstract void chown(String path, int uid, int gid) throws IOException; + public abstract void remove(String path) throws IOException; + public abstract void removeall(String path) throws IOException; + public abstract void rename(String src, String dst) throws IOException; +} diff --git a/curvefs/sdk/libcurvefs/examples/Makefile b/curvefs/sdk/libcurvefs/examples/Makefile index 2e4acf367d..676f6334d2 100644 --- a/curvefs/sdk/libcurvefs/examples/Makefile +++ b/curvefs/sdk/libcurvefs/examples/Makefile @@ -6,7 +6,8 @@ hrd_opt?= -I/curve build_opt?= -L$(so_path) link_opt?= -Wl,-rpath=$(so_path) -Wall flags?= $(hrd_opt) $(build_opt) $(link_opt) -targets= mkdir rmdir ls touch read write unlink append rename stat statfs fstat chmod +#targets= mkdir rmdir ls touch read write unlink append rename stat statfs fstat chmod +targets= ls libcurvefs: mkdir -p bin @@ -16,7 +17,7 @@ libcurvefs: --copt -g \ --copt -DUNINSTALL_SIGSEGV=1 \ --copt -DCLIENT_CONF_PATH="\"$(root)/curvefs/conf/client.conf\"" \ - //curvefs/sdk/java/native:curvefs_jni + //curvefs/sdk/libcurvefs:curvefs $(targets): libcurvefs gcc ${flags} $@.c -o bin/$@ -lcurvefs diff --git a/curvefs/sdk/libcurvefs/examples/common.h b/curvefs/sdk/libcurvefs/examples/common.h index dc262eb0b2..a80acd97b1 100644 --- a/curvefs/sdk/libcurvefs/examples/common.h +++ b/curvefs/sdk/libcurvefs/examples/common.h @@ -23,11 +23,7 @@ #include "curvefs/sdk/libcurvefs/libcurvefs.h" -const char* KEY_FSNAME = "CURVEFS_FSNAME"; -const char* KEY_S3_AK = "s3.ak"; -const char* KEY_S3_SK = "s3.sk"; -const char* KEY_S3_ENDPOINT = "s3.endpoint"; -const char* KEY_S3_BUCKET = "s3.bucket_name"; +const char* KEY_FSNAME = "curvefs.name"; const char* KEY_MDS_ADDRS = "mdsOpt.rpcRetryOpt.addrs"; char* @@ -42,7 +38,8 @@ require_string(const char* name) { char* get_filesystem_name() { - return require_string(KEY_FSNAME); + //return require_string(KEY_FSNAME); + return "perf02"; } char* @@ -52,12 +49,9 @@ get_mountpoint() { void load_cfg_from_environ(uintptr_t instance) { - curvefs_conf_set(instance, KEY_S3_AK, require_string(KEY_S3_AK)); - curvefs_conf_set(instance, KEY_S3_SK, require_string(KEY_S3_SK)); - curvefs_conf_set(instance, KEY_S3_ENDPOINT, - require_string(KEY_S3_ENDPOINT)); - curvefs_conf_set(instance, KEY_S3_BUCKET, require_string(KEY_S3_BUCKET)); - curvefs_conf_set(instance, KEY_MDS_ADDRS, require_string(KEY_MDS_ADDRS)); + //curvefs_conf_set(instance, KEY_MDS_ADDRS, require_string(KEY_MDS_ADDRS)); + curvefs_conf_set(instance, KEY_MDS_ADDRS, "10.221.103.160:6700,10.221.103.160:6701,10.221.103.160:6702"); + curvefs_conf_set(instance, "client.common.logDir", "/tmp"); curvefs_conf_set(instance, "fs.accessLogging", "true"); curvefs_conf_set(instance, "client.loglevel", "6"); curvefs_conf_set(instance, "diskCache.diskCacheType", "0"); diff --git a/curvefs/sdk/libcurvefs/examples/ls.c b/curvefs/sdk/libcurvefs/examples/ls.c index 71df5f30be..30ddb9764f 100644 --- a/curvefs/sdk/libcurvefs/examples/ls.c +++ b/curvefs/sdk/libcurvefs/examples/ls.c @@ -17,17 +17,17 @@ main(int argc, char** argv) { } // opendir - dir_stream_t dir_stream; - rc = curvefs_opendir(instance, argv[1], &dir_stream); + uint64_t fd; + rc = curvefs_opendir(instance, argv[1], &fd); if (rc != 0) { fprintf(stderr, "opendir failed: retcode = %d\n", rc); return rc; } // readdir - dirent_t dirent; + dirent_t dirents[8192]; for ( ;; ) { - ssize_t n = curvefs_readdir(instance, &dir_stream, &dirent); + ssize_t n = curvefs_readdir(instance, fd, dirents, 8192); if (n < 0) { rc = n; fprintf(stderr, "readdir failed: retcode = %d\n", rc); @@ -36,12 +36,14 @@ main(int argc, char** argv) { break; } - printf("%s: ino=%d size=%d\n", dirent.name, - dirent.stat.st_ino, - dirent.stat.st_size); + for (int i = 0; i < n; i++) { + printf("%s: ino=%d size=%d\n", dirents[i].name, + dirents[i].stat.st_ino, + dirents[i].stat.st_size); + } } - rc = curvefs_closedir(instance, &dir_stream); + rc = curvefs_closedir(instance, fd); if (rc != 0) { fprintf(stderr, "closedir failed: retcode = %d\n", rc); } diff --git a/curvefs/sdk/libcurvefs/libcurvefs.cpp b/curvefs/sdk/libcurvefs/libcurvefs.cpp index fb54c3b509..550b4b6f66 100644 --- a/curvefs/sdk/libcurvefs/libcurvefs.cpp +++ b/curvefs/sdk/libcurvefs/libcurvefs.cpp @@ -21,30 +21,25 @@ */ #include "curvefs/src/client/filesystem/error.h" +#include "curvefs/src/client/vfs/meta.h" #include "curvefs/sdk/libcurvefs/libcurvefs.h" using ::curvefs::client::filesystem::CURVEFS_ERROR; using ::curvefs::client::filesystem::SysErr; -using ::curvefs::client::vfs::DirStream; -using ::curvefs::client::vfs::DirEntry; +using ::curvefs::client::vfs::File; +using ::curvefs::client::vfs::Dirent; static curvefs_mount_t* get_instance(uintptr_t instance_ptr) { return reinterpret_cast(instance_ptr); } -static void stream_cast(DirStream* stream, dir_stream_t* dir_stream) { - dir_stream->fh = stream->fh; - dir_stream->ino = stream->ino; - dir_stream->offset = stream->offset; -} - -uintptr_t curvefs_create() { +uintptr_t curvefs_new() { auto mount = new curvefs_mount_t(); mount->cfg = Configure::Default(); return reinterpret_cast(mount); } -void curvefs_release(uintptr_t instance_ptr) { +void curvefs_delete(uintptr_t instance_ptr) { auto mount = get_instance(instance_ptr); delete mount; mount = nullptr; @@ -94,61 +89,52 @@ int curvefs_rmdir(uintptr_t instance_ptr, const char* path) { int curvefs_opendir(uintptr_t instance_ptr, const char* path, - dir_stream_t* dir_stream) { - DirStream stream; + uint64_t* fd) { auto mount = get_instance(instance_ptr); - auto rc = mount->vfs->OpenDir(path, &stream); - if (rc == CURVEFS_ERROR::OK) { - stream_cast(&stream, dir_stream); - } + auto rc = mount->vfs->OpenDir(path, fd); return SysErr(rc); } ssize_t curvefs_readdir(uintptr_t instance_ptr, - dir_stream_t* dir_stream, - dirent_t* dirent) { - DirEntry dirEntry; + uint64_t fd, + dirent_t dirents[], + size_t count) { + size_t nread = 0; auto mount = get_instance(instance_ptr); - DirStream* stream = reinterpret_cast(dir_stream); - auto rc = mount->vfs->ReadDir(stream, &dirEntry); + auto rc = mount->vfs->ReadDir( + fd, reinterpret_cast(dirents), count, &nread); if (rc == CURVEFS_ERROR::OK) { - std::string name = dirEntry.name; - name.copy(dirent->name, name.size() ); - dirent->name[name.size()] = '\0'; - mount->vfs->Attr2Stat(&dirEntry.attr, &dirent->stat); - return 1; + return nread; } else if (rc == CURVEFS_ERROR::END_OF_FILE) { return 0; } return SysErr(rc); } -int curvefs_closedir(uintptr_t instance_ptr, dir_stream_t* dir_stream) { - DirStream* stream = reinterpret_cast(dir_stream); +int curvefs_closedir(uintptr_t instance_ptr, uint64_t fd) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->CloseDir(fd); + return SysErr(rc); +} + +int curvefs_create(uintptr_t instance_ptr, + const char* path, + uint16_t mode, + file_t* file) { + CURVEFS_ERROR rc; auto mount = get_instance(instance_ptr); - auto rc = mount->vfs->CloseDir(stream); + rc = mount->vfs->Create(path, mode, reinterpret_cast(file)); return SysErr(rc); } int curvefs_open(uintptr_t instance_ptr, const char* path, uint32_t flags, - uint16_t mode) { + file_t* file) { CURVEFS_ERROR rc; auto mount = get_instance(instance_ptr); - if (flags & O_CREAT) { - rc = mount->vfs->Create(path, mode); - if (rc != CURVEFS_ERROR::OK) { - return SysErr(rc); - } - } - - uint64_t fd = 0; - rc = mount->vfs->Open(path, flags, mode, &fd); - if (rc != CURVEFS_ERROR::OK) { - return SysErr(rc); - } - return static_cast(fd); + rc = mount->vfs->Open(path, flags, reinterpret_cast(file)); + return SysErr(rc); } int curvefs_lseek(uintptr_t instance_ptr, @@ -254,3 +240,15 @@ int curvefs_rename(uintptr_t instance_ptr, auto rc = mount->vfs->Rename(oldpath, newpath); return SysErr(rc); } + +int curvefs_remove(uintptr_t instance_ptr, const char* path) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->Remove(path); + return SysErr(rc); +} + +int curvefs_removeall(uintptr_t instance_ptr, const char* path) { + auto mount = get_instance(instance_ptr); + auto rc = mount->vfs->RemoveAll(path); + return SysErr(rc); +} diff --git a/curvefs/sdk/libcurvefs/libcurvefs.h b/curvefs/sdk/libcurvefs/libcurvefs.h index 35d2f743df..f78621aee7 100644 --- a/curvefs/sdk/libcurvefs/libcurvefs.h +++ b/curvefs/sdk/libcurvefs/libcurvefs.h @@ -46,25 +46,23 @@ typedef struct { #endif // __cplusplus -// Must be synchronized with DirStream if changed typedef struct { - uint64_t ino; - uint64_t fh; - uint64_t offset; -} dir_stream_t; + uint64_t fd; + uint64_t length; +} file_t; typedef struct { - struct stat stat; - char name[256]; + char name[256]; // TODO(Wine93): smaller buffer + struct stat stat; // sizeof(stat) = 144 } dirent_t; #ifdef __cplusplus extern "C" { #endif -uintptr_t curvefs_create(); +uintptr_t curvefs_new(); -void curvefs_release(uintptr_t instance_ptr); +void curvefs_delete(uintptr_t instance_ptr); // NOTE: instance_ptr is the pointer of curvefs_mount_t instance. void curvefs_conf_set(uintptr_t instance_ptr, @@ -88,19 +86,25 @@ int curvefs_rmdir(uintptr_t instance_ptr, const char* path); int curvefs_opendir(uintptr_t instance_ptr, const char* path, - dir_stream_t* dir_stream); + uint64_t* fd); ssize_t curvefs_readdir(uintptr_t instance_ptr, - dir_stream_t* dir_stream, - dirent_t* dirent); + uint64_t fd, + dirent_t dirents[], + size_t count); -int curvefs_closedir(uintptr_t instance_ptr, dir_stream_t* dir_stream); +int curvefs_closedir(uintptr_t instance_ptr, uint64_t fd); // file +int curvefs_create(uintptr_t instance_ptr, + const char* path, + uint16_t mode, + file_t* file); + int curvefs_open(uintptr_t instance_ptr, const char* path, uint32_t flags, - uint16_t mode); + file_t* file); int curvefs_lseek(uintptr_t instance_ptr, int fd, @@ -142,6 +146,10 @@ int curvefs_chown(uintptr_t instance_ptr, uint32_t uid, uint32_t gid); +int curvefs_remove(uintptr_t instance_ptr, const char* path); + +int curvefs_removeall(uintptr_t instance_ptr, const char* path); + int curvefs_rename(uintptr_t instance_ptr, const char* oldpath, const char* newpath); diff --git a/curvefs/src/client/dentry_cache_manager.cpp b/curvefs/src/client/dentry_cache_manager.cpp index afb5e49eef..3b91a350ad 100644 --- a/curvefs/src/client/dentry_cache_manager.cpp +++ b/curvefs/src/client/dentry_cache_manager.cpp @@ -63,10 +63,12 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent, return CURVEFS_ERROR::OK; } -CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) { +CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry, + const InodeParam& param, + Inode* inode) { std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name()); NameLockGuard lock(nameLock_, key); - MetaStatusCode ret = metaClient_->CreateDentry(dentry); + MetaStatusCode ret = metaClient_->CreateDentry(dentry, param, inode); if (ret != MetaStatusCode::OK) { LOG(ERROR) << "metaClient_ CreateDentry failed, MetaStatusCode = " << ret diff --git a/curvefs/src/client/dentry_cache_manager.h b/curvefs/src/client/dentry_cache_manager.h index 84f9f20f53..a81ff3b7d2 100644 --- a/curvefs/src/client/dentry_cache_manager.h +++ b/curvefs/src/client/dentry_cache_manager.h @@ -33,6 +33,7 @@ #include #include "curvefs/src/client/rpcclient/metaserver_client.h" +#include "curvefs/src/client/rpcclient/base_client.h" #include "src/common/concurrent/concurrent.h" #include "src/common/concurrent/name_lock.h" #include "curvefs/src/client/filesystem/error.h" @@ -44,6 +45,7 @@ namespace client { using rpcclient::MetaServerClient; using rpcclient::MetaServerClientImpl; +using rpcclient::InodeParam; using ::curvefs::client::filesystem::CURVEFS_ERROR; static const char* kDentryKeyDelimiter = ":"; @@ -60,7 +62,9 @@ class DentryCacheManager { virtual CURVEFS_ERROR GetDentry(uint64_t parent, const std::string &name, Dentry *out) = 0; - virtual CURVEFS_ERROR CreateDentry(const Dentry &dentry) = 0; + virtual CURVEFS_ERROR CreateDentry(const Dentry &dentry, + const InodeParam& param, + Inode* inode) = 0; virtual CURVEFS_ERROR DeleteDentry(uint64_t parent, const std::string &name, @@ -86,7 +90,9 @@ class DentryCacheManagerImpl : public DentryCacheManager { CURVEFS_ERROR GetDentry(uint64_t parent, const std::string &name, Dentry *out) override; - CURVEFS_ERROR CreateDentry(const Dentry &dentry) override; + CURVEFS_ERROR CreateDentry(const Dentry &dentry, + const InodeParam& param, + Inode* inode) override; CURVEFS_ERROR DeleteDentry(uint64_t parent, const std::string &name, diff --git a/curvefs/src/client/filesystem/dir_cache.cpp b/curvefs/src/client/filesystem/dir_cache.cpp index f71a1651b8..b535d926e4 100644 --- a/curvefs/src/client/filesystem/dir_cache.cpp +++ b/curvefs/src/client/filesystem/dir_cache.cpp @@ -57,15 +57,6 @@ bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) { return true; } -bool DirEntryList::At(uint32_t index, DirEntry* dirEntry) { - ReadLockGuard lk(rwlock_); - if (index >= entries_.size()) { - return false; - } - *dirEntry = entries_[index]; - return true; -} - bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) { WriteLockGuard lk(rwlock_); auto iter = index_.find(ino); @@ -104,6 +95,17 @@ void DirEntryList::Iterate(IterateHandler handler) { } } +void DirEntryList::IterateRange(uint64_t offset, + uint64_t count, + IterateHandler handler) { + ReadLockGuard lk(rwlock_); + auto iter = entries_.begin() + offset; + while (count > 0 && iter != entries_.end()) { + handler(&(*iter++)); + count--; + } +} + void DirEntryList::Clear() { WriteLockGuard lk(rwlock_); entries_.clear(); diff --git a/curvefs/src/client/filesystem/dir_cache.h b/curvefs/src/client/filesystem/dir_cache.h index 6c07ae1221..2be4da8c8b 100644 --- a/curvefs/src/client/filesystem/dir_cache.h +++ b/curvefs/src/client/filesystem/dir_cache.h @@ -59,9 +59,9 @@ class DirEntryList { void Iterate(IterateHandler handler); - bool Get(Ino ino, DirEntry* dirEntry); + void IterateRange(uint64_t offset, uint64_t count, IterateHandler handler); - bool At(uint32_t index, DirEntry* dirEntry); + bool Get(Ino ino, DirEntry* dirEntry); bool UpdateAttr(Ino ino, const InodeAttr& attr); diff --git a/curvefs/src/client/fuse_client.cpp b/curvefs/src/client/fuse_client.cpp index 6f1c0b4771..f4c572fd76 100644 --- a/curvefs/src/client/fuse_client.cpp +++ b/curvefs/src/client/fuse_client.cpp @@ -49,6 +49,7 @@ #include "src/client/client_common.h" #include "src/common/dummyserver.h" #include "src/common/net_common.h" +#include "curvefs/proto/metaserver.pb.h" #define PORT_LIMIT 65535 @@ -438,6 +439,7 @@ CURVEFS_ERROR FuseClient::MakeNode( param.rdev = rdev; param.parent = parent; + /* CURVEFS_ERROR ret = inodeManager_->CreateInode(param, inodeWrapper); if (ret != CURVEFS_ERROR::OK) { LOG(ERROR) << "inodeManager CreateInode fail, ret = " << ret @@ -450,17 +452,20 @@ CURVEFS_ERROR FuseClient::MakeNode( << ", parent = " << parent << ", name = " << name << ", mode = " << mode << ", inode id = " << inodeWrapper->GetInodeId(); + */ Dentry dentry; dentry.set_fsid(fsInfo_->fsid()); - dentry.set_inodeid(inodeWrapper->GetInodeId()); + dentry.set_inodeid(0); dentry.set_parentinodeid(parent); dentry.set_name(name); - dentry.set_type(inodeWrapper->GetType()); + dentry.set_type(type); if (type == FsFileType::TYPE_FILE || type == FsFileType::TYPE_S3) { dentry.set_flag(DentryFlag::TYPE_FILE_FLAG); } - ret = dentryManager_->CreateDentry(dentry); + + Inode inode; + CURVEFS_ERROR ret = dentryManager_->CreateDentry(dentry, param, &inode); if (ret != CURVEFS_ERROR::OK) { LOG(ERROR) << "dentryManager_ CreateDentry fail, ret = " << ret << ", parent = " << parent << ", name = " << name @@ -475,10 +480,14 @@ CURVEFS_ERROR FuseClient::MakeNode( return ret; } + inodeManager_->ConvertInode(inode, inodeWrapper); + + VLOG(6) << "dentryManager_ CreateDentry success" << ", parent = " << parent << ", name = " << name << ", mode = " << mode; + /* if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; @@ -497,6 +506,7 @@ CURVEFS_ERROR FuseClient::MakeNode( << ", xattr = " << xattr.DebugString(); } } + */ return ret; } @@ -513,9 +523,12 @@ CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req, CURVEFS_ERROR rc = MakeNode(req, parent, name, S_IFDIR | mode, FsFileType::TYPE_DIRECTORY, 0, internal, inode); if (rc != CURVEFS_ERROR::OK) { + LOG(ERROR) << "<<<< " << rc; return rc; } + LOG(ERROR) << "<<<< CURVEFS_ERROR::OK"; + inode->GetInodeAttr(&entryOut->attr); return CURVEFS_ERROR::OK; } @@ -558,6 +571,7 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent, << ", parent = " << parent << ", name = " << name; } + /* if (enableSumInDir_.load()) { // update parent summary info XAttr xattr; @@ -576,6 +590,7 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent, << ", xattr = " << xattr.DebugString(); } } + */ return ret; } @@ -630,7 +645,8 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, if (type == FsFileType::TYPE_FILE || type == FsFileType::TYPE_S3) { dentry.set_flag(DentryFlag::TYPE_FILE_FLAG); } - ret = dentryManager_->CreateDentry(dentry); + Inode inode; + ret = dentryManager_->CreateDentry(dentry, param, &inode); if (ret != CURVEFS_ERROR::OK) { LOG(ERROR) << "dentryManager_ CreateDentry fail, ret = " << ret << ", parent = " << parent << ", name = " << name @@ -1226,7 +1242,8 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, dentry.set_parentinodeid(parent); dentry.set_name(name); dentry.set_type(inodeWrapper->GetType()); - ret = dentryManager_->CreateDentry(dentry); + Inode inode; + ret = dentryManager_->CreateDentry(dentry, param, &inode); if (ret != CURVEFS_ERROR::OK) { LOG(ERROR) << "dentryManager_ CreateDentry fail, ret = " << ret << ", parent = " << parent << ", name = " << name @@ -1289,7 +1306,8 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, dentry.set_parentinodeid(newparent); dentry.set_name(newname); dentry.set_type(inodeWrapper->GetType()); - ret = dentryManager_->CreateDentry(dentry); + Inode inode; + ret = dentryManager_->CreateDentry(dentry, InodeParam(), &inode); if (ret != CURVEFS_ERROR::OK) { LOG(ERROR) << "dentryManager_ CreateDentry fail, ret = " << ret << ", parent = " << newparent << ", name = " << newname; diff --git a/curvefs/src/client/inode_cache_manager.cpp b/curvefs/src/client/inode_cache_manager.cpp index b30dd4ee16..4ffc3d99fa 100644 --- a/curvefs/src/client/inode_cache_manager.cpp +++ b/curvefs/src/client/inode_cache_manager.cpp @@ -289,6 +289,14 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateInode( return CURVEFS_ERROR::OK; } +CURVEFS_ERROR InodeCacheManagerImpl::ConvertInode(Inode inode, + std::shared_ptr &out) { + out = std::make_shared(inode, metaClient_, + s3ChunkInfoMetric_, option_.maxDataSize, + option_.refreshDataIntervalSec); + return CURVEFS_ERROR::OK; +} + CURVEFS_ERROR InodeCacheManagerImpl::CreateManageInode( const InodeParam ¶m, std::shared_ptr &out) { diff --git a/curvefs/src/client/inode_cache_manager.h b/curvefs/src/client/inode_cache_manager.h index 49c642c0e3..84bb84b91a 100644 --- a/curvefs/src/client/inode_cache_manager.h +++ b/curvefs/src/client/inode_cache_manager.h @@ -106,6 +106,9 @@ class InodeCacheManager { virtual CURVEFS_ERROR CreateInode(const InodeParam ¶m, std::shared_ptr &out) = 0; // NOLINT + virtual CURVEFS_ERROR ConvertInode(Inode inode, + std::shared_ptr &out) = 0; // NOLINT + virtual CURVEFS_ERROR CreateManageInode(const InodeParam ¶m, std::shared_ptr &out) = 0; // NOLINT @@ -178,6 +181,9 @@ class InodeCacheManagerImpl : public InodeCacheManager, CURVEFS_ERROR CreateManageInode(const InodeParam ¶m, std::shared_ptr &out) override; + CURVEFS_ERROR ConvertInode(Inode inode, + std::shared_ptr &out) override; + CURVEFS_ERROR DeleteInode(uint64_t inodeId) override; void ShipToFlush( diff --git a/curvefs/src/client/logger/access_log.cpp b/curvefs/src/client/logger/access_log.cpp index 28c9255b9f..740c0e812c 100644 --- a/curvefs/src/client/logger/access_log.cpp +++ b/curvefs/src/client/logger/access_log.cpp @@ -34,8 +34,13 @@ bool InitAccessLog(const std::string& prefix) { return true; } - std::string filename = StrFormat("%s/access.%d.log", prefix, getpid()); - Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0); + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + std::string name = StrFormat("access-%d-%d", now.tv_sec, now.tv_nsec); + + std::string filename = StrFormat("%s/access.%d.%d.log", + prefix, getpid(), now.tv_sec); + Logger = spdlog::daily_logger_mt(name, filename, 0, 0); spdlog::flush_every(std::chrono::seconds(1)); inited = true; return true; diff --git a/curvefs/src/client/rpcclient/metaserver_client.cpp b/curvefs/src/client/rpcclient/metaserver_client.cpp index febde15dc2..bf49fa413e 100644 --- a/curvefs/src/client/rpcclient/metaserver_client.cpp +++ b/curvefs/src/client/rpcclient/metaserver_client.cpp @@ -237,9 +237,40 @@ MetaStatusCode MetaServerClientImpl::ListDentry(uint32_t fsId, uint64_t inodeid, return ConvertToMetaStatusCode(excutor.DoRPCTask()); } -MetaStatusCode MetaServerClientImpl::CreateDentry(const Dentry &dentry) { +namespace { + CreateInodeRequest NewCreateInodeRequest(const InodeParam& param, + uint32_t poolId, + uint32_t copysetId, + uint32_t partitionId) { + CreateInodeRequest request; + + request.set_poolid(poolId); + request.set_copysetid(copysetId); + request.set_partitionid(partitionId); + request.set_fsid(param.fsId); + request.set_length(param.length); + request.set_uid(param.uid); + request.set_gid(param.gid); + request.set_mode(param.mode); + request.set_type(param.type); + request.set_rdev(param.rdev); + request.set_symlink(param.symlink); + request.set_parent(param.parent); + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + Time *tm = new Time(); + tm->set_sec(now.tv_sec); + tm->set_nsec(now.tv_nsec); + request.set_allocated_create(tm); + return request; + } +}; + +MetaStatusCode MetaServerClientImpl::CreateDentry(const Dentry &dentry, + const InodeParam& param, + Inode* inode) { auto task = RPCTask { - (void)taskExecutorDone; + (void) taskExecutorDone; metric_.createDentry.qps.count << 1; LatencyUpdater updater(&metric_.createDentry.latency); CreateDentryResponse response; @@ -261,6 +292,12 @@ MetaStatusCode MetaServerClientImpl::CreateDentry(const Dentry &dentry) { tm->set_sec(now.tv_sec); tm->set_nsec(now.tv_nsec); request.set_allocated_create(tm); + //auto ignore = NewCreateInodeRequest(param); + + CreateInodeRequest* req = new CreateInodeRequest(); + *req = NewCreateInodeRequest(param, poolID, copysetID, partitionID); + request.set_allocated_createinoderequest(req); + curvefs::metaserver::MetaServerService_Stub stub(channel); stub.CreateDentry(cntl, &request, &response, nullptr); @@ -290,6 +327,8 @@ MetaStatusCode MetaServerClientImpl::CreateDentry(const Dentry &dentry) { << (ret == MetaStatusCode::OK ? "success" : "failure") << ", request: " << request.DebugString() << "response: " << response.DebugString(); + + *inode = response.inode(); return ret; }; diff --git a/curvefs/src/client/rpcclient/metaserver_client.h b/curvefs/src/client/rpcclient/metaserver_client.h index cd2a233da3..636d56dfe0 100644 --- a/curvefs/src/client/rpcclient/metaserver_client.h +++ b/curvefs/src/client/rpcclient/metaserver_client.h @@ -92,7 +92,9 @@ class MetaServerClient { bool onlyDir, std::list *dentryList) = 0; - virtual MetaStatusCode CreateDentry(const Dentry &dentry) = 0; + virtual MetaStatusCode CreateDentry(const Dentry &dentry, + const InodeParam& param, + Inode* inode) = 0; virtual MetaStatusCode DeleteDentry(uint32_t fsId, uint64_t inodeid, const std::string &name, @@ -200,7 +202,9 @@ class MetaServerClientImpl : public MetaServerClient { bool onlyDir, std::list *dentryList) override; - MetaStatusCode CreateDentry(const Dentry &dentry) override; + MetaStatusCode CreateDentry(const Dentry &dentry, + const InodeParam& param, + Inode* inode) override; MetaStatusCode DeleteDentry(uint32_t fsId, uint64_t inodeid, const std::string &name, diff --git a/curvefs/src/client/rpcclient/task_excutor.cpp b/curvefs/src/client/rpcclient/task_excutor.cpp index 191cff6e08..0f89003f89 100644 --- a/curvefs/src/client/rpcclient/task_excutor.cpp +++ b/curvefs/src/client/rpcclient/task_excutor.cpp @@ -72,11 +72,13 @@ int TaskExecutor::DoRPCTaskInner(TaskExecutorDone *done) { break; } + /* if (task_->refreshTxId && !metaCache_->RefreshTxId()) { LOG(ERROR) << "Refresh partition txid failed."; bthread_usleep(opt_.retryIntervalUS); continue; } + */ if (!HasValidTarget() && !GetTarget()) { LOG(WARNING) << "get target fail for " << task_->TaskContextStr() diff --git a/curvefs/src/client/s3/client_s3_adaptor.h b/curvefs/src/client/s3/client_s3_adaptor.h index dcc25c3795..af917a147a 100644 --- a/curvefs/src/client/s3/client_s3_adaptor.h +++ b/curvefs/src/client/s3/client_s3_adaptor.h @@ -116,7 +116,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor { public: S3ClientAdaptorImpl() {} virtual ~S3ClientAdaptorImpl() { - LOG(INFO) << "delete S3ClientAdaptorImpl"; + VLOG(3) << "delete S3ClientAdaptorImpl"; } /** * @brief Initailize s3 client diff --git a/curvefs/src/client/s3/client_s3_cache_manager.cpp b/curvefs/src/client/s3/client_s3_cache_manager.cpp index 1bd098eb71..5345827a85 100644 --- a/curvefs/src/client/s3/client_s3_cache_manager.cpp +++ b/curvefs/src/client/s3/client_s3_cache_manager.cpp @@ -2636,7 +2636,7 @@ void FsCacheManager::ReadCacheReleaseExecutor::Stop() { FsCacheManager::ReadCacheReleaseExecutor::~ReadCacheReleaseExecutor() { Stop(); t_.join(); - LOG(INFO) << "ReadCacheReleaseExecutor stopped"; + VLOG(3) << "ReadCacheReleaseExecutor stopped"; } void FsCacheManager::ReadCacheReleaseExecutor::Release( diff --git a/curvefs/src/client/vfs/handlers.cpp b/curvefs/src/client/vfs/handlers.cpp index 2bd4d2a7ac..b53995cda9 100644 --- a/curvefs/src/client/vfs/handlers.cpp +++ b/curvefs/src/client/vfs/handlers.cpp @@ -30,9 +30,10 @@ FileHandlers::FileHandlers() : nextHandler_(0), handlers_() {} -uint64_t FileHandlers::NextHandler(Ino ino, uint64_t offset) { + +uint64_t FileHandlers::NextHandler(const FileHandler& fh) { LockGuard lk(mutex_); - handlers_[nextHandler_] = std::make_shared(ino, offset); + handlers_[nextHandler_] = std::make_shared(fh); return nextHandler_++; } diff --git a/curvefs/src/client/vfs/handlers.h b/curvefs/src/client/vfs/handlers.h index 0e7af208d8..305bab177f 100644 --- a/curvefs/src/client/vfs/handlers.h +++ b/curvefs/src/client/vfs/handlers.h @@ -38,19 +38,48 @@ namespace vfs { using ::curve::common::Mutex; using ::curve::common::LockGuard; +enum HandlerType : uint8_t { + DIR = 0, + FILE = 1, +}; + struct FileHandler { - FileHandler() : ino(0), offset(0) {} - FileHandler(Ino ino, uint64_t offset): ino(ino), offset(offset) {} + // for file + FileHandler(HandlerType type, Ino ino, uint64_t offset, + uint64_t length, uint32_t flags) + : type(type), + ino(ino), + offset(offset), + length(length), + flags(flags), + fh(0), + entries(nullptr) {} + // for directory + FileHandler(HandlerType type, Ino ino) + : type(type), + ino(ino), + offset(0), + length(0), + flags(0), + fh(0), + entries(nullptr) {} + + HandlerType type; Ino ino; uint64_t offset; + uint64_t length; + uint32_t flags; + + uint64_t fh; + std::shared_ptr entries; }; class FileHandlers { public: FileHandlers(); - uint64_t NextHandler(Ino ino, uint64_t offset); + uint64_t NextHandler(const FileHandler& fh); bool GetHandler(uint64_t fd, std::shared_ptr* handler); diff --git a/curvefs/src/client/vfs/meta.h b/curvefs/src/client/vfs/meta.h index 789e093bbd..6d08b44d5b 100644 --- a/curvefs/src/client/vfs/meta.h +++ b/curvefs/src/client/vfs/meta.h @@ -45,10 +45,14 @@ using Ino = uint64_t; const Ino ROOT_INO = ::curvefs::ROOTINODEID; -struct DirStream { - Ino ino; - uint64_t fh; - uint64_t offset; +struct File { + uint64_t fd; + uint64_t length; +}; + +struct Dirent { + char name[256]; // TODO(Wine93): smaller buffer + struct stat stat; // sizeof(stat) = 144 }; struct Entry { diff --git a/curvefs/src/client/vfs/operations.cpp b/curvefs/src/client/vfs/operations.cpp index c466c17e74..e26bed77cb 100644 --- a/curvefs/src/client/vfs/operations.cpp +++ b/curvefs/src/client/vfs/operations.cpp @@ -114,13 +114,12 @@ CURVEFS_ERROR OperationsImpl::Create(Ino parent, mode, fi, entryOut); } -CURVEFS_ERROR OperationsImpl::Open(Ino ino, uint32_t flags) { - FileOut fileOut; +CURVEFS_ERROR OperationsImpl::Open(Ino ino, uint32_t flags, FileOut* fileOut) { auto ctx = NewFuseContext(); auto req = ctx->GetRequest(); auto fi = ctx->GetFileInfo(); fi->flags = flags; - return client_->FuseOpOpen(req, ino, fi, &fileOut); + return client_->FuseOpOpen(req, ino, fi, fileOut); } CURVEFS_ERROR OperationsImpl::Read(Ino ino, @@ -138,19 +137,12 @@ CURVEFS_ERROR OperationsImpl::Write(Ino ino, uint64_t offset, const char* buffer, size_t size, - size_t* nwritten) { - FileOut fileOut; + FileOut* fileOut) { auto ctx = NewFuseContext(); auto req = ctx->GetRequest(); auto fi = ctx->GetFileInfo(); - auto rc = client_->FuseOpWrite(req, ino, buffer, size, - offset, fi, &fileOut); - if (rc == CURVEFS_ERROR::OK) { - *nwritten = fileOut.nwritten; - } else { - *nwritten = 0; - } - return rc; + return client_->FuseOpWrite(req, ino, buffer, size, + offset, fi, fileOut); } CURVEFS_ERROR OperationsImpl::Flush(Ino ino) { diff --git a/curvefs/src/client/vfs/operations.h b/curvefs/src/client/vfs/operations.h index 73f2a38f88..2e6bdf9e66 100644 --- a/curvefs/src/client/vfs/operations.h +++ b/curvefs/src/client/vfs/operations.h @@ -75,7 +75,7 @@ class Operations { uint16_t mode, EntryOut* entryOut) = 0; - virtual CURVEFS_ERROR Open(Ino ino, uint32_t flags) = 0; + virtual CURVEFS_ERROR Open(Ino ino, uint32_t flags, FileOut* fileOut) = 0; virtual CURVEFS_ERROR Read(Ino ino, uint64_t offset, @@ -87,7 +87,7 @@ class Operations { uint64_t offset, const char* buffer, size_t size, - size_t* nwritten) = 0; + FileOut* fileOut) = 0; virtual CURVEFS_ERROR Flush(Ino ino) = 0; @@ -152,7 +152,7 @@ class OperationsImpl : public Operations { uint16_t mode, EntryOut* entryOut) override; - CURVEFS_ERROR Open(Ino ino, uint32_t flags) override; + CURVEFS_ERROR Open(Ino ino, uint32_t flags, FileOut* fileOut) override; CURVEFS_ERROR Read(Ino ino, uint64_t offset, @@ -164,7 +164,7 @@ class OperationsImpl : public Operations { uint64_t offset, const char* buffer, size_t size, - size_t* nwritten) override; + FileOut* fileOut) override; CURVEFS_ERROR Flush(Ino ino) override; diff --git a/curvefs/src/client/vfs/vfs.cpp b/curvefs/src/client/vfs/vfs.cpp index 620163dd89..c719e19f28 100644 --- a/curvefs/src/client/vfs/vfs.cpp +++ b/curvefs/src/client/vfs/vfs.cpp @@ -37,6 +37,7 @@ #include "curvefs/src/client/vfs/config.h" #include "curvefs/src/client/vfs/meta.h" #include "curvefs/src/client/vfs/utils.h" +#include "curvefs/src/client/vfs/handlers.h" #include "curvefs/src/client/vfs/vfs.h" namespace curvefs { @@ -92,7 +93,7 @@ CURVEFS_ERROR VFS::Umount(const std::string& fsname, const std::string& mountpoint) { CURVEFS_ERROR rc; AccessLogGuard log([&]() { - return StrFormat("umount: %s", StrErr(rc)); + return StrFormat("umount (%s,%s): %s", fsname, mountpoint, StrErr(rc)); }); rc = op_->Umount(fsname, mountpoint); @@ -185,17 +186,19 @@ CURVEFS_ERROR VFS::RmDir(const std::string& path) { return rc; } -CURVEFS_ERROR VFS::OpenDir(const std::string& path, DirStream* stream) { +CURVEFS_ERROR VFS::OpenDir(const std::string& path, uint64_t* fd) { Entry entry; CURVEFS_ERROR rc; AccessLogGuard log([&](){ return StrFormat("opendir (%s): %s [fh:%d]", - path, StrErr(rc), stream->fh); + path, StrErr(rc), *fd); }); rc = Lookup(path, true, &entry); if (rc != CURVEFS_ERROR::OK) { return rc; + } else if (!IsDir(entry.attr)) { + return CURVEFS_ERROR::NOT_A_DIRECTORY; } rc = permission_->Check(entry.attr, Permission::WANT_READ); @@ -203,57 +206,96 @@ CURVEFS_ERROR VFS::OpenDir(const std::string& path, DirStream* stream) { return rc; } - stream->ino = entry.ino; - stream->offset = 0; - rc = op_->OpenDir(entry.ino, &stream->fh); + FileHandler fh(HandlerType::DIR, entry.ino); + rc = op_->OpenDir(entry.ino, &fh.fh); + if (rc == CURVEFS_ERROR::OK) { + *fd = handlers_->NextHandler(fh); + } return rc; } -CURVEFS_ERROR VFS::ReadDir(DirStream* stream, DirEntry* dirEntry) { - uint64_t nread = 0; +CURVEFS_ERROR VFS::ReadDir(uint64_t fd, + Dirent dirents[], + size_t count, + uint64_t* nread) { + uint64_t offset = 0; CURVEFS_ERROR rc; AccessLogGuard log([&](){ - return StrFormat("readdir (%d,%d): %s (%d)", - stream->fh, stream->offset, StrErr(rc), nread); + return StrFormat("readdir (%d,%d,%d): %s (%d)", + fd, count, offset, StrErr(rc), *nread); }); - auto entries = std::make_shared(); - rc = op_->ReadDir(stream->ino, stream->fh, &entries); - if (rc != CURVEFS_ERROR::OK) { + std::shared_ptr fh; + bool yes = handlers_->GetHandler(fd, &fh); + if (!yes) { + rc = CURVEFS_ERROR::BAD_FD; return rc; } - if (stream->offset >= entries->Size()) { + if (nullptr == fh->entries) { + fh->entries = std::make_shared(); + rc = op_->ReadDir(fh->ino, fh->fh, &fh->entries); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + fh->entries->Iterate([&](DirEntry* dirEntry) { + // TODO(Wine93): add entry&attr cache + }); + } + + offset = fh->offset; + size_t size = fh->entries->Size(); + if (offset >= size) { rc = CURVEFS_ERROR::END_OF_FILE; return rc; } + uint64_t i = 0; + fh->entries->IterateRange(offset, count, [&](DirEntry* dirEntry) { + const std::string& name = dirEntry->name; + name.copy(dirents[i].name, name.size()); + dirents[i].name[name.size()] = '\0'; + op_->Attr2Stat(&dirEntry->attr, &dirents[i].stat); + i++; + }); + + *nread = i; + fh->offset = fh->offset + *nread; rc = CURVEFS_ERROR::OK; - entries->At(stream->offset, dirEntry); - stream->offset++; - nread = 1; return rc; } -CURVEFS_ERROR VFS::CloseDir(DirStream* stream) { +CURVEFS_ERROR VFS::CloseDir(uint64_t fd) { CURVEFS_ERROR rc; AccessLogGuard log([&](){ - return StrFormat("closedir (%d): %s", - stream->fh, StrErr(rc)); + return StrFormat("closedir (%d): %s", fd, StrErr(rc)); }); - rc = op_->CloseDir(stream->ino); + std::shared_ptr fh; + bool yes = handlers_->GetHandler(fd, &fh); + if (!yes) { + rc = CURVEFS_ERROR::BAD_FD; + return rc; + } + + rc = op_->CloseDir(fh->ino); + if (rc == CURVEFS_ERROR::OK) { + handlers_->FreeHandler(fd); + } return rc; } -CURVEFS_ERROR VFS::Create(const std::string& path, uint16_t mode) { +CURVEFS_ERROR VFS::Create(const std::string& path, + uint16_t mode, + File* file) { Entry parent; EntryOut entryOut; CURVEFS_ERROR rc; AccessLogGuard log([&](){ - return StrFormat("create (%s,%s:0%04o): %s%s", + return StrFormat("create (%s,%s:0%04o): %s%s [fh:%d]", path, StrMode(mode), mode, - StrErr(rc), StrEntry(entryOut)); + StrErr(rc), StrEntry(entryOut), file->fd); }); rc = Lookup(filepath::ParentDir(path), true, &parent); @@ -268,20 +310,25 @@ CURVEFS_ERROR VFS::Create(const std::string& path, uint16_t mode) { mode = permission_->GetFileMode(S_IFREG, mode); rc = op_->Create(parent.ino, filepath::Filename(path), mode, &entryOut); - if (rc == CURVEFS_ERROR::OK) { - PurgeEntryCache(parent.ino, filepath::Filename(path)); + if (rc != CURVEFS_ERROR::OK) { + return rc; } + + PurgeEntryCache(parent.ino, filepath::Filename(path)); + FileHandler fh = + FileHandler(HandlerType::FILE, entryOut.attr.inodeid(), 0, 0, O_WRONLY); + file->fd = handlers_->NextHandler(fh); + file->length = 0; return rc; } CURVEFS_ERROR VFS::Open(const std::string& path, uint32_t flags, - uint16_t mode, - uint64_t* fd) { + File* file) { Entry entry; CURVEFS_ERROR rc; AccessLogGuard log([&](){ - return StrFormat("open (%s): %s [fh:%d]", path, StrErr(rc), *fd); + return StrFormat("open (%s): %s [fh:%d]", path, StrErr(rc), file->fd); }); rc = Lookup(path, true, &entry); @@ -295,16 +342,29 @@ CURVEFS_ERROR VFS::Open(const std::string& path, return rc; } - rc = op_->Open(entry.ino, flags); - if (rc != CURVEFS_ERROR::OK) { - return rc; + FileOut fileOut; + for (int retries = 0; retries < 2; retries++) { + rc = op_->Open(entry.ino, flags, &fileOut); + if (rc == CURVEFS_ERROR::OK) { + break; + } else if (rc == CURVEFS_ERROR::STALE) { + PurgeAttrCache(entry.ino); + continue; + } else { // Encourage error + return rc; + } } uint64_t offset = 0; + uint64_t length = fileOut.attr.length(); if (flags & O_APPEND) { - offset = entry.attr.length(); + offset = length; } - *fd = handlers_->NextHandler(entry.ino, offset); + + auto fh = FileHandler(HandlerType::FILE, entry.ino, + offset, length, flags); + file->fd = handlers_->NextHandler(fh); + file->length = length; return CURVEFS_ERROR::OK; } @@ -323,11 +383,6 @@ CURVEFS_ERROR VFS::LSeek(uint64_t fd, uint64_t offset, int whence) { return rc; } - rc = op_->GetAttr(fh->ino, &attrOut); - if (rc != CURVEFS_ERROR::OK) { - return rc; - } - switch (whence) { case SEEK_SET: fh->offset = offset; @@ -338,11 +393,7 @@ CURVEFS_ERROR VFS::LSeek(uint64_t fd, uint64_t offset, int whence) { break; case SEEK_END: - rc = op_->GetAttr(fh->ino, &attrOut); - if (rc != CURVEFS_ERROR::OK) { - return rc; - } - fh->offset = attrOut.attr.length() + offset; + fh->offset = fh->length + offset; break; default: @@ -358,9 +409,8 @@ CURVEFS_ERROR VFS::Read(uint64_t fd, char* buffer, size_t count, size_t* nread) { - AttrOut attrOut; - std::shared_ptr fh; CURVEFS_ERROR rc; + std::shared_ptr fh; uint64_t offset = 0; AccessLogGuard log([&](){ return StrFormat("read (%d,%d,%d): %s (%d)", @@ -373,15 +423,10 @@ CURVEFS_ERROR VFS::Read(uint64_t fd, return rc; } - rc = op_->GetAttr(fh->ino, &attrOut); - if (rc != CURVEFS_ERROR::OK) { - return rc; - } - offset = fh->offset; rc = op_->Read(fh->ino, fh->offset, buffer, count, nread); if (rc == CURVEFS_ERROR::OK) { - fh->offset += *nread; + fh->offset += *nread; // FIXME(Wine93): need lock here? } return rc; } @@ -390,9 +435,8 @@ CURVEFS_ERROR VFS::Write(uint64_t fd, const char* buffer, size_t count, size_t* nwritten) { - std::shared_ptr fh; - AttrOut attrOut; CURVEFS_ERROR rc; + std::shared_ptr fh; uint64_t offset = 0; AccessLogGuard log([&](){ return StrFormat("write (%d,%d,%d): %s (%d)", @@ -405,24 +449,21 @@ CURVEFS_ERROR VFS::Write(uint64_t fd, return rc; } - rc = op_->GetAttr(fh->ino, &attrOut); - if (rc != CURVEFS_ERROR::OK) { - return rc; - } - + FileOut fileOut; offset = fh->offset; - rc = op_->Write(fh->ino, fh->offset, buffer, count, nwritten); + rc = op_->Write(fh->ino, fh->offset, buffer, count, &fileOut); if (rc == CURVEFS_ERROR::OK) { - fh->offset += *nwritten; + *nwritten = fileOut.nwritten; + fh->offset += *nwritten; // FIXME(Wine93): need lock here? + fh->length = fileOut.attr.length(); PurgeAttrCache(fh->ino); } return rc; } CURVEFS_ERROR VFS::FSync(uint64_t fd) { - std::shared_ptr fh; - AttrOut attrOut; CURVEFS_ERROR rc; + std::shared_ptr fh; AccessLogGuard log([&](){ return StrFormat("fsync (%d): %s", fd, StrErr(rc)); }); @@ -433,18 +474,13 @@ CURVEFS_ERROR VFS::FSync(uint64_t fd) { return rc; } - rc = op_->GetAttr(fh->ino, &attrOut); - if (rc != CURVEFS_ERROR::OK) { - return rc; - } - rc = op_->Flush(fh->ino); return rc; } CURVEFS_ERROR VFS::Close(uint64_t fd) { - std::shared_ptr fh; CURVEFS_ERROR rc; + std::shared_ptr fh; AccessLogGuard log([&](){ return StrFormat("close (%d): %s", fd, StrErr(rc)); }); @@ -455,9 +491,12 @@ CURVEFS_ERROR VFS::Close(uint64_t fd) { return rc; } - rc = op_->Flush(fh->ino); - if (rc != CURVEFS_ERROR::OK) { - return rc; + uint32_t flags = fh->flags; + if (flags & O_WRONLY || flags & O_RDWR) { + rc = op_->Flush(fh->ino); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } } rc = op_->Close(fh->ino); @@ -641,6 +680,51 @@ CURVEFS_ERROR VFS::Chown(const std::string &path, uint32_t uid, uint32_t gid) { return rc; } +CURVEFS_ERROR VFS::Remove(const std::string& path) { + CURVEFS_ERROR rc; + AccessLogGuard log([&](){ + return StrFormat("remove (%s): %s", path, StrErr(rc)); + }); + + Entry parent; + rc = Lookup(filepath::ParentDir(path), true, &parent); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + rc = permission_->Check(parent.attr, Permission::WANT_WRITE); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + Entry entry; + rc = Lookup(path, true, &entry); + if (rc != CURVEFS_ERROR::OK) { + return rc; + } + + if (IsDir(entry.attr)) { + rc = op_->RmDir(parent.ino, filepath::Filename(path)); + } else { + rc = op_->Unlink(parent.ino, filepath::Filename(path)); + } + + if (rc == CURVEFS_ERROR::OK) { + PurgeEntryCache(parent.ino, filepath::Filename(path)); + } + return rc; +} + +CURVEFS_ERROR VFS::RemoveAll(const std::string& path) { + CURVEFS_ERROR rc; + AccessLogGuard log([&](){ + return StrFormat("removeall (%s): %s", path, StrErr(rc)); + }); + + rc = CURVEFS_ERROR::NOT_SUPPORT; + return rc; +} + CURVEFS_ERROR VFS::Rename(const std::string& oldpath, const std::string& newpath) { Entry oldParent, newParent; diff --git a/curvefs/src/client/vfs/vfs.h b/curvefs/src/client/vfs/vfs.h index 0af8bbb304..dc63bcdfde 100644 --- a/curvefs/src/client/vfs/vfs.h +++ b/curvefs/src/client/vfs/vfs.h @@ -63,19 +63,21 @@ class VFS { CURVEFS_ERROR RmDir(const std::string& path); - CURVEFS_ERROR OpenDir(const std::string& path, DirStream* stream); + CURVEFS_ERROR OpenDir(const std::string& path, uint64_t* fd); - CURVEFS_ERROR ReadDir(DirStream* stream, DirEntry* dirEntry); + CURVEFS_ERROR ReadDir(uint64_t fd, + Dirent dirents[], + size_t count, + uint64_t* nread); - CURVEFS_ERROR CloseDir(DirStream* stream); + CURVEFS_ERROR CloseDir(uint64_t fd); // file* - CURVEFS_ERROR Create(const std::string& path, uint16_t mode); + CURVEFS_ERROR Create(const std::string& path, uint16_t mode, File* file); CURVEFS_ERROR Open(const std::string& path, uint32_t flags, - uint16_t mode, - uint64_t* fd); + File* file); CURVEFS_ERROR LSeek(uint64_t fd, uint64_t offset, int whence); @@ -107,6 +109,10 @@ class VFS { CURVEFS_ERROR Chown(const std::string& path, uint32_t uid, uint32_t gid); + CURVEFS_ERROR Remove(const std::string& path); + + CURVEFS_ERROR RemoveAll(const std::string& path); + CURVEFS_ERROR Rename(const std::string& oldpath, const std::string& newpath); diff --git a/curvefs/src/metaserver/metastore.cpp b/curvefs/src/metaserver/metastore.cpp index 875f71cffc..a463f4a7f6 100644 --- a/curvefs/src/metaserver/metastore.cpp +++ b/curvefs/src/metaserver/metastore.cpp @@ -357,12 +357,29 @@ std::shared_ptr MetaStoreImpl::GetStreamServer() { response->set_statuscode(status); \ return status; \ } - - // dentry MetaStatusCode MetaStoreImpl::CreateDentry(const CreateDentryRequest* request, CreateDentryResponse* response, int64_t logIndex) { + // create inode + uint64_t inodeid; + Inode* inode = response->mutable_inode(); + { + CreateInodeRequest req = request->createinoderequest(); + CreateInodeResponse resp; + if (request->has_createinoderequest()) { + auto status = CreateInode(&req, &resp, logIndex); + if (status != MetaStatusCode::OK) { + response->set_statuscode(status); + return status; + } + } + + *inode = resp.inode(); + inodeid = resp.inode().inodeid(); + } + + // create dentry ReadLockGuard readLockGuard(rwLock_); std::shared_ptr partition; GET_PARTITION_OR_RETURN(partition); @@ -375,9 +392,17 @@ MetaStatusCode MetaStoreImpl::CreateDentry(const CreateDentryRequest* request, Time tm; tm.set_sec(now); tm.set_nsec(now_ns); + + Dentry dentry = request->dentry(); + dentry.set_inodeid(inodeid); + MetaStatusCode status = - partition->CreateDentry(request->dentry(), tm, logIndex); + partition->CreateDentry(dentry, tm, logIndex); response->set_statuscode(status); + + LOG(ERROR) << "<<<< inode = " << response->inode().ShortDebugString(); + + return status; } diff --git a/src/common/concurrent/rw_lock.h b/src/common/concurrent/rw_lock.h index 807afb3b8c..6d10ec39dd 100644 --- a/src/common/concurrent/rw_lock.h +++ b/src/common/concurrent/rw_lock.h @@ -70,7 +70,7 @@ class PthreadRWLockBase : public RWLockBase { int ret = pthread_rwlock_wrlock(&rwlock_); CHECK(0 == ret) << "wlock failed: " << ret << ", " << strerror(ret); #if CURVE_CHECK_PTHREAD_WRLOCK_TID - tid_ = gettid(); + //tid_ = gettid(); #endif } @@ -81,7 +81,7 @@ class PthreadRWLockBase : public RWLockBase { } #if CURVE_CHECK_PTHREAD_WRLOCK_TID - tid_ = gettid(); + //tid_ = gettid(); #endif return 0; } @@ -97,17 +97,17 @@ class PthreadRWLockBase : public RWLockBase { void Unlock() override { #if CURVE_CHECK_PTHREAD_WRLOCK_TID - if (tid_ != 0) { - const pid_t current = gettid(); - // If CHECK here is triggered, please look at the comments at the - // beginning of the file. - // In the meantime, the simplest solution might be to use - // `BthreadRWLock` locks everywhere. - CHECK(tid_ == current) - << ", tid has changed, previous tid: " << tid_ - << ", current tid: " << current; - tid_ = 0; - } + //if (tid_ != 0) { + // const pid_t current = gettid(); + // // If CHECK here is triggered, please look at the comments at the + // // beginning of the file. + // // In the meantime, the simplest solution might be to use + // // `BthreadRWLock` locks everywhere. + // CHECK(tid_ == current) + // << ", tid has changed, previous tid: " << tid_ + // << ", current tid: " << current; + // tid_ = 0; + //} #endif pthread_rwlock_unlock(&rwlock_); } diff --git a/src/common/s3_adapter.cpp b/src/common/s3_adapter.cpp index d2226147c9..de817abfa6 100644 --- a/src/common/s3_adapter.cpp +++ b/src/common/s3_adapter.cpp @@ -206,9 +206,9 @@ void S3Adapter::Deinit() { void S3Adapter::Shutdown() { // one program should only call once auto shutdownSDK = [&]() { - Aws::ShutdownAPI(AWS_SDK_OPTIONS); + // Aws::ShutdownAPI(AWS_SDK_OPTIONS); }; - std::call_once(S3SHUTDOWN_FLAG, shutdownSDK); + shutdownSDK(); } void S3Adapter::Reinit(const S3AdapterOption& option) { diff --git a/util/build.sh b/util/build.sh index 6a66d0f5f3..ccd339ad1c 100644 --- a/util/build.sh +++ b/util/build.sh @@ -26,7 +26,7 @@ source "$(dirname "${BASH_SOURCE}")/docker_opts.sh" main() { source "util/basic.sh" - source "util/build_functions.sh" +source "util/build_functions.sh" get_options "$@" get_version diff --git a/util/build_in_image.sh b/util/build_in_image.sh index 7ec37f83b3..0bac36f8c8 100644 --- a/util/build_in_image.sh +++ b/util/build_in_image.sh @@ -21,7 +21,7 @@ g_build_opts=( "--copt -DUSE_BTHREAD_MUTEX" "--copt -DCLIENT_CONF_PATH=\"${g_root}/curvefs/conf/client.conf\"" ) -# allow user to specify extra build options +# allow user to specify extra build options # using environment variable BUILD_OPTS , if any. # custom build options will be appended to g_build_opts if [ -n "$BUILD_OPTS" ]; then @@ -49,6 +49,7 @@ main() { else if [ "$g_depend" -eq 1 ]; then build_requirements + (cd curvefs/sdk/java && mvn package) fi if [ -n "$g_target" ]; then build_target diff --git a/util/clean.sh b/util/clean.sh new file mode 100644 index 0000000000..070c8cc335 --- /dev/null +++ b/util/clean.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + + +rm -rf thirdparties/rocksdb/lib/ +rm -rf thirdparties/rocksdb/include/ +rm -rf thirdparties/rocksdb/rocksdb/ +rm -rf thirdparties/rocksdb/*log +rm -rf thirdparties/rocksdb/*.tar.gz +rm -rf thirdparties/aws/*.tar.gz +rm -rf thirdparties/etcdclient/tmp/ +rm -rf thirdparties/etcdclient/*.h +rm -rf thirdparties/memcache/*.tar.gz +rm -rf thirdparties/memcache/libmemcached-*/ +rm -rf curvefs/sdk/java/target +rm -rf curvefs/sdk/output +rm -rf ~/.cache +rm -rf curvefs/sdk/java/dependency-reduced-pom.xml +rm -rf bazel-* +rm -rf curvefs/sdk/java/src/main/java/io/opencurve/curve/fs/libfs/*.class +rm -rf curvefs/sdk/java/native/build diff --git a/util/init-hadoop.sh b/util/init-hadoop.sh index 3b7fc82714..ad2514bf56 100644 --- a/util/init-hadoop.sh +++ b/util/init-hadoop.sh @@ -64,6 +64,11 @@ cat << EOF > "${g_hadoop_etc}" 10.0.0.1:6700,10.0.0.2:6700,10.0.0.3:6700 + +curvefs.client.common.logDir +/tmp + + curvefs.fs.accessLogging true diff --git a/util/sdk.sh b/util/sdk.sh index c9bb862845..03cc0c4c3d 100644 --- a/util/sdk.sh +++ b/util/sdk.sh @@ -8,13 +8,15 @@ build_jni_library() { # 1) generate jni native header file local maven_repo="/home/${USER}/.m2/repository" cd "${g_root}/curvefs/sdk/java/src/main/java" - javac -cp "${maven_repo}/org/apache/commons/commons-compress/1.24.0/commons-compress-1.24.0.jar":. \ + javac -cp "${maven_repo}/org/apache/commons/commons-compress/1.24.0/commons-compress-1.24.0.jar":"${maven_repo}/org/apache/hadoop/hadoop-common/2.7.3/hadoop-common-2.7.3.jar":. \ -h \ "${g_root}/curvefs/sdk/java/native/" \ - io/opencurve/curve/fs/libfs/CurveFSMount.java + io/opencurve/curve/fs/libfs/CurveFsMount.java # 2) build libcurvefs_jni.so cd "${g_root}" + + # TODO: --config=gcc7-later bazel build \ --config=gcc7-later \ --compilation_mode=opt \