From d57ff5da539b26905617a0c228737246fdebb84e Mon Sep 17 00:00:00 2001 From: sekaiacg Date: Fri, 24 Feb 2023 13:09:24 +0800 Subject: [PATCH] extract: Add extract.erofs usage: [options] -h, --help Display this help and exit -i, --image=[FILE] Image file -p Print all entrys --print=X Print the target of path X -x Extract all items --extract=X Extract the target of path X -f, --overwrite [default: skip] overwrite files that already exist -T# [1-X] Use # threads, -T0: X/2 --only-cfg Only extract fs_config and file_contexts -o, --outdir=X Output dir -V, --version Print the version info Signed-off-by: sekaiacg --- README.md | 34 ++ build.sh | 5 +- build/cmake/erofs-tools/CMakeLists.txt | 1 + build/cmake/erofs-tools/extract.cmake | 12 + extract/ErofsNode.cpp | 148 +++++++ extract/ExtractHelper.cpp | 555 +++++++++++++++++++++++++ extract/ExtractOperation.cpp | 249 +++++++++++ extract/include/ErofsNode.h | 85 ++++ extract/include/ExtractHelper.h | 108 +++++ extract/include/ExtractOperation.h | 141 +++++++ extract/include/ExtractState.h | 20 + extract/include/Logging.h | 82 ++++ extract/include/Utils.h | 67 +++ extract/include/threadpool.h | 158 +++++++ extract/main.cpp | 248 +++++++++++ 15 files changed, 1911 insertions(+), 2 deletions(-) create mode 100644 README.md create mode 100644 build/cmake/erofs-tools/extract.cmake create mode 100644 extract/ErofsNode.cpp create mode 100644 extract/ExtractHelper.cpp create mode 100644 extract/ExtractOperation.cpp create mode 100644 extract/include/ErofsNode.h create mode 100644 extract/include/ExtractHelper.h create mode 100644 extract/include/ExtractOperation.h create mode 100644 extract/include/ExtractState.h create mode 100644 extract/include/Logging.h create mode 100644 extract/include/Utils.h create mode 100644 extract/include/threadpool.h create mode 100644 extract/main.cpp diff --git a/README.md b/README.md new file mode 100644 index 00000000..24066b39 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +**extract.erofs** +=========== +**使用[erofs-utils](https://github.com/hsiangkao/erofs-utils)实现的提取erofs镜像的工具** +**提取包含：** +- fs_config +- files_context + +**A tool for extracting erofs images implemented using [erofs-utils](https://github.com/hsiangkao/erofs-utils)** +**Extract contains:** +- fs_config +- files_context + +**fs_config:**`vendor/bin/cnd 1000 1000 0755 capabilities=0x1000001400` + +**files_context:** `/vendor/bin/hw/android\.hardware\.bluetooth@1\.0-service-qti u:object_r:hal_bluetooth_default_exec:s0` + +``` +usage: [options] + -h, --help Display this help and exit + -i, --image=[FILE] Image file + -p Print all entrys + --print=X Print the target of path X + -x Extract all items + --extract=X Extract the target of path X + -f, --overwrite [default: skip] overwrite files that already exist + -T# [1-24] Use # threads, -T0: 12 + --only-cfg Only extract fs_config and file_contexts + -o, --outdir=X Output dir + -V, --version Print the version info +``` + +**Contributors** +- 感谢[lateautumn233](https://github.com/lateautumn233)提供的[erofs-utils](https://github.com/lateautumn233/erofs-utils)编译方法 +- Thanks to [lateautumn233](https://github.com/lateautumn233) for the [erofs-utils](https://github.com/lateautumn233/erofs-utils) compilation method. diff --git a/build.sh b/build.sh index cc0c3ce6..8ee80350 100755 --- a/build.sh +++ b/build.sh @@ -60,17 +60,18 @@ build() local FSCK_BIN="$BUILD/fsck.erofs" local FUSE_BIN="$BUILD/fuse.erofs" local MKFS_BIN="$BUILD/mkfs.erofs" + local EXTRACT_BIN="$BUILD/extract.erofs" local TARGE_DIR_NAME="erofs-utils-${EROFS_VERSION}-${TARGET}_${ABI}-$(TZ=UTC-8 date +%y%m%d%H%M)" local TARGET_DIR_PATH="./target/${TARGET}_${ABI}/${TARGE_DIR_NAME}" - if [ -f "$DUMP_BIN" -a -f "$FSCK_BIN" -a -f "$FUSE_BIN" -a -f "$MKFS_BIN" ]; then + if [ -f "$DUMP_BIN" -a -f "$FSCK_BIN" -a -f "$FUSE_BIN" -a -f "$MKFS_BIN" -a -f "$EXTRACT_BIN" ]; then echo "复制文件中..." [[ ! -d "$TARGET_DIR_PATH" ]] && mkdir -p ${TARGET_DIR_PATH} cp -af $BUILD/*.erofs ${TARGET_DIR_PATH} echo "编译成功: ${TARGE_DIR_NAME}" else echo "error" - exit -1 + exit 1 fi } diff --git a/build/cmake/erofs-tools/CMakeLists.txt b/build/cmake/erofs-tools/CMakeLists.txt index 05378075..2585d1c9 100644 --- a/build/cmake/erofs-tools/CMakeLists.txt +++ b/build/cmake/erofs-tools/CMakeLists.txt @@ -1 +1,2 @@ include(erofs_tools.cmake) +include(extract.cmake) diff --git a/build/cmake/erofs-tools/extract.cmake b/build/cmake/erofs-tools/extract.cmake new file mode 100644 index 00000000..98b7370d --- /dev/null +++ b/build/cmake/erofs-tools/extract.cmake @@ -0,0 +1,12 @@ +###############################------extract.erofs------############################### +set(TARGET_extract extract.erofs) +set(TARGET_SRC_DIR "${PROJECT_ROOT_DIR}/extract") +file(GLOB extract_srcs "${TARGET_SRC_DIR}/*.cpp") +add_executable(${TARGET_extract} ${extract_srcs}) +target_include_directories(${TARGET_extract} PRIVATE + "${TARGET_SRC_DIR}/include" + ${common_headers} + ${liblog_headers} +) +target_link_libraries(${TARGET_extract} ${common_static_link_lib}) +target_compile_options(${TARGET_extract} PRIVATE ${common_compile_flags}) diff --git a/extract/ErofsNode.cpp b/extract/ErofsNode.cpp new file mode 100644 index 00000000..5bcb05bb --- /dev/null +++ b/extract/ErofsNode.cpp @@ -0,0 +1,148 @@ +#include "ErofsNode.h" +#include "ExtractHelper.h" +#include "ExtractState.h" + +#define FS_CONFIG_BUF_SIZE (PATH_MAX + 256) + +namespace skkk { + + ErofsNode::ErofsNode(const char *path, short typeId, struct erofs_inode *inode) { + this->path = path; + this->typeId = typeId; + this->nid = inode->nid; + this->i_mode = inode->i_mode; + this->i_uid = inode->i_uid; + this->i_gid = inode->i_gid; + this->i_mtime = inode->i_mtime; + this->i_mtime_nsec = inode->i_mtime_nsec; + this->dataLayout = inode->datalayout; + char buf[FS_CONFIG_BUF_SIZE + 1]; + snprintf(buf, FS_CONFIG_BUF_SIZE, "%s %u %u %04o", + path, + inode->i_uid, + inode->i_gid, + inode->i_mode & 0777 + ); + this->fsConfig = buf; + this->inode = new erofs_inode; + this->inode->nid = inode->nid; + erofs_read_inode_from_disk(this->inode); + } + + ErofsNode::~ErofsNode() { delete inode; } + + const string &ErofsNode::getPath() const { return path; } + + short ErofsNode::getTypeId() const { return typeId; } + + erofs_inode *ErofsNode::getErofsInode() const { return inode; } + + const char *ErofsNode::getTypeIdCStr() const { + switch (typeId) { + case EROFS_FT_DIR: + return "DIR"; + case EROFS_FT_REG_FILE: + return "FILE"; + case EROFS_FT_SYMLINK: + return "LINK"; + case EROFS_FT_CHRDEV: + return "CHR"; + case EROFS_FT_BLKDEV: + return "BLK"; + case EROFS_FT_FIFO: + return "FIFO"; + case EROFS_FT_SOCK: + return "SOCK"; + } + return "UNKNOWN"; + } + + const char *ErofsNode::getDataLayoutStr() const { + switch (dataLayout) { + case EROFS_INODE_FLAT_PLAIN: + return "PLAIN"; + case EROFS_INODE_FLAT_INLINE: + return "INLINE"; + case EROFS_INODE_CHUNK_BASED: + return "CHUNK"; + case EROFS_INODE_FLAT_COMPRESSION_LEGACY: + return "COMPRESSION_LEGACY"; + case EROFS_INODE_FLAT_COMPRESSION: + return "COMPRESSION"; + } + return "UNKNOWN"; + } + + const string &ErofsNode::getFsConfig() const { return fsConfig; } + + const string &ErofsNode::getSeLabel() const { return seContext; } + + void ErofsNode::setSeContext(const string &_seContext) { this->seContext = _seContext; } + +#ifdef WITH_ANDROID + + uint64_t ErofsNode::getCapability() const { return capabilities; } + + void ErofsNode::setCapability(uint64_t _capabilities) { this->capabilities = _capabilities; } + + void ErofsNode::setFsConfigCapabilities(const char *capabilitiesStr) { fsConfig.append(capabilitiesStr); } + +#endif + + bool ErofsNode::initExceptionInfo(int err) { + if (err && err != RET_EXTRACT_FAIL_SKIP) [[unlikely]] { + char buf[256] = {0}; + snprintf(buf, 256, "err=%d type=%s dataLayout=%s name=%s", + err, + getTypeIdCStr(), + getDataLayoutStr(), + getPath().c_str() + ); + extractExceptionInfo = buf; + return true; + } + return false; + } + + void + ErofsNode::writeFsConfigAndSeContext2File(FILE *fsConfigFile, FILE *seContextFile, const char *imgBaseName) const { + if (path == "/") [[unlikely]] { + fprintf(fsConfigFile, "%s\n", fsConfig.c_str()); + fprintf(seContextFile, "/%s %s\n", imgBaseName, seContext.c_str()); + } + fprintf(fsConfigFile, "%s%s\n", imgBaseName, fsConfig.c_str()); + const string seCFilePath = handleSpecialSymbols(path); + fprintf(seContextFile, "/%s%s %s\n", imgBaseName, seCFilePath.c_str(), seContext.c_str()); + } + + int ErofsNode::writeNodeEntity2File(const string &outDir) { + int err = RET_EXTRACT_DONE; + string _tmp = outDir + path; + const char *filePath = _tmp.c_str(); + switch (this->typeId) { + case EROFS_FT_DIR: + err = erofs_extract_dir(filePath); + break; + case EROFS_FT_REG_FILE: + err = erofs_extract_file(inode, filePath); + break; + case EROFS_FT_SYMLINK: + err = erofs_extract_symlink(inode, filePath); + break; + case EROFS_FT_CHRDEV: + case EROFS_FT_BLKDEV: + case EROFS_FT_FIFO: + case EROFS_FT_SOCK: + err = erofs_extract_special(inode, filePath); + break; + } + if (!err) set_attributes(inode, filePath); + return err; + } + + void ErofsNode::writeExceptionInfo2FileIfExists(FILE *infoFile) const { + if (!extractExceptionInfo.empty()) [[unlikely]] { + fprintf(infoFile, "%s\n", extractExceptionInfo.c_str()); + } + } +} \ No newline at end of file diff --git a/extract/ExtractHelper.cpp b/extract/ExtractHelper.cpp new file mode 100644 index 00000000..e1c80889 --- /dev/null +++ b/extract/ExtractHelper.cpp @@ -0,0 +1,555 @@ +#include +#include +#include +#include + +#include "ExtractHelper.h" +#include "ErofsNode.h" +#include "ExtractState.h" +#include "ExtractOperation.h" +#include "Logging.h" + +#ifdef WITH_ANDROID + +#include + +#endif + +namespace skkk { + + static int doInitNode(erofs_nid_t nid); + + /** + * Copy from fsck.erofs + * Modified + * + * @param ctx + * @return + */ + static int dirent_iter(struct erofs_dir_context *ctx) { + int ret; + size_t prev_pos = eo->iter_pos; + + if (ctx->dot_dotdot) + return 0; + + if (eo->iter_path) { + size_t curr_pos = prev_pos; + + eo->iter_path[curr_pos++] = '/'; + strncpy(eo->iter_path + curr_pos, ctx->dname, + ctx->de_namelen); + curr_pos += ctx->de_namelen; + eo->iter_path[curr_pos] = '\0'; + eo->iter_pos = curr_pos; + } + + ret = doInitNode(ctx->de_nid); + + if (eo->iter_path) { + eo->iter_path[prev_pos] = '\0'; + eo->iter_pos = prev_pos; + } + return ret; + } + + /** + * Copy from fsck.erofs + * Modified + * + * @param pnid + * @param nid + * @return + */ + static int doInitNode(erofs_nid_t nid) { + int ret; + + struct erofs_inode inode = {}; + short typeId = EROFS_FT_UNKNOWN; + + inode.nid = nid; + ret = erofs_read_inode_from_disk(&inode); + if (ret) { + goto out; + } + switch (inode.i_mode & S_IFMT) { + case S_IFDIR: + typeId = EROFS_FT_DIR; + break; + case S_IFREG: + if (erofs_is_packed_inode(&inode)) [[unlikely]] + break; + typeId = EROFS_FT_REG_FILE; + break; + case S_IFLNK: + typeId = EROFS_FT_SYMLINK; + break; + case S_IFCHR: + typeId = EROFS_FT_CHRDEV; + break; + case S_IFBLK: + typeId = EROFS_FT_BLKDEV; + break; + case S_IFIFO: + typeId = EROFS_FT_FIFO; + break; + case S_IFSOCK: + typeId = EROFS_FT_SOCK; + break; + default: + break; + } + + { +#ifdef NDEBUG + ExtractOperation::createErofsNode(eo->iter_path, typeId, &inode); +#else + + const ErofsNode *eNode = ExtractOperation::createErofsNode(eo->iter_path, typeId, &inode); + LOGCD("type=%s dataLayout=%s %s", + eNode->getTypeIdCStr(), + eNode->getDataLayoutStr(), + eNode->getFsConfig().c_str() + ); +#endif + } + + { + struct erofs_dir_context ctx = { + .dir = &inode, + .cb = dirent_iter + }; + if (S_ISDIR(inode.i_mode)) { + ret = erofs_iterate_dir(&ctx, false); + } + } +out: + return ret; + } + + /** + * Copy from fsck.erofs + * Modified + * + * @param inode + * @param outfd + * @return + */ + static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd) { + struct erofs_map_blocks map = { + .index = UINT_MAX, + }; + int ret = 0; + bool compressed; + erofs_off_t pos = 0; + unsigned int raw_size = 0, buffer_size = 0; + char *raw = nullptr, *buffer = nullptr; + + switch (inode->datalayout) { + case EROFS_INODE_FLAT_PLAIN: + case EROFS_INODE_FLAT_INLINE: + case EROFS_INODE_CHUNK_BASED: + compressed = false; + break; + case EROFS_INODE_FLAT_COMPRESSION_LEGACY: + case EROFS_INODE_FLAT_COMPRESSION: + compressed = true; + break; + default: + return -EINVAL; + } + + while (pos < inode->i_size) { + map.m_la = pos; + if (compressed) + ret = z_erofs_map_blocks_iter(inode, &map, + EROFS_GET_BLOCKS_FIEMAP); + else + ret = erofs_map_blocks(inode, &map, + EROFS_GET_BLOCKS_FIEMAP); + if (ret) + goto out; + + if (!compressed && map.m_llen != map.m_plen) { + ret = -EFSCORRUPTED; + goto out; + } + + /* the last lcluster can be divided into 3 parts */ + if (map.m_la + map.m_llen > inode->i_size) + map.m_llen = inode->i_size - map.m_la; + + pos += map.m_llen; + + /* should skip decomp? */ + if (!(map.m_flags & EROFS_MAP_MAPPED) || !eo->check_decomp) + continue; + + if (map.m_plen > raw_size) { + raw_size = map.m_plen; + raw = (char *) realloc(raw, raw_size); + BUG_ON(!raw); + } + + if (compressed) { + if (map.m_llen > buffer_size) { + buffer_size = map.m_llen; + buffer = (char *) realloc(buffer, buffer_size); + BUG_ON(!buffer); + } + ret = z_erofs_read_one_data(inode, &map, raw, buffer, + 0, map.m_llen, false); + } else { + ret = erofs_read_one_data(&map, raw, 0, map.m_plen); + } + if (ret) + goto out; + + if (outfd > 0 && write(outfd, compressed ? buffer : raw, + map.m_llen) < 0) { + ret = -EIO; + goto out; + } + } + +out: + if (raw) + free(raw); + if (buffer) + free(buffer); + return ret < 0 ? ret : 0; + } + + /** + * Copy from fsck.erofs + * Modified + * + * @param dirPath + * @return + */ + int erofs_extract_dir(const char *dirPath) { + + if (mkdirs(dirPath, 0700) < 0) { + struct stat st = {}; + + if (errno != EEXIST) { + return -errno; + } + + if (lstat(dirPath, &st) || + !S_ISDIR(st.st_mode)) { + return -ENOTDIR; + } + + if (chmod(dirPath, 0700) < 0) { + return -errno; + } + } + return 0; + } + + /** + * Copy from fsck.erofs + * Modified + * + * @param filePath + * @param inode + * @return + */ + int erofs_extract_file(struct erofs_inode *inode, const char *filePath) { + bool tryagain = true; + int ret, fd; + +again: + fd = open(filePath, + O_WRONLY | O_CREAT | O_NOFOLLOW | + (eo->overwrite ? O_TRUNC : O_EXCL), 0700); + if (fd < 0) { + if (eo->overwrite && tryagain) { + if (errno == EISDIR) { + if (rmdir(filePath) < 0) { + return -EISDIR; + } + } else if (errno == EACCES && + chmod(filePath, 0700) < 0) { + return -errno; + } + tryagain = false; + goto again; + } + if (errno == EEXIST && !eo->overwrite) { + return RET_EXTRACT_FAIL_SKIP; + } + return -errno; + } + + /* verify data chunk layout */ + ret = erofs_verify_inode_data(inode, fd); + if (ret) + return ret; + + if (close(fd)) + return -errno; + return ret; + + } + + /** + * Copy from fsck.erofs + * Modified by skkk + * + * @param filePath + * @param inode + * @return + */ + int erofs_extract_symlink(erofs_inode *inode, const char *filePath) { + bool tryagain = true; + int ret; + char *buf; + + buf = (char *) malloc(inode->i_size + 1); + if (!buf) { + ret = -ENOMEM; + goto out; + } + + ret = erofs_pread(inode, buf, inode->i_size, 0); + if (ret) { + goto out; + } + + buf[inode->i_size] = '\0'; +again: + if (symlink(buf, filePath) < 0) { + if (errno == EEXIST && eo->overwrite && tryagain) { + if (unlink(filePath) < 0) { + ret = -errno; + goto out; + } + tryagain = false; + goto again; + } + if (errno == EEXIST && !eo->overwrite) { + return RET_EXTRACT_FAIL_SKIP; + } + ret = -errno; + } +out: + if (buf) + free(buf); + return ret; + + } + + /** + * Copy from fsck.erofs + * Modified + * + * @param filePath + * @param inode + * @return + */ + int erofs_extract_special(erofs_inode *inode, const char *filePath) { + + bool tryagain = true; + int ret = 0; + +again: + if (mknod(filePath, inode->i_mode, inode->u.i_rdev) < 0) { + if (errno == EEXIST && eo->overwrite && tryagain) { + if (unlink(filePath) < 0) { + return -errno; + } + tryagain = false; + goto again; + } + if (errno == EEXIST || eo->superuser) { + ret = -errno; + } else { + ret = -ECANCELED; + } + } + return ret; + } + + /** + * Copy from fsck.erofs + * Modified + * + * @param inode + * @param path + */ + void set_attributes(struct erofs_inode *inode, const char *path) { + int ret; + +#ifdef HAVE_UTIMENSAT + if (utimensat(AT_FDCWD, path, (struct timespec[]) { + { + .tv_sec = (__time_t) inode->i_mtime, +#if defined(__LP64__) + .tv_nsec = inode->i_mtime_nsec +#else + .tv_nsec = (long int) inode->i_mtime_nsec +#endif + }, + { + .tv_sec = (__time_t) inode->i_mtime, +#if defined(__LP64__) + .tv_nsec = inode->i_mtime_nsec +#else + .tv_nsec = (long int) inode->i_mtime_nsec +#endif + }, + }, AT_SYMLINK_NOFOLLOW) < 0) +#else + struct utimbuf ub = { + .actime = (__time_t)inode->i_mtime, + .modtime = (__time_t)inode->i_mtime + }; + if (utime(path, &ub) <0 ) +#endif + LOGCW("failed to set times: %s", path); + if (!S_ISLNK(inode->i_mode)) { + if (eo->preserve_perms) + ret = chmod(path, inode->i_mode); + else + ret = chmod(path, inode->i_mode & ~eo->umask); + if (ret < 0) + LOGCW("failed to set permissions: %s", path); + } + + if (eo->preserve_owner) { + ret = lchown(path, inode->i_uid, inode->i_gid); + if (ret < 0) + LOGCW("failed to change ownership: %s", path); + } + } + + int writeErofsNode2File(ErofsNode *eNode, const string &outDir) { + int err = RET_EXTRACT_DONE; + string _tmp = outDir + eNode->getPath(); + const char *filePath = _tmp.c_str(); + erofs_inode *inode = eNode->getErofsInode(); + switch (eNode->getTypeId()) { + case EROFS_FT_DIR: + err = erofs_extract_dir(filePath); + break; + case EROFS_FT_REG_FILE: + err = erofs_extract_file(inode, filePath); + break; + case EROFS_FT_SYMLINK: + err = erofs_extract_symlink(inode, filePath); + break; + case EROFS_FT_CHRDEV: + case EROFS_FT_BLKDEV: + case EROFS_FT_FIFO: + case EROFS_FT_SOCK: + err = erofs_extract_special(inode, filePath); + break; + } + if (!err) set_attributes(inode, filePath); + return err; + } + + void initSecurityContext(ErofsNode *eNode, struct erofs_inode *inode) { + char buf[128] = {0}; + int len; + + // "security.selinux" + len = erofs_getxattr(inode, XATTR_NAME_SELINUX, buf, 128); + if (len > 0) { + eNode->setSeContext(string(buf, len)); + } + +#ifdef WITH_ANDROID + // security.capability + len = erofs_getxattr(inode, XATTR_NAME_CAPABILITY, buf, 128); + if (len > 0) { + uint64_t capabilities = 0; + auto *fileCapData = (struct vfs_cap_data *) buf; + uint32_t cap_version = le32_to_cpu(fileCapData->magic_etc) & VFS_CAP_REVISION_MASK; + // check version size + switch (cap_version) { + case VFS_CAP_REVISION_1: + if (len != XATTR_CAPS_SZ_1) + return; + capabilities |= le32_to_cpu(fileCapData->data[0].permitted); + break; + case VFS_CAP_REVISION_2: + if (len != XATTR_CAPS_SZ_2) + return; + capabilities = (uint64_t) le32_to_cpu(fileCapData->data[0].permitted) | + (((uint64_t) le32_to_cpu(fileCapData->data[1].permitted)) << 32); + break; + default: + return; + } + + if (capabilities) { + eNode->setCapability(capabilities); + char capBuf[32] = {0}; +#if defined(__LP64__) + snprintf(capBuf, 32, " capabilities=0x%lX", capabilities); +#else + snprintf(capBuf, 32, " capabilities=0x%llX", capabilities); +#endif + eNode->setFsConfigCapabilities(capBuf); + } + } +#endif + } + + int initErofsNodeByRoot() { + int rc = RET_EXTRACT_DONE, err; + eo->iter_path = (char *) malloc(PATH_MAX + 1); + memset(eo->iter_path, 0, PATH_MAX + 1); + + // root is '/' + eo->iter_path[0] = '/'; + eo->iter_pos = 0; + + err = doInitNode(sbi.root_nid); + if (err) { + rc = RET_EXTRACT_INIT_NODE_FAIL; + LOGCE("failed to initialize ErofsNode!"); + } + if (eo->iter_path) free(eo->iter_path); + eo->iter_pos = 0; + return rc; + } + + int initErofsNodeByTargetPath(const string &targetPath) { + char nameBuf[PATH_MAX + 1] = {0}; + int rc = RET_EXTRACT_INIT_NODE_FAIL, err; + if (targetPath.empty()) return RET_EXTRACT_INIT_NODE_FAIL; + + eo->iter_path = (char *) malloc(PATH_MAX); + memset(eo->iter_path, 0, PATH_MAX); + + // find targetPath + struct erofs_inode vi = {.nid = sbi.root_nid}; + err = erofs_ilookup(targetPath.c_str(), &vi); + if (err) { + LOGCE("path not found: '%s'", targetPath.c_str()); + goto exit; + } + + err = erofs_get_pathname(vi.nid, nameBuf, PATH_MAX); + if (err) { + goto exit; + } + eo->iter_pos = snprintf(eo->iter_path, PATH_MAX, nameBuf, nullptr); + + err = doInitNode(vi.nid); + if (err) { + LOGCE("failed to initialize ErofsNode, path: '%s'", targetPath.c_str()); + goto exit; + } + + rc = RET_EXTRACT_DONE; +exit: + if (eo->iter_path) free(eo->iter_path); + eo->iter_pos = 0; + return rc; + } +} diff --git a/extract/ExtractOperation.cpp b/extract/ExtractOperation.cpp new file mode 100644 index 00000000..aec98efa --- /dev/null +++ b/extract/ExtractOperation.cpp @@ -0,0 +1,249 @@ +#include + +#include "ExtractOperation.h" +#include "ExtractState.h" +#include "ExtractHelper.h" +#include "Logging.h" +#include "Utils.h" +#include "threadpool.h" + +namespace skkk { + + void ExtractOperation::setImgPath(const char *path) { + imgPath = path; + strTrim(imgPath); + LOGCD("config: imagePath=%s", imgPath.c_str()); + imgBaseName = path; + if (!imgPath.empty()) { + auto ps = imgPath.rfind('/'); + if (ps != string::npos) + imgBaseName = imgPath.substr(ps + 1, imgPath.size()); + ps = imgBaseName.find('.'); + if (ps != string::npos) imgBaseName.erase(ps, imgBaseName.size()); + LOGCD("config: imgBaseName=%s", imgBaseName.c_str()); + } + } + + void ExtractOperation::erofsOperationExit() { + for_each(erofsNodes.begin(), erofsNodes.end(), + [](auto *eNode) { delete eNode; }); + erofsNodes.clear(); + nodeDirs.clear(); + nodeOther.clear(); + } + + const string &ExtractOperation::getImgPath() const { return imgPath; } + + const string &ExtractOperation::getImgBaseName() const { return imgBaseName; } + + void ExtractOperation::setOutDir(const char *path) { outDir = path; } + + int ExtractOperation::initOutDir() { + int rc = RET_EXTRACT_DONE; + strTrim(outDir); + + if (outDir.empty()) { + configDir = "./config"; + outDir = "./" + imgBaseName; + } else { + const char *oDir = outDir.c_str(); + auto oSize = outDir.size(); + // check dir is root: "/","//","///",... + bool isRoot = true; + for (int i = 0; i < oSize; i++) { + isRoot = oDir[i] == '/'; + } + if (isRoot) { + LOGCE("Not allow extracting to root: '%s'", outDir.c_str()); + rc = RET_EXTRACT_OUTDIR_ROOT; + } + configDir = outDir + "/config"; + outDir = outDir + "/" + imgBaseName; + } + return rc; + } + + int ExtractOperation::createExtractOutDir() const { + int rc = RET_EXTRACT_DONE, err; + if (!dirExists(outDir)) { + err = mkdirs(outDir.c_str(), 0700); + if (err) { + rc = RET_EXTRACT_CREATE_DIR_FAIL; + LOGCE("create out dir fail: '%s'", outDir.c_str()); + } + } + return rc; + } + + int ExtractOperation::createExtractConfigDir() const { + int rc = RET_EXTRACT_DONE, err; + if (!dirExists(configDir)) { + err = mkdirs(configDir.c_str(), 0700); + if (err) { + rc = RET_EXTRACT_CREATE_DIR_FAIL; + LOGCE("create config dir fail: '%s'", configDir.c_str()); + } + } + return rc; + } + + const string &ExtractOperation::getOutDir() const { return outDir; } + + const string &ExtractOperation::getConfDir() const { return configDir; } + + int ExtractOperation::initAllErofsNode() const { return initErofsNodeByRoot(); } + + int ExtractOperation::initErofsNodeByTarget() const { return initErofsNodeByTargetPath(targetPath); } + + int ExtractOperation::initErofsNodeAuto() const { + return targetPath.empty() ? + initErofsNodeByRoot() : initErofsNodeByTargetPath(targetPath); + } + + const ErofsNode + *ExtractOperation::createErofsNode(const char *path, short typeId, struct erofs_inode *inode) { + auto *eNode = new ErofsNode{path, typeId, inode}; + erofsNodes.push_back(eNode); + initSecurityContext(eNode, inode); + return eNode; + } + + void ExtractOperation::initDirAndOther() { + for_each(erofsNodes.begin(), erofsNodes.end(), [](auto &eNode) { + if (eNode->getTypeId() == EROFS_FT_DIR) { + nodeDirs.push_back(eNode); + } else { + nodeOther.push_back(eNode); + } + }); + LOGCD("initDirAndOther done"); + } + + void ExtractOperation::addErofsNode(ErofsNode *eNode) { erofsNodes.push_back(eNode); } + + const vector &ExtractOperation::getErofsNodes() { return erofsNodes; } + + static inline void printFsConfWithColor(const ErofsNode *eNode) { + LOGCI("type=%s dataLayout=%s fsConfig=[%s] seLabel=[%s]", + eNode->getTypeIdCStr(), + eNode->getDataLayoutStr(), + eNode->getFsConfig().c_str(), + eNode->getSeLabel().c_str() + ); + } + + static inline void printFsConf(const ErofsNode *eNode) { + LOGI("type=%s dataLayout=%s fsConfig=[%s] seLabel=[%s]", + eNode->getTypeIdCStr(), + eNode->getDataLayoutStr(), + eNode->getFsConfig().c_str(), + eNode->getSeLabel().c_str() + ); + } + + void ExtractOperation::printInitializedNode() { + for_each(erofsNodes.begin(), erofsNodes.end(), printFsConf); + } + + void ExtractOperation::extractFsConfigAndSeLabel() const { + string fsConfigPath = configDir + "/" + imgBaseName + "_fs_config"; + string fsSeContextPath = configDir + "/" + imgBaseName + "_file_contexts"; + FILE *fsConfigFile = fopen(fsConfigPath.c_str(), "wb"); + FILE *seContextFile = fopen(fsSeContextPath.c_str(), "wb"); + const char *_imgBaseName = imgBaseName.c_str(); + LOGCI(BROWN "fs_config|file_contexts" LOG_RESET_COLOR " " GREEN2_BOLD "saving..." LOG_RESET_COLOR); + if (fsConfigFile && seContextFile) { + for_each(erofsNodes.begin(), erofsNodes.end(), + [&fsConfigFile, &seContextFile, &_imgBaseName](auto *eNode) { + eNode->writeFsConfigAndSeContext2File(fsConfigFile, seContextFile, _imgBaseName); + }); + LOGCI(BROWN "fs_config|files_context" LOG_RESET_COLOR " " GREEN2_BOLD "done." LOG_RESET_COLOR); + } else + LOGCE(BROWN "fs_config|files_context" LOG_RESET_COLOR " " RED2_BOLD "fail!" LOG_RESET_COLOR); + if (fsConfigFile) fclose(fsConfigFile); + if (seContextFile) fclose(seContextFile); + } + + void ExtractOperation::writeExceptionInfo2File() const { + if (exceptionSize > 0) { + FILE *infoFile = fopen((configDir + "/exception.log").c_str(), "w"); + for (const auto &eNode: erofsNodes) { + eNode->writeExceptionInfo2FileIfExists(infoFile); + } + LOGCE(RED2 "An exception occurred while fetching, the info has been saved!" COLOR_NONE); + } + } + + static inline void extractNodeTask(ErofsNode *eNode, const string &outdir) { + if (eNode->initExceptionInfo(eNode->writeNodeEntity2File(outdir))) + ExtractOperation::exceptionSize++; + } + + static inline void extractNodeTaskMultiThread(ErofsNode *eNode, const string &outdir) { + if (eNode->initExceptionInfo(eNode->writeNodeEntity2File(outdir))) + ExtractOperation::exceptionSize++; + ExtractOperation::extractTaskRunCount++; + } + + static inline void printExtractProgress(int totalSize, int index, int perPrint, bool hasEnter) { + if (index % perPrint == 0 || index == totalSize) { + float p = (float) index / (float) totalSize * 100.0f; + printf(BROWN2_BOLD "Extract: " COLOR_NONE + GREEN2_BOLD "[ " COLOR_NONE RED2 "%.2f%%" LOG_RESET_COLOR GREEN2_BOLD " ]" COLOR_NONE + "\r", + p + ); + fflush(stdout); + if (hasEnter && p == 100) [[unlikely]] { + printf("\n"); + } + } + } + + void ExtractOperation::extractNodeDirs() const { + ExtractOperation::initDirAndOther(); + if (!nodeDirs.empty()) { + for (const auto &eNode: nodeDirs) { + extractNodeTask(eNode, outDir); + } + } + } + + void ExtractOperation::extractErofsNode() const { + eo->extractNodeDirs(); + if (!nodeOther.empty()) { + int nodeOtherSize = nodeOther.size(); + for (int i = 0; i < nodeOtherSize; i++) { + extractNodeTask(nodeOther[i], outDir); + printExtractProgress(nodeOtherSize, i + 1, 4, true); + } + } + // If there is an exception + writeExceptionInfo2File(); + } + + void ExtractOperation::extractErofsNodeMultiThread() const { + eo->extractNodeDirs(); + LOGCI(GREEN2_BOLD "Use " COLOR_NONE RED2 "%d" COLOR_NONE GREEN2_BOLD " therads" COLOR_NONE, + eo->threadNum); + + int nodeOtherSize = nodeOther.size(); + threadpool tp(eo->threadNum); + for (const auto &eNode: nodeOther) { + tp.commit(extractNodeTaskMultiThread, eNode, outDir); + } + + int i = 0; + while (extractTaskRunCount < nodeOtherSize) { + if (i != extractTaskRunCount) { + printExtractProgress(nodeOtherSize, extractTaskRunCount, 2, false); + i = extractTaskRunCount; + } + sleep(0); + } + printExtractProgress(1, 1, 1, true); + LOGCD("extractTaskRunCount=%d nodeFilesSize=%d", i, nodeOtherSize, nodeOther.size()); + + writeExceptionInfo2File(); + } +} diff --git a/extract/include/ErofsNode.h b/extract/include/ErofsNode.h new file mode 100644 index 00000000..0c78e132 --- /dev/null +++ b/extract/include/ErofsNode.h @@ -0,0 +1,85 @@ +#ifndef EXTRACT_EROFS_NODE_H +#define EXTRACT_EROFS_NODE_H + +#include +#include + +#include "Utils.h" + +using namespace std; + +namespace skkk { + + static inline string handleSpecialSymbols(const string &str) { + string tmp = string(str); + strReplaceAll(tmp, ".", "\\."); + strReplaceAll(tmp, "+", "\\+"); + strReplaceAll(tmp, "[", "\\["); + strReplaceAll(tmp, "]", "\\]"); + return tmp; + } + + /** + * erofs node + */ + class ErofsNode { + private: + string path; + short typeId = EROFS_FT_UNKNOWN; + erofs_inode *inode = nullptr; + string fsConfig; + string seContext; + erofs_nid_t nid; + umode_t i_mode; + u32 i_uid; + u32 i_gid; + uint64_t i_mtime; + u32 i_mtime_nsec; + unsigned char dataLayout; +#ifdef WITH_ANDROID + uint64_t capabilities = 0; +#endif + string extractExceptionInfo; + + public: + ErofsNode(const char *path, short typeId, erofs_inode *inode); + + ~ErofsNode(); + + const string &getPath() const; + + short getTypeId() const; + + erofs_inode *getErofsInode() const; + + const char *getTypeIdCStr() const; + + const char *getDataLayoutStr() const; + + const string &getFsConfig() const; + + const string &getSeLabel() const; + + void setSeContext(const string &_seContext); + +#ifdef WITH_ANDROID + + uint64_t getCapability() const; + + void setCapability(uint64_t _capabilities); + + void setFsConfigCapabilities(const char *capabilitiesStr); + +#endif + + bool initExceptionInfo(int err); + + void writeFsConfigAndSeContext2File(FILE *fsConfigFile, FILE *seContextFile, const char *imgBaseName) const; + + int writeNodeEntity2File(const string &outDir); + + void writeExceptionInfo2FileIfExists(FILE *infoFile) const; + + }; +} +#endif //EXTRACT_EROFS_NODE_H diff --git a/extract/include/ExtractHelper.h b/extract/include/ExtractHelper.h new file mode 100644 index 00000000..0045f349 --- /dev/null +++ b/extract/include/ExtractHelper.h @@ -0,0 +1,108 @@ +#ifndef EXTRACT_HELPER_H +#define EXTRACT_HELPER_H + +#include +#include + +#include "ErofsNode.h" + +using namespace std; + +#ifndef XATTR_NAME_SELINUX +#define XATTR_NAME_SELINUX "security.selinux" +#endif + +#ifdef WITH_ANDROID +#ifndef XATTR_NAME_CAPABILITY +#define XATTR_NAME_CAPABILITY "security.capability" +#endif +#endif + +namespace skkk { + + /** + * erofs_extract_dir + * Copy from fsck.erofs + * + * @param dirPath + * @param inode + * @return + */ + int erofs_extract_dir(const char *dirPath); + + /** + * erofs_extract_file + * Copy from fsck.erofs + * + * @param filePath + * @param inode + * @return + */ + int erofs_extract_file(struct erofs_inode *inode, const char *filePath); + + /** + * erofs_extract_file + * Copy from fsck.erofs + * + * @param filePath + * @param inode + * @return + */ + int erofs_extract_symlink(struct erofs_inode *inode, const char *filePath); + + /** + * erofs_extract_special + * Copy from fsck.erofs + * + * @param filePath + * @param inode + * @return + */ + int erofs_extract_special(struct erofs_inode *inode, const char *filePath); + + /** + * set_attributes + * Copy from fsck.erofs + * + * @param inode + * @param path + * @return + */ + void set_attributes(struct erofs_inode *inode, const char *path); + + /** + * writeErofsNode2File + * + * @param eNode + * @param outDir + * @return + */ + int writeErofsNode2File(ErofsNode *eNode, const string &outDir); + + /** + * + * Initialize Security context + * + * @param eNode + * @param inode + */ + void initSecurityContext(ErofsNode * eNode, struct erofs_inode *inode); + + /** + * Initialize all nodes + * + * @return + */ + int initErofsNodeByRoot(); + + /** + * Initialize the specified node + * + * @param targetPath + * @return + */ + int initErofsNodeByTargetPath(const string &targetPath); + +} + +#endif //EXTRACT_HELPER_H diff --git a/extract/include/ExtractOperation.h b/extract/include/ExtractOperation.h new file mode 100644 index 00000000..9743712a --- /dev/null +++ b/extract/include/ExtractOperation.h @@ -0,0 +1,141 @@ +#ifndef EXTRACT_OPERATION_H +#define EXTRACT_OPERATION_H + +#include +#include +#include +#include +#include +#include +#include + +#include "ErofsNode.h" + +using namespace std; + +namespace skkk { + + /** + * extract operation and config + */ + class ExtractOperation { + private: + /* instance */ + static inline ExtractOperation *instance = nullptr; + /* erofs node list */ + static inline vector erofsNodes; + /* there are only dir type nodes */ + static inline vector nodeDirs; + /* there are only other type nodes */ + static inline vector nodeOther; + + string imgPath; + string imgBaseName; + string outDir; + string configDir; + + private: + ExtractOperation() = default; + + ~ExtractOperation() = default; + + ExtractOperation(ExtractOperation const &); + + ExtractOperation &operator=(ExtractOperation const &); + + public: + char *iter_path = nullptr; + size_t iter_pos = 0; + static inline atomic_int extractTaskRunCount = 0; + static inline atomic_int exceptionSize = 0; + mode_t umask = ::umask(0); + bool superuser = geteuid() == 0; + bool preserve_owner = superuser; + bool preserve_perms = superuser; + bool isPrintAllNode = false; + bool isPrintTarget = false; + bool check_decomp = false; + bool isExtractAllNode = false; + bool isExtractTarget = false; + bool useMultiThread = false; + unsigned int threadNum = 0; + unsigned int hardwareConcurrency = thread::hardware_concurrency(); + unsigned int limitHardwareConcurrency = hardwareConcurrency * 2; + bool overwrite = false; + string targetPath; + string targetConfPath; + bool extractOnlyConfAndSeLabel = false; + + public: + + static inline ExtractOperation *getInstance() { + if (!instance) { + instance = new ExtractOperation(); + } + return instance; + } + + static void erofsOperationExit(); + + void setImgPath(const char *path); + + const string &getImgPath() const; + + const string &getImgBaseName() const; + + void setOutDir(const char *path); + + int initOutDir(); + + int createExtractOutDir() const; + + int createExtractConfigDir() const; + + const string &getOutDir() const; + + const string &getConfDir() const; + + int initAllErofsNode() const; + + int initErofsNodeByTarget() const; + + int initErofsNodeAuto() const; + + /** + * new ErofsNode(const char *path, short typeId, struct erofs_inode *inode) + * + * @param path + * @param typeId + * @param inode + */ + static const ErofsNode *createErofsNode(const char *path, short typeId, struct erofs_inode *inode); + + static void initDirAndOther(); + + static void addErofsNode(ErofsNode *eNode); + + static const vector &getErofsNodes(); + + /** + * print all extract entity + * "Extract: type=DIR dataLayout= / 0 0 0644 capabilities=0x0" + */ + static void printInitializedNode() ; + + void extractFsConfigAndSeLabel() const; + + void writeExceptionInfo2File() const; + + void extractNodeDirs() const; + + void extractErofsNode() const; + + void extractErofsNodeMultiThread() const; + }; + + /** + * Global instance + */ + static ExtractOperation *eo = ExtractOperation::getInstance(); +} +#endif // end EXTRACT_OPERATION_H diff --git a/extract/include/ExtractState.h b/extract/include/ExtractState.h new file mode 100644 index 00000000..64eca058 --- /dev/null +++ b/extract/include/ExtractState.h @@ -0,0 +1,20 @@ +#ifndef EXTRACT_STATE_H +#define EXTRACT_STATE_H + +/** + * extract.erofs result + */ +enum extract_result { + RET_EXTRACT_DONE = 0, + RET_EXTRACT_CONFIG_DONE = 1, + RET_EXTRACT_CONFIG_FAIL, + RET_EXTRACT_INIT_FAIL, + RET_EXTRACT_INIT_NODE_FAIL, + RET_EXTRACT_OUTDIR_ROOT, + RET_EXTRACT_CREATE_DIR_FAIL, + RET_EXTRACT_CREATE_FILE_FAIL, + RET_EXTRACT_THREAD_NUM_ERROR, + RET_EXTRACT_FAIL_SKIP, + RET_EXTRACT_FAIL_EXIT, +}; +#endif // EXTRACT_STATE_H diff --git a/extract/include/Logging.h b/extract/include/Logging.h new file mode 100644 index 00000000..b0cc300c --- /dev/null +++ b/extract/include/Logging.h @@ -0,0 +1,82 @@ +#ifndef _EXTRACT_LOGGING_H +#define _EXTRACT_LOGGING_H + +/** + * Partial code reference：https://blog.csdn.net/m_pfly_fish/article/details/118541894 + * Partial code reference: erofs/print.h + */ +#define LOG_COLOR_BLACK "30" +#define LOG_COLOR_RED "31" +#define LOG_COLOR_RED2 "91" +#define LOG_COLOR_GREEN "32" +#define LOG_COLOR_GREEN2 "92" +#define LOG_COLOR_BROWN "33" +#define LOG_COLOR_BROWN2 "93" +#define LOG_COLOR_BLUE "34" +#define LOG_COLOR_BLUE2 "94" +#define LOG_COLOR_PURPLE "35" + +#define LOG_COLOR(COLOR) "\033[0;" COLOR "m" +#define LOG_BOLD(COLOR) "\033[1;" COLOR "m" +#define LOG_RESET_COLOR "\033[0m" + +#define RED LOG_COLOR(LOG_COLOR_RED) +#define RED_BOLD LOG_BOLD(LOG_COLOR_RED) +#define RED2 LOG_COLOR(LOG_COLOR_RED2) +#define RED2_BOLD LOG_BOLD(LOG_COLOR_RED2) +#define GREEN2_BOLD LOG_BOLD(LOG_COLOR_GREEN2) +#define BROWN LOG_COLOR(LOG_COLOR_BROWN) +#define BROWN2 LOG_COLOR(LOG_COLOR_BROWN2) +#define BROWN2_BOLD LOG_BOLD(LOG_COLOR_BROWN2) +#define BROWN_BOLD LOG_BOLD(LOG_COLOR_BROWN) +#define BLUE LOG_COLOR(LOG_COLOR_BLUE) +#define BLUE_BOLD LOG_BOLD(LOG_COLOR_BLUE) +#define BLUE2_BOLD LOG_BOLD(LOG_COLOR_BLUE2) +#define COLOR_NONE LOG_RESET_COLOR + +#ifndef LOG_TAG +#define LOG_TAG "Extract" +#endif + +/** + * _log_print + * + * @param fmt + * @param ... + */ +static inline void _log_print(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + vfprintf(stdout, fmt, ap); + va_end(ap); + fflush(stdout); +} + +#ifndef NDEBUG +#define EX_FUNC_LINE_FMT "%s:%d, " +#define ex_fmt(fmt) LOG_TAG ": " EX_FUNC_LINE_FMT fmt "\n" +#define EX_FMT_FUNC_LINE(fmt) ex_fmt(fmt), __func__, __LINE__ +#define ex_tag_color_fmt(color, fmt) color LOG_TAG ": " COLOR_NONE EX_FUNC_LINE_FMT fmt "\n" +#define EX_TAG_C_FMT_FUNC_LINE(color, fmt) ex_tag_color_fmt(color, fmt), __func__, __LINE__ +#define LOGCD(fmt, ...) _log_print(EX_TAG_C_FMT_FUNC_LINE(BLUE2_BOLD, fmt), ##__VA_ARGS__) +#define LOGD(fmt, ...) _log_print(EX_FMT_FUNC_LINE(fmt), ##__VA_ARGS__) +#else +#define ex_fmt(fmt) LOG_TAG ": " fmt "\n" +#define EX_FMT_FUNC_LINE(fmt) ex_fmt(fmt) +#define ex_tag_color_fmt(color, fmt) color LOG_TAG ": " COLOR_NONE fmt "\n" +#define EX_TAG_C_FMT_FUNC_LINE(color, fmt) ex_tag_color_fmt(color, fmt) +#define LOGCD(fmt, ...) 0 +#define LOGD(fmt, ...) 0 +#endif + +#define LOGCV(fmt, ...) _log_print(EX_TAG_C_FMT_FUNC_LINE(BLUE2_BOLD, fmt), ##__VA_ARGS__) +#define LOGCI(fmt, ...) _log_print(EX_TAG_C_FMT_FUNC_LINE(BROWN2_BOLD, fmt), ##__VA_ARGS__) +#define LOGCW(fmt, ...) _log_print(EX_TAG_C_FMT_FUNC_LINE(BROWN2_BOLD, fmt), ##__VA_ARGS__) +#define LOGCE(fmt, ...) _log_print(EX_TAG_C_FMT_FUNC_LINE(RED2_BOLD, fmt), ##__VA_ARGS__) + +#define LOGV(fmt, ...) _log_print(EX_FMT_FUNC_LINE(fmt), ##__VA_ARGS__) +#define LOGI(fmt, ...) _log_print(EX_FMT_FUNC_LINE(fmt), ##__VA_ARGS__) +#define LOGW(fmt, ...) _log_print(EX_FMT_FUNC_LINE(fmt), ##__VA_ARGS__) +#define LOGE(fmt, ...) _log_print(EX_FMT_FUNC_LINE(fmt), ##__VA_ARGS__) + +#endif // _EXTRACT_LOGGING_H diff --git a/extract/include/Utils.h b/extract/include/Utils.h new file mode 100644 index 00000000..2645a595 --- /dev/null +++ b/extract/include/Utils.h @@ -0,0 +1,67 @@ +#ifndef EXTRACT_UTILS_H +#define EXTRACT_UTILS_H + +#include +#include +#include +#include + +using namespace std; + +static inline bool dirExists(const string &dirPath) { + struct stat st = {}; + + if (stat(dirPath.c_str(), &st) == 0) { + return S_ISDIR(st.st_mode); + } + return false; +} + +static inline bool fileExists(const string &filePath) { + struct stat st = {}; + + if (stat(filePath.c_str(), &st) == 0) { + return S_ISREG(st.st_mode); + } + return false; +} + +static inline int mkdirs(const char *dirPath, mode_t mode) { + int len, err = 0; + char str[PATH_MAX + 1] = {0}; + strncpy(str, dirPath, PATH_MAX); + len = strlen(str); + for (int i = 0; i < len; i++) { + if (str[i] == '/' && i > 0) { + str[i] = '\0'; + if (access(str, F_OK) != 0) { + err = mkdir(str, mode); + if (err) return err; + } + str[i] = '/'; + } + } + if (len > 0 && access(str, F_OK) != 0) { + err = mkdir(str, mode); + } + return err; +} + +static inline void strTrim(string &str) { + if (!str.empty()) { + str.erase(0, str.find_first_not_of(" \n\r\t\v\f")); + str.erase(str.find_last_not_of(" \n\r\t\v\f") + 1); + } +} + +static inline void strReplaceAll(string &str, const string &oldValue, const string &newValue) { + auto oldValueSize = oldValue.size(); + auto newValueSize = newValue.size(); + auto pos = str.find(oldValue); + while (pos != string::npos) { + str.replace(pos, oldValueSize, newValue); + pos = str.find(oldValue, pos + newValueSize); + } +} + +#endif // EXTRACT_UTILS_H diff --git a/extract/include/threadpool.h b/extract/include/threadpool.h new file mode 100644 index 00000000..698894f1 --- /dev/null +++ b/extract/include/threadpool.h @@ -0,0 +1,158 @@ +#pragma once +#ifndef THREAD_POOL_H +#define THREAD_POOL_H + +#include +#include +#include +#include +#include +#include +#include +//#include + +/** + * based on C++11 , a mini threadpool , accept variable number of parameters 基于C++11的线程池,简洁且可以带任意多的参数 + * https://github.com/sekaiacg/threadpool + */ +namespace std { +//线程池最大容量,应尽量设小一点 +#define THREADPOOL_MAX_NUM 128 +//线程池是否可以自动增长(如果需要,且不超过 THREADPOOL_MAX_NUM) +//#define THREADPOOL_AUTO_GROW + +//线程池,可以提交变参函数或拉姆达表达式的匿名函数执行,可以获取执行返回值 +//不直接支持类成员函数, 支持类静态成员函数或全局函数,Opteron()函数等 + class threadpool { + unsigned short _initSize; //初始化线程数量 + using Task = function; //定义类型 + vector _pool; //线程池 + queue _tasks; //任务队列 + mutex _lock; //任务队列同步锁 +#ifdef THREADPOOL_AUTO_GROW + mutex _lockGrow; //线程池增长同步锁 +#endif // !THREADPOOL_AUTO_GROW + condition_variable _task_cv; //条件阻塞 + atomic _run{true}; //线程池是否执行 + atomic _idlThrNum{0}; //空闲线程数量 + + public: + inline threadpool(unsigned short size = 1) { + _initSize = size; + addThread(size); + } + + inline ~threadpool() { + _run = false; + _task_cv.notify_all(); // 唤醒所有线程执行 + for (thread &thread: _pool) { +// thread.detach(); // 让线程“自生自灭” + if (thread.joinable()) + thread.join(); // 等待任务结束， 前提：线程一定会执行完 + } + } + + public: + // 提交一个任务 + // 调用.get()获取返回值会等待任务执行完,获取返回值 + // 有两种方法可以实现调用类成员， + // 一种是使用 bind： .commit(std::bind(&Dog::sayHello, &dog)); + // 一种是用 mem_fn： .commit(std::mem_fn(&Dog::sayHello), this) + template + auto commit(F &&f, Args &&... args) -> future { +// if (!_run) // stoped ?? +// throw runtime_error("commit on ThreadPool is stopped."); +// return nullptr; + + using RetType = decltype(f(args...)); // typename std::result_of::type, 函数 f 的返回值类型 + auto task = make_shared>( + ::bind(::forward(f), ::forward(args)...) + ); // 把函数入口及参数,打包(绑定) + future future = task->get_future(); + { // 添加任务到队列 + lock_guard lock{ + _lock};//对当前块的语句加锁 lock_guard 是 mutex 的 stack 封装类，构造的时候 lock()，析构的时候 unlock() + _tasks.emplace([task]() { // push(Task{...}) 放到队列后面 + (*task)(); + }); + } +#ifdef THREADPOOL_AUTO_GROW + if (_idlThrNum < 1 && _pool.size() < THREADPOOL_MAX_NUM) + addThread(1); +#endif // !THREADPOOL_AUTO_GROW + _task_cv.notify_one(); // 唤醒一个线程执行 + + return future; + } + + // 提交一个无参任务, 且无返回值 + template + void commit2(F &&task) { + if (!_run) return; + { + lock_guard lock{_lock}; + _tasks.emplace(std::forward(task)); + } +#ifdef THREADPOOL_AUTO_GROW + if (_idlThrNum < 1 && _pool.size() < THREADPOOL_MAX_NUM) + addThread(1); +#endif // !THREADPOOL_AUTO_GROW + _task_cv.notify_one(); + } + + //空闲线程数量 + int idlCount() { return _idlThrNum; } + + //线程数量 + int thrCount() { return _pool.size(); } + +#ifndef THREADPOOL_AUTO_GROW + private: +#endif // !THREADPOOL_AUTO_GROW + + //添加指定数量的线程 + void addThread(unsigned short size) { +#ifdef THREADPOOL_AUTO_GROW + if (!_run) // stoped ?? + throw runtime_error("Grow on ThreadPool is stopped."); + unique_lock lockGrow{ _lockGrow }; //自动增长锁 +#endif // !THREADPOOL_AUTO_GROW + for (; _pool.size() < THREADPOOL_MAX_NUM && size > 0; --size) { //增加线程数量,但不超过 预定义数量 THREADPOOL_MAX_NUM + _pool.emplace_back([this] { //工作线程函数 + while (true) //防止 _run==false 时立即结束,此时任务队列可能不为空 + { + Task task; // 获取一个待执行的 task + { + // unique_lock 相比 lock_guard 的好处是：可以随时 unlock() 和 lock() + unique_lock lock{_lock}; + _task_cv.wait(lock, [this] { // wait 直到有 task, 或需要停止 + return !_run || !_tasks.empty(); + }); + if (!_run && _tasks.empty()) + return; + _idlThrNum--; + task = ::move(_tasks.front()); // 按先进先出从队列取一个 task + _tasks.pop(); + } + task();//执行任务 +#ifdef THREADPOOL_AUTO_GROW + if (_idlThrNum>0 && _pool.size() > _initSize) //支持自动释放空闲线程,避免峰值过后大量空闲线程 + return; +#endif // !THREADPOOL_AUTO_GROW + { + unique_lock lock{_lock}; + _idlThrNum++; + } + } + }); + { + unique_lock lock{_lock}; + _idlThrNum++; + } + } + } + }; + +} + +#endif //https://github.com/lzpong/ diff --git a/extract/main.cpp b/extract/main.cpp new file mode 100644 index 00000000..76303abf --- /dev/null +++ b/extract/main.cpp @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2023 skkk + */ + +#include +#include +#include +#include + +#include "ExtractState.h" +#include "ExtractOperation.h" +#include "Logging.h" + +using namespace skkk; + +static void usage() { + char buf[1024]; + sprintf(buf, + BROWN "usage: [options]" COLOR_NONE "\n" + " " GREEN2_BOLD "-h, --help" COLOR_NONE " " BROWN "Display this help and exit" COLOR_NONE "\n" + " " GREEN2_BOLD "-i, --image=[FILE]" COLOR_NONE " " BROWN "Image file" COLOR_NONE "\n" + " " GREEN2_BOLD "-p" COLOR_NONE " " BROWN "Print all entrys" COLOR_NONE "\n" + " " GREEN2_BOLD "--print=X" COLOR_NONE " " BROWN "Print the target of path X" COLOR_NONE "\n" + " " GREEN2_BOLD "-x" COLOR_NONE " " BROWN "Extract all items" COLOR_NONE "\n" + " " GREEN2_BOLD "--extract=X" COLOR_NONE " " BROWN "Extract the target of path X" COLOR_NONE "\n" + " " GREEN2_BOLD "-f, --overwrite" COLOR_NONE " " BROWN "[" GREEN2_BOLD "default: skip" COLOR_NONE BROWN "] overwrite files that already exist" COLOR_NONE "\n" + " " GREEN2_BOLD "-T#" COLOR_NONE " " BROWN "[" GREEN2_BOLD "1-%u" COLOR_NONE BROWN "] Use # threads, -T0: " GREEN2_BOLD "%u" COLOR_NONE COLOR_NONE "\n" + " " GREEN2_BOLD "--only-cfg" COLOR_NONE " " BROWN "Only extract fs_config and file_contexts" COLOR_NONE "\n" + " " GREEN2_BOLD "-o, --outdir=X" COLOR_NONE " " BROWN "Output dir" COLOR_NONE "\n" + " " GREEN2_BOLD "-V, --version" COLOR_NONE " " BROWN "Print the version info" COLOR_NONE "\n", + eo->limitHardwareConcurrency, + eo->hardwareConcurrency + ); + fputs(buf, stderr); +} + +static inline void print_version() { + printf(" " BROWN "extract.erofs:" COLOR_NONE " " RED2_BOLD "1.0.0" COLOR_NONE "\n"); + printf(" " BROWN "Author:" COLOR_NONE " " RED2_BOLD "skkk" COLOR_NONE "\n"); +} + +static struct option arg_options[] = { + {"help", no_argument, nullptr, 'h'}, + {"version", no_argument, nullptr, 'V'}, + {"image", required_argument, nullptr, 'i'}, + {"outdir", required_argument, nullptr, 'o'}, + {"print", required_argument, nullptr, 'P'}, + {"extract", required_argument, nullptr, 'X'}, + {"overwrite", no_argument, nullptr, 'f'}, + {"only-cfg", no_argument, nullptr, 1}, + {nullptr, no_argument, nullptr, 0}, +}; + +static int parseAndCheckExtractCfg(int argc, char **argv) { + int opt; + int rc = RET_EXTRACT_CONFIG_FAIL; + bool enterParseOpt = false; + while ((opt = getopt_long(argc, argv, "hi:pxfT:o:V", arg_options, nullptr)) != -1) { + enterParseOpt = true; + switch (opt) { + case 'h': + usage(); + goto exit; + case 'V': + print_version(); + goto exit; + case 'i': + if (optarg) { + eo->setImgPath(optarg); + } + LOGCD("imgPath=%s", eo->getImgPath().c_str()); + break; + case 'o': + if (optarg) { + eo->setOutDir(optarg); + } + LOGCD("outDir=%s", eo->getOutDir().c_str()); + break; + case 'p': + eo->isPrintAllNode = true; + LOGCD("isPrintAllNode=%d", eo->isPrintAllNode); + break; + case 'P': + eo->isPrintTarget = true; + if (optarg) eo->targetPath = optarg; + LOGCD("isPrintTarget=%d targetPath=%s", eo->isPrintTarget, eo->targetPath.c_str()); + break; + case 'x': + eo->check_decomp = true; + eo->isExtractAllNode = true; + LOGCD("isExtractAllNode=%d check_decomp=%d", eo->isExtractAllNode, eo->check_decomp); + break; + case 'X': + eo->check_decomp = true; + eo->isExtractTarget = true; + if (optarg) eo->targetPath = optarg; + LOGCD("isExtractTarget=%d targetPath=%s", eo->isExtractTarget, eo->targetPath.c_str()); + break; + case 'f': + eo->overwrite = true; + LOGCD("overwrite=%d", eo->overwrite); + break; + case 'T': + eo->useMultiThread = true; + if (optarg) { + unsigned int n = strtoul(optarg, nullptr, 10); + if (n > 0) { + eo->threadNum = n; + } + } + break; + case 1: + eo->extractOnlyConfAndSeLabel = true; + LOGCD("extractOnlyConfAndSeLabel=%d", eo->extractOnlyConfAndSeLabel); + break; + default: + usage(); + print_version(); + goto exit; + } + } + + if (enterParseOpt) { + bool err; + // check needed arg + err = !eo->getImgPath().empty() && fileExists(eo->getImgPath()); + if (!err) { + LOGCE("img file '%s' does not exist", eo->getImgPath().c_str()); + goto exit; + } + rc = !eo->initOutDir(); + if (!rc) { + goto exit; + } + LOGCD("outDir=%s confDir=%s", eo->getOutDir().c_str(), eo->getConfDir().c_str()); + + if (eo->useMultiThread) { + if (eo->threadNum > eo->limitHardwareConcurrency) { + rc = RET_EXTRACT_THREAD_NUM_ERROR; + LOGCE("Threads min: 1 , max: %u", eo->limitHardwareConcurrency); + goto exit; + } else if (eo->threadNum == 0) { + eo->threadNum = eo->hardwareConcurrency; + } + LOGCD("Threads num=%u", eo->threadNum); + } + rc = RET_EXTRACT_CONFIG_DONE; + } else { + usage(); + } + +exit: + return rc; +} + +static inline void printOperationTime(struct timeval *start, struct timeval *end) { + LOGCI(GREEN2_BOLD "The operation took: " COLOR_NONE RED2 "%.3f" COLOR_NONE "%s", + (end->tv_sec - start->tv_sec) + (float) (end->tv_usec - start->tv_usec) / 1000000, + GREEN2_BOLD " second(s)." COLOR_NONE + ); +} + +int main(int argc, char **argv) { + int ret = RET_EXTRACT_DONE, err; + + struct timeval start = {}, end = {}; + // Start time + gettimeofday(&start, nullptr); + + // Initialize erofs config + erofs_init_configure(); + cfg.c_dbg_lvl = EROFS_ERR; + + // Initialize extract config + err = parseAndCheckExtractCfg(argc, argv); + if (err != RET_EXTRACT_CONFIG_DONE) { + ret = err; + goto exit; + } + + err = dev_open_ro(eo->getImgPath().c_str()); + if (err) { + ret = RET_EXTRACT_INIT_FAIL; + LOGCE("failed to open '%s'", eo->getImgPath().c_str()); + goto exit; + } + + err = erofs_read_superblock(); + if (err) { + ret = RET_EXTRACT_INIT_FAIL; + LOGCE("failed to read superblock"); + goto exit_dev_close; + } + + if (eo->isPrintTarget || eo->isExtractTarget) + err = eo->initErofsNodeByTarget(); + else if (eo->isPrintAllNode || eo->isExtractAllNode) + err = eo->initAllErofsNode(); + if (err) { + ret = RET_EXTRACT_INIT_NODE_FAIL; + goto exit_dev_close; + } + + if (eo->isPrintTarget || eo->isPrintAllNode) { + ExtractOperation::printInitializedNode(); + goto exit_dev_close; + } + + LOGCI(GREEN2_BOLD "Starting..." COLOR_NONE); + + if ((eo->isExtractTarget || eo->isExtractAllNode) && eo->extractOnlyConfAndSeLabel) { + err = eo->createExtractConfigDir(); + if (err) { + ret = RET_EXTRACT_CREATE_DIR_FAIL; + goto exit_dev_close; + } + eo->extractFsConfigAndSeLabel(); + goto end; + } + + if (eo->isExtractTarget || eo->isExtractAllNode) { + err = eo->createExtractConfigDir() && eo->createExtractOutDir(); + if (err) { + ret = RET_EXTRACT_CREATE_DIR_FAIL; + goto exit_dev_close; + } + eo->extractFsConfigAndSeLabel(); + eo->useMultiThread ? eo->extractErofsNodeMultiThread() : eo->extractErofsNode(); + goto end; + } + +end: + // End time + gettimeofday(&end, nullptr); + printOperationTime(&start, &end); + +exit_dev_close: + dev_close(); + LOGCD("ErofsNode size=%lu", eo->getErofsNodes().size()); + LOGCD("main exit ret=%d", ret); + +exit: + blob_closeall(); + erofs_exit_configure(); + ExtractOperation::erofsOperationExit(); + return ret; +}