Skip to content

Commit

Permalink
Undo autoformatting
Browse files Browse the repository at this point in the history 
Signed-off-by: Joaquin Anton <[email protected]>
  • Loading branch information
@jantonguirao
jantonguirao committed Apr 16, 2024
1 parent cb175ef commit 4faad07
Show file tree
Hide file tree
Showing 11 changed files with 189 additions and 190 deletions.
62 changes: 33 additions & 29 deletions dali/operators/reader/loader/coco_loader.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ struct Annotation {
};

template <typename T>
std::enable_if_t<std::is_pod<T>::value, void> Read(std::ifstream& file, T &data,
const char* filename) {
std::enable_if_t<std::is_pod<T>::value, void>
Read(std::ifstream& file, T& data, const char* filename) {
int64_t bytes = sizeof(T);
file.read(reinterpret_cast<char *>(&data), bytes);
DALI_ENFORCE(file.gcount() == bytes,
Expand All @@ -79,16 +79,16 @@ void Read(std::ifstream& file, span<T> data, const char* filename) {
}

template <typename T>
void Write(std::ofstream &file, T data, const char* filename) {
file.write(reinterpret_cast<const char *>(&data), sizeof(T));
void Write(std::ofstream& file, T data, const char* filename) {
file.write(reinterpret_cast<const char*>(&data), sizeof(T));
DALI_ENFORCE(file.good(), make_string("Error reading from path: ", filename));
}

template <typename T>
void Write(std::ofstream &file, span<const T> data, const char* filename) {
void Write(std::ofstream& file, span<const T> data, const char* filename) {
if (data.empty())
return;
file.write(reinterpret_cast<const char *>(data.data()), sizeof(T) * data.size());
file.write(reinterpret_cast<const char*>(data.data()), sizeof(T) * data.size());
DALI_ENFORCE(file.good(), make_string("Error reading from path: ", filename));
}

Expand Down Expand Up @@ -123,7 +123,7 @@ void SaveToFile(const std::vector<RLEMaskPtr> &input, const std::string path) {
}

template <typename T>
void SaveToFile(const std::vector<std::vector<T>> &input, const std::string path) {
void SaveToFile(const std::vector<std::vector<T> > &input, const std::string path) {
if (input.empty())
return;
std::ofstream file(path, std::ios_base::binary | std::ios_base::out);
Expand All @@ -132,7 +132,7 @@ void SaveToFile(const std::vector<std::vector<T>> &input, const std::string path
unsigned size = input.size();
Write(file, size, path.c_str());

for (auto &v : input) {
for (auto& v : input) {
size = v.size();
assert(size > 0);
Write(file, size, path.c_str());
Expand Down Expand Up @@ -186,7 +186,7 @@ void LoadFromFile(std::vector<RLEMaskPtr> &output, const std::string path) {
}

template <typename T>
void LoadFromFile(std::vector<std::vector<T>> &output, const std::string path) {
void LoadFromFile(std::vector<std::vector<T> > &output, const std::string path) {
std::ifstream file(path);
output.clear();
if (!file.good())
Expand All @@ -212,7 +212,7 @@ void LoadFromFile(std::vector<filesystem::FileLabelEntry> &entries, const std::s
int id = 0;
std::string filename;
while (file >> filename) {
entries.push_back({std::move(filename), id});
entries.emplace_back(std::move(filename), int{id});
++id;
}
}
Expand All @@ -236,7 +236,7 @@ void ParseImageInfo(LookaheadParser &parser, std::vector<ImageInfo> &image_infos
} else if (0 == std::strcmp(internal_key, "file_name")) {
image_info.filename_ = parser.GetString();
} else {
parser.SkipValue();
parser.SkipValue();
}
}
image_infos.emplace_back(std::move(image_info));
Expand Down Expand Up @@ -271,7 +271,8 @@ void ParseCategories(LookaheadParser &parser, std::map<int, int> &category_ids)
}

void ParseAnnotations(LookaheadParser &parser, std::vector<Annotation> &annotations,
float min_size_threshold, bool ltrb, bool parse_segmentation, bool parse_rle,
float min_size_threshold, bool ltrb,
bool parse_segmentation, bool parse_rle,
bool include_iscrowd = true) {
std::string rle_str;
std::vector<uint32_t> rle_uints;
Expand Down Expand Up @@ -344,8 +345,8 @@ void ParseAnnotations(LookaheadParser &parser, std::vector<Annotation> &annotati
} else if (parser.PeekType() == kArrayType) {
annotation.tag_ = Annotation::POLYGON;
int coord_offset = 0;
auto &segm_meta = annotation.poly_.segm_meta_;
auto &segm_coords = annotation.poly_.segm_coords_;
auto& segm_meta = annotation.poly_.segm_meta_;
auto& segm_coords = annotation.poly_.segm_coords_;
parser.EnterArray();
while (parser.NextArrayValue()) {
segm_meta.push_back(coord_offset);
Expand Down Expand Up @@ -376,7 +377,8 @@ void ParseAnnotations(LookaheadParser &parser, std::vector<Annotation> &annotati
}

void ParseJsonFile(const OpSpec &spec, std::vector<detail::ImageInfo> &image_infos,
std::vector<detail::Annotation> &annotations, std::map<int, int> &category_ids,
std::vector<detail::Annotation> &annotations,
std::map<int, int> &category_ids,
bool parse_segmentation, bool parse_rle) {
const auto annotations_file = spec.GetArgument<string>("annotations_file");

Expand Down Expand Up @@ -405,8 +407,8 @@ void ParseJsonFile(const OpSpec &spec, std::vector<detail::ImageInfo> &image_inf
} else if (0 == std::strcmp(key, "categories")) {
detail::ParseCategories(parser, category_ids);
} else if (0 == std::strcmp(key, "annotations")) {
ParseAnnotations(parser, annotations, sz_threshold, ltrb, parse_segmentation, parse_rle,
include_iscrowd);
ParseAnnotations(parser, annotations, sz_threshold, ltrb, parse_segmentation,
parse_rle, include_iscrowd);
} else {
parser.SkipValue();
}
Expand All @@ -416,7 +418,7 @@ void ParseJsonFile(const OpSpec &spec, std::vector<detail::ImageInfo> &image_inf
} // namespace detail

void CocoLoader::SavePreprocessedAnnotations(
const std::string &path, const std::vector<filesystem::FileLabelEntry> &entries) {
const std::string &path, const std::vector<filesystem::FileLabelEntry> &entries) {
using detail::SaveToFile;
SaveToFile(offsets_, path + "/offsets.dat");
SaveToFile(boxes_, path + "/boxes.dat");
Expand Down Expand Up @@ -449,9 +451,9 @@ void CocoLoader::SavePreprocessedAnnotations(

void CocoLoader::ParsePreprocessedAnnotations() {
assert(HasPreprocessedAnnotations(spec_));
const auto path = spec_.HasArgument("meta_files_path") ?
spec_.GetArgument<string>("meta_files_path") :
spec_.GetArgument<string>("preprocessed_annotations");
const auto path = spec_.HasArgument("meta_files_path")
? spec_.GetArgument<string>("meta_files_path")
: spec_.GetArgument<string>("preprocessed_annotations");
using detail::LoadFromFile;
LoadFromFile(offsets_, path + "/offsets.dat");
LoadFromFile(boxes_, path + "/boxes.dat");
Expand Down Expand Up @@ -488,12 +490,13 @@ void CocoLoader::ParseJsonAnnotations() {
std::map<int, int> category_ids;

bool parse_segmentation = output_polygon_masks_ || output_pixelwise_masks_;
detail::ParseJsonFile(spec_, image_infos, annotations, category_ids, parse_segmentation,
output_pixelwise_masks_);
detail::ParseJsonFile(spec_, image_infos, annotations, category_ids,
parse_segmentation, output_pixelwise_masks_);

if (images_.empty()) {
std::sort(image_infos.begin(), image_infos.end(),
[&](auto &left, auto &right) { return left.original_id_ < right.original_id_; });
std::sort(image_infos.begin(), image_infos.end(), [&](auto &left, auto &right) {
return left.original_id_ < right.original_id_;
});
for (auto &info : image_infos) {
images_.push_back(info.filename_);
}
Expand Down Expand Up @@ -541,7 +544,7 @@ void CocoLoader::ParseJsonAnnotations() {
int64_t sample_vertices_count = 0;
int64_t mask_offset = masks_rles_.size();
int64_t mask_count = 0;
for (const auto *annotation_ptr : img_annotations_map[image_id]) {
for (const auto* annotation_ptr : img_annotations_map[image_id]) {
const auto &annotation = *annotation_ptr;
if (remap_classes) {
labels_.push_back(category_ids[annotation.category_id_]);
Expand Down Expand Up @@ -625,7 +628,7 @@ void CocoLoader::ParseJsonAnnotations() {
}
}

file_label_entries_.push_back({std::move(image_info.filename_), new_image_id});
file_label_entries_.emplace_back(std::move(image_info.filename_), new_image_id);
new_image_id++;
}
}
Expand All @@ -634,8 +637,9 @@ void CocoLoader::ParseJsonAnnotations() {
images_.clear();

if (spec_.GetArgument<bool>("save_preprocessed_annotations")) {
SavePreprocessedAnnotations(spec_.GetArgument<std::string>("save_preprocessed_annotations_dir"),
file_label_entries_);
SavePreprocessedAnnotations(
spec_.GetArgument<std::string>("save_preprocessed_annotations_dir"),
file_label_entries_);
}
}

Expand Down
40 changes: 18 additions & 22 deletions dali/operators/reader/loader/coco_loader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include <unordered_set>
#include <utility>
#include <vector>

#include "dali/operators/reader/loader/file_label_loader.h"
#include "dali/core/common.h"
#include "dali/core/error_handling.h"
#include "dali/core/geom/vec.h"
#include "dali/core/unique_handle.h"
#include "dali/operators/reader/loader/file_label_loader.h"

extern "C" {
#include "third_party/cocoapi/common/maskApi.h"
Expand Down Expand Up @@ -60,8 +60,7 @@ inline bool HasSavePreprocessedAnnotations(const OpSpec &spec) {

inline bool HasSavePreprocessedAnnotationsDir(const OpSpec &spec) {
return spec.HasArgument("save_preprocessed_annotations_dir") ||
(spec.HasArgument("dump_meta_files_path") &&
spec.GetArgument<bool>("dump_meta_files_path"));
(spec.HasArgument("dump_meta_files_path") && spec.GetArgument<bool>("dump_meta_files_path"));
}

struct RLEMask : public UniqueHandle<RLE, RLEMask> {
Expand Down Expand Up @@ -90,26 +89,22 @@ struct RLEMask : public UniqueHandle<RLE, RLEMask> {
rleFree(&handle);
}

const RLE *operator->() const {
return &handle_;
}
RLE *operator->() {
return &handle_;
}
const RLE* operator->() const { return &handle_; }
RLE* operator->() { return &handle_; }
};

using RLEMaskPtr = std::shared_ptr<RLEMask>;

class DLL_PUBLIC CocoLoader : public FileLabelLoaderBase<true> {
public:
explicit inline CocoLoader(const OpSpec &spec)
: FileLabelLoaderBase<true>(spec, spec.GetArgument<bool>("shuffle_after_epoch")),
spec_(spec) {
: FileLabelLoaderBase<true>(spec, spec.GetArgument<bool>("shuffle_after_epoch"))
, spec_(spec) {
has_preprocessed_annotations_ = HasPreprocessedAnnotations(spec);
DALI_ENFORCE(has_preprocessed_annotations_ || spec.HasArgument("annotations_file"),
"Either ``annotations_file`` or ``preprocessed_annotations`` must be provided");
"Either ``annotations_file`` or ``preprocessed_annotations`` must be provided");
if (has_preprocessed_annotations_) {
for (const char *arg_name : {"annotations_file", "skip_empty", "ratio", "ltrb", "images",
for (const char* arg_name : {"annotations_file", "skip_empty", "ratio", "ltrb", "images",
"size_threshold", "dump_meta_files", "dump_meta_files_path"}) {
if (spec.HasArgument(arg_name))
DALI_FAIL(make_string("When reading data from preprocessed annotation files, \"",
Expand All @@ -126,9 +121,8 @@ class DLL_PUBLIC CocoLoader : public FileLabelLoaderBase<true> {
}

if (HasSavePreprocessedAnnotations(spec_) != HasSavePreprocessedAnnotationsDir(spec_)) {
DALI_FAIL(
"``save_preprocessed_annotations`` and ``save_preprocessed_annotations_dir`` "
"should be provided together");
DALI_FAIL("``save_preprocessed_annotations`` and ``save_preprocessed_annotations_dir`` "
"should be provided together");
}
}

Expand Down Expand Up @@ -159,9 +153,11 @@ class DLL_PUBLIC CocoLoader : public FileLabelLoaderBase<true> {

PixelwiseMasksInfo pixelwise_masks_info(int image_idx) const {
assert(output_pixelwise_masks_);
return {{heights_[image_idx], widths_[image_idx], 1},
{masks_rles_.data() + mask_offsets_[image_idx], mask_counts_[image_idx]},
{masks_rles_idx_.data() + mask_offsets_[image_idx], mask_counts_[image_idx]}};
return {
{heights_[image_idx], widths_[image_idx], 1},
{masks_rles_.data() + mask_offsets_[image_idx], mask_counts_[image_idx]},
{masks_rles_idx_.data() + mask_offsets_[image_idx], mask_counts_[image_idx]}
};
}

span<const ivec3> polygons(int image_idx) const {
Expand Down Expand Up @@ -205,8 +201,8 @@ class DLL_PUBLIC CocoLoader : public FileLabelLoaderBase<true> {

void ParseJsonAnnotations();

void SavePreprocessedAnnotations(const std::string &path,
const std::vector<filesystem::FileLabelEntry> &entries);
void SavePreprocessedAnnotations(
const std::string &path, const std::vector<filesystem::FileLabelEntry> &image_id_pairs);

private:
const OpSpec spec_;
Expand Down
5 changes: 2 additions & 3 deletions dali/operators/reader/loader/file_label_loader.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ void FileLabelLoaderBase<checkpointing_supported>::PrepareEmpty(ImageLabelWrappe

template<bool checkpointing_supported>
void FileLabelLoaderBase<checkpointing_supported>::ReadSample(ImageLabelWrapper &image_label) {
auto entry = file_label_entries_[current_index_++];
auto image_pair = file_label_entries_[current_index_++];

// handle wrap-around
MoveToNextShard(current_index_);
Expand All @@ -50,11 +50,10 @@ void FileLabelLoaderBase<checkpointing_supported>::ReadSample(ImageLabelWrapper
return;
}

auto uri = filesystem::join_path(file_root_, entry.filename);
auto current_image = FileStream::Open(uri, {read_ahead_, !copy_read_data_, false}, entry.size);
Index image_size = current_image->Size();

if (copy_read_data_ || !current_image->CanMemoryMap()) {
if (copy_read_data_) {
if (image_label.image.shares_data()) {
image_label.image.Reset();
}
Expand Down
Loading

0 comments on commit 4faad07

Please sign in to comment.