Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

hash: bound the hashtable size #431

Merged
merged 5 commits into from
Nov 22, 2017
Merged

hash: bound the hashtable size #431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3431181
hash_table: new structure to support counters and eviction modes
edsiper Nov 17, 2017
6d4f483
env: use new hashtable prototype
edsiper Nov 17, 2017
891485c
filter_kubernetes: use new hashtable prototype
edsiper Nov 17, 2017
00b173a
tests: internal: hashtable: use new hashtable func prototype
edsiper Nov 17, 2017
60c68c8
hash: bound the hashtable size
Nov 14, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions include/fluent-bit/flb_hash.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,22 @@
#include <stdio.h>
#include <stdlib.h>

/* Eviction modes when the table reach full capacity (if any) */
#define FLB_HASH_EVICT_NONE 0
#define FLB_HASH_EVICT_OLDER 1
#define FLB_HASH_EVICT_LESS_USED 2
#define FLB_HASH_EVICT_RANDOM 3

struct flb_hash_entry {
time_t created;
uint64_t hits;
char *key;
size_t key_len;
char *val;
size_t val_size;
struct mk_list _head;
struct flb_hash_table *table; /* link to parent flb_hash_table */
struct mk_list _head; /* link to flb_hash_table->chains */
struct mk_list _head_parent; /* link to flb_hash->entries */
};

struct flb_hash_table {
Expand All @@ -40,11 +50,15 @@ struct flb_hash_table {
};

struct flb_hash {
int evict_mode;
int max_entries;
int total_count;
size_t size;
struct mk_list entries;
struct flb_hash_table *table;
};

struct flb_hash *flb_hash_create(size_t size);
struct flb_hash *flb_hash_create(int evict_mode, size_t size, int max_entries);
void flb_hash_destroy(struct flb_hash *ht);

int flb_hash_add(struct flb_hash *ht, char *key, int key_len,
Expand Down
4 changes: 3 additions & 1 deletion plugins/filter_kubernetes/kube_conf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,9 @@ struct flb_kube *flb_kube_conf_create(struct flb_filter_instance *i,
ctx->api_https ? "https" : "http",
ctx->api_host, ctx->api_port);

ctx->hash_table = flb_hash_create(FLB_HASH_TABLE_SIZE);
ctx->hash_table = flb_hash_create(FLB_HASH_EVICT_RANDOM,
FLB_HASH_TABLE_SIZE,
FLB_HASH_TABLE_SIZE);
if (!ctx->hash_table) {
flb_kube_conf_destroy(ctx);
return NULL;
Expand Down
2 changes: 1 addition & 1 deletion src/flb_env.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ struct flb_env *flb_env_create()
}

/* Create the hash-table */
ht = flb_hash_create(FLB_ENV_SIZE);
ht = flb_hash_create(FLB_HASH_EVICT_NONE, FLB_ENV_SIZE, -1);
if (!ht) {
flb_free(env);
return NULL;
Expand Down
65 changes: 57 additions & 8 deletions src/flb_hash.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,15 +84,18 @@ static unsigned int gen_hash(const void *key, int len)
return (unsigned int) h;
}

static inline void flb_hash_entry_free(struct flb_hash_entry *entry)
static inline void flb_hash_entry_free(struct flb_hash *ht, struct flb_hash_entry *entry)
{
mk_list_del(&entry->_head);
mk_list_del(&entry->_head_parent);
entry->table->count--;
ht->total_count--;
flb_free(entry->key);
flb_free(entry->val);
flb_free(entry);
}

struct flb_hash *flb_hash_create(size_t size)
struct flb_hash *flb_hash_create(int evict_mode, size_t size, int max_entries)
{
int i;
struct flb_hash_table *tmp;
Expand All @@ -108,7 +111,12 @@ struct flb_hash *flb_hash_create(size_t size)
return NULL;
}

mk_list_init(&ht->entries);
ht->evict_mode = evict_mode;
ht->max_entries = max_entries;
ht->total_count = 0;
ht->size = size;
ht->total_count = 0;
ht->table = flb_calloc(1, sizeof(struct flb_hash_table) * size);
if (!ht->table) {
flb_errno();
Expand Down Expand Up @@ -138,15 +146,33 @@ void flb_hash_destroy(struct flb_hash *ht)
table = &ht->table[i];
mk_list_foreach_safe(head, tmp, &table->chains) {
entry = mk_list_entry(head, struct flb_hash_entry, _head);
flb_hash_entry_free(entry);
table->count--;
flb_hash_entry_free(ht, entry);
}
}

flb_free(ht->table);
flb_free(ht);
}

static void flb_hash_evict_random(struct flb_hash *ht)
{
int id;
int count = 0;
struct mk_list *tmp;
struct mk_list *head;
struct flb_hash_entry *entry;

id = random() % ht->total_count;
mk_list_foreach_safe(head, tmp, &ht->entries) {
if (id == count) {
entry = mk_list_entry(head, struct flb_hash_entry, _head_parent);
flb_hash_entry_free(ht, entry);
break;
}
count++;
}
}

int flb_hash_add(struct flb_hash *ht, char *key, int key_len,
char *val, size_t val_size)
{
Expand All @@ -162,6 +188,23 @@ int flb_hash_add(struct flb_hash *ht, char *key, int key_len,
return -1;
}

/* Check capacity */
if (ht->max_entries > 0 && ht->total_count >= ht->max_entries) {
/* FIXME: handle eviction mode */
if (ht->evict_mode == FLB_HASH_EVICT_NONE) {

}
else if (ht->evict_mode == FLB_HASH_EVICT_OLDER) {

}
else if (ht->evict_mode == FLB_HASH_EVICT_LESS_USED) {

}
else if (ht->evict_mode == FLB_HASH_EVICT_RANDOM) {
flb_hash_evict_random(ht);
}
}

/* Generate hash number */
hash = gen_hash(key, key_len);
id = (hash % ht->size);
Expand All @@ -172,6 +215,8 @@ int flb_hash_add(struct flb_hash *ht, char *key, int key_len,
flb_errno();
return -1;
}
entry->created = time(NULL);
entry->hits = 0;

/* Store the key and value as a new memory region */
entry->key = flb_strdup(key);
Expand All @@ -183,6 +228,7 @@ int flb_hash_add(struct flb_hash *ht, char *key, int key_len,
flb_free(entry);
return -1;
}

/*
* Copy the buffer and append a NULL byte in case the caller set and
* expects a string.
Expand All @@ -193,24 +239,27 @@ int flb_hash_add(struct flb_hash *ht, char *key, int key_len,

/* Link the new entry in our table at the end of the list */
table = &ht->table[id];
entry->table = table;

/* Check if the new key already exists */
if (table->count == 0) {
mk_list_add(&entry->_head, &table->chains);
mk_list_add(&entry->_head_parent, &ht->entries);
}
else {
mk_list_foreach_safe(head, tmp, &table->chains) {
old = mk_list_entry(head, struct flb_hash_entry, _head);
if (strcmp(old->key, entry->key) == 0) {
flb_hash_entry_free(old);
table->count--;
flb_hash_entry_free(ht, old);
break;
}
}
mk_list_add(&entry->_head, &table->chains);
mk_list_add(&entry->_head_parent, &ht->entries);
}

table->count++;
ht->total_count++;

return id;
}
Expand Down Expand Up @@ -270,6 +319,7 @@ int flb_hash_get(struct flb_hash *ht, char *key, int key_len,
return -1;
}

entry->hits++;
*out_buf = entry->val;
*out_size = entry->val_size;

Expand Down Expand Up @@ -357,8 +407,7 @@ int flb_hash_del(struct flb_hash *ht, char *key)
return -1;
}

flb_hash_entry_free(entry);
table->count--;
flb_hash_entry_free(ht, entry);

return 0;
}
48 changes: 38 additions & 10 deletions tests/internal/hashtable.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ void test_create_zero()
{
struct flb_hash *ht;

ht = flb_hash_create(0);
ht = flb_hash_create(FLB_HASH_EVICT_NONE, 0, -1);
TEST_CHECK(ht == NULL);
}

Expand All @@ -91,7 +91,7 @@ void test_single()
size_t out_size;
struct flb_hash *ht;

ht = flb_hash_create(1);
ht = flb_hash_create(FLB_HASH_EVICT_NONE, 1, -1);
TEST_CHECK(ht != NULL);

ret = ht_add(ht, "key", "value");
Expand All @@ -112,7 +112,7 @@ void test_small_table()
struct map *m;
struct flb_hash *ht;

ht = flb_hash_create(1);
ht = flb_hash_create(FLB_HASH_EVICT_NONE, 8, -1);
TEST_CHECK(ht != NULL);

for (i = 0; i < sizeof(entries) / sizeof(struct map); i++) {
Expand All @@ -129,7 +129,7 @@ void test_medium_table()
struct map *m;
struct flb_hash *ht;

ht = flb_hash_create(8);
ht = flb_hash_create(FLB_HASH_EVICT_NONE, 8, -1);
TEST_CHECK(ht != NULL);

for (i = 0; i < sizeof(entries) / sizeof(struct map); i++) {
Expand All @@ -145,16 +145,16 @@ void test_chaining()
int i;
int inserts = 0;
int count;
int total = 0;
int chains = 0;
struct map *m;
struct mk_list *head;
struct flb_hash_table *table;
struct flb_hash *ht;

ht = flb_hash_create(8);
ht = flb_hash_create(FLB_HASH_EVICT_NONE, 8, -1);
TEST_CHECK(ht != NULL);

for (i = 0; i < sizeof(entries) / sizeof(struct map); i++) {
for (i = 0; i < 8; i++) {
m = &entries[i];
ht_add(ht, m->key, m->val);
inserts++;
Expand All @@ -168,11 +168,13 @@ void test_chaining()
}
TEST_CHECK(count == table->count);

total += count;
if (count > 0) {
chains++;
}
}

/* Tests diff between total, new minus 3 overrides */
TEST_CHECK(total == inserts - 3);
TEST_CHECK(chains == inserts - 3);
flb_hash_destroy(ht);
}

Expand All @@ -187,7 +189,7 @@ void test_delete_all()
struct flb_hash_table *table;
struct flb_hash *ht;

ht = flb_hash_create(8);
ht = flb_hash_create(FLB_HASH_EVICT_NONE, 8, -1);
TEST_CHECK(ht != NULL);

total = sizeof(entries) / sizeof(struct map);
Expand All @@ -214,12 +216,38 @@ void test_delete_all()
flb_hash_destroy(ht);
}

void test_random_eviction()
{
int ret;
char *out_buf;
size_t out_size;
struct flb_hash *ht;

ht = flb_hash_create(FLB_HASH_EVICT_RANDOM, 8, 1);
TEST_CHECK(ht != NULL);

ret = ht_add(ht, "key1", "value1");
TEST_CHECK(ret != -1);

ret = ht_add(ht, "key2", "value2");
TEST_CHECK(ret != -1);

ret = flb_hash_get(ht, "key1", 4, &out_buf, &out_size);
TEST_CHECK(ret == -1);

ret = flb_hash_get(ht, "key2", 4, &out_buf, &out_size);
TEST_CHECK(ret >= 0);

flb_hash_destroy(ht);
}

TEST_LIST = {
{ "zero_size", test_create_zero },
{ "single", test_single },
{ "small_table", test_small_table },
{ "medium_table", test_medium_table },
{ "chaining_count", test_chaining },
{ "delete_all", test_delete_all },
{ "random_eviction", test_random_eviction },
{ 0 }
};