Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] bpo-23689: Memory leak in Modules/sre_lib.h #11926

Closed
wants to merge 7 commits into from
Closed

[WIP] bpo-23689: Memory leak in Modules/sre_lib.h #11926

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9206c6c
Merge pull request #1 from python/master
animalize Jan 19, 2019
ed20d2f
Merge pull request #3 from python/master
animalize Jan 30, 2019
93ff15f
Merge pull request #4 from python/master
animalize Feb 9, 2019
4e15c9a
Merge pull request #5 from python/master
animalize Feb 15, 2019
ff729b1
Merge pull request #6 from python/master
animalize Feb 18, 2019
2d4996b
stage 1
wjssz Feb 19, 2019
740f53c
pass test_re.py
wjssz Feb 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Modules/_sre.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ state_reset(SRE_STATE* state)
state->lastmark = -1;
state->lastindex = -1;

state->repeat = NULL;
state->repeat = -1;

data_stack_dealloc(state);
}
Expand Down Expand Up @@ -409,6 +409,7 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string,
}
state->lastmark = -1;
state->lastindex = -1;
state->repeat = -1;

state->buffer.buf = NULL;
ptr = getstring(string, &length, &isbytes, &charsize, &state->buffer);
Expand Down
4 changes: 2 additions & 2 deletions Modules/sre.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ typedef struct SRE_REPEAT_T {
Py_ssize_t count;
SRE_CODE* pattern; /* points to REPEAT operator arguments */
void* last_ptr; /* helper to check for infinite loops */
struct SRE_REPEAT_T *prev; /* points to previous repeat context */
Py_ssize_t prev; /* points to previous repeat context in stack */
} SRE_REPEAT;

typedef struct {
Expand All @@ -82,7 +82,7 @@ typedef struct {
size_t data_stack_size;
size_t data_stack_base;
/* current repeat context */
SRE_REPEAT *repeat;
Py_ssize_t repeat;
} SRE_STATE;

typedef struct {
Expand Down
113 changes: 62 additions & 51 deletions Modules/sre_lib.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,8 +420,10 @@ do { \
if (sizeof(type) > state->data_stack_size - alloc_pos) { \
int j = data_stack_grow(state, sizeof(type)); \
if (j < 0) return j; \
if (ctx_pos != -1) \
if (ctx_pos >= 0) \
DATA_STACK_LOOKUP_AT(state, SRE(match_context), ctx, ctx_pos); \
if (ctx->u.rep >= 0) \
DATA_STACK_LOOKUP_AT(state, SRE_REPEAT, temp_repeat, ctx->u.rep); \
} \
ptr = (type*)(state->data_stack+alloc_pos); \
state->data_stack_base += sizeof(type); \
Expand All @@ -441,8 +443,10 @@ do { \
if (size > state->data_stack_size - state->data_stack_base) { \
int j = data_stack_grow(state, size); \
if (j < 0) return j; \
if (ctx_pos != -1) \
if (ctx_pos >= 0) \
DATA_STACK_LOOKUP_AT(state, SRE(match_context), ctx, ctx_pos); \
if (ctx->u.rep >= 0) \
DATA_STACK_LOOKUP_AT(state, SRE_REPEAT, temp_repeat, ctx->u.rep); \
} \
memcpy(state->data_stack+state->data_stack_base, data, size); \
state->data_stack_base += size; \
Expand Down Expand Up @@ -538,7 +542,7 @@ typedef struct {
Py_ssize_t lastindex;
union {
SRE_CODE chr;
SRE_REPEAT* rep;
Py_ssize_t rep;
} u;
int toplevel;
} SRE(match_context);
Expand All @@ -556,6 +560,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel)

SRE(match_context)* ctx;
SRE(match_context)* nextctx;
SRE_REPEAT* temp_repeat;

TRACE(("|%p|%p|ENTER\n", pattern, state->ptr));

Expand Down Expand Up @@ -798,7 +803,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel)
TRACE(("|%p|%p|BRANCH\n", ctx->pattern, ctx->ptr));
LASTMARK_SAVE();
ctx->u.rep = state->repeat;
if (ctx->u.rep)
if (ctx->u.rep >= 0)
MARK_PUSH(ctx->lastmark);
for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) {
if (ctx->pattern[1] == SRE_OP_LITERAL &&
Expand All @@ -813,16 +818,16 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel)
state->ptr = ctx->ptr;
DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1);
if (ret) {
if (ctx->u.rep)
if (ctx->u.rep >= 0)
MARK_POP_DISCARD(ctx->lastmark);
RETURN_ON_ERROR(ret);
RETURN_SUCCESS;
}
if (ctx->u.rep)
if (ctx->u.rep >= 0)
MARK_POP_KEEP(ctx->lastmark);
LASTMARK_RESTORE();
}
if (ctx->u.rep)
if (ctx->u.rep >= 0)
MARK_POP_DISCARD(ctx->lastmark);
RETURN_FAILURE;

Expand Down Expand Up @@ -988,21 +993,19 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel)
ctx->pattern[1], ctx->pattern[2]));

/* install new repeat context */
ctx->u.rep = (SRE_REPEAT*) PyObject_MALLOC(sizeof(*ctx->u.rep));
if (!ctx->u.rep) {
PyErr_NoMemory();
RETURN_FAILURE;
}
ctx->u.rep->count = -1;
ctx->u.rep->pattern = ctx->pattern;
ctx->u.rep->prev = state->repeat;
ctx->u.rep->last_ptr = NULL;
DATA_ALLOC(SRE_REPEAT, temp_repeat);
ctx->u.rep = alloc_pos;
temp_repeat->count = -1;
temp_repeat->pattern = ctx->pattern;
temp_repeat->prev = state->repeat;
temp_repeat->last_ptr = NULL;
state->repeat = ctx->u.rep;

state->ptr = ctx->ptr;
DO_JUMP(JUMP_REPEAT, jump_repeat, ctx->pattern+ctx->pattern[0]);
state->repeat = ctx->u.rep->prev;
PyObject_FREE(ctx->u.rep);
DATA_LOOKUP_AT(SRE_REPEAT, temp_repeat, ctx->u.rep);
state->repeat = temp_repeat->prev;
DATA_POP_DISCARD(temp_repeat);

if (ret) {
RETURN_ON_ERROR(ret);
Expand All @@ -1018,59 +1021,63 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel)
matches in here... */

ctx->u.rep = state->repeat;
if (!ctx->u.rep)
if (ctx->u.rep < 0)
RETURN_ERROR(SRE_ERROR_STATE);
DATA_LOOKUP_AT(SRE_REPEAT, temp_repeat, ctx->u.rep);

state->ptr = ctx->ptr;

ctx->count = ctx->u.rep->count+1;
ctx->count = temp_repeat->count+1;

TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern,
ctx->ptr, ctx->count));

if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) {
if (ctx->count < (Py_ssize_t) temp_repeat->pattern[1]) {
/* not enough matches */
ctx->u.rep->count = ctx->count;
temp_repeat->count = ctx->count;
DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1,
ctx->u.rep->pattern+3);
temp_repeat->pattern+3);
DATA_LOOKUP_AT(SRE_REPEAT, temp_repeat, ctx->u.rep);
if (ret) {
RETURN_ON_ERROR(ret);
RETURN_SUCCESS;
}
ctx->u.rep->count = ctx->count-1;
temp_repeat->count = ctx->count-1;
state->ptr = ctx->ptr;
RETURN_FAILURE;
}

if ((ctx->count < (Py_ssize_t) ctx->u.rep->pattern[2] ||
ctx->u.rep->pattern[2] == SRE_MAXREPEAT) &&
state->ptr != ctx->u.rep->last_ptr) {
if ((ctx->count < (Py_ssize_t) temp_repeat->pattern[2] ||
temp_repeat->pattern[2] == SRE_MAXREPEAT) &&
state->ptr != temp_repeat->last_ptr) {
/* we may have enough matches, but if we can
match another item, do so */
ctx->u.rep->count = ctx->count;
temp_repeat->count = ctx->count;
LASTMARK_SAVE();
MARK_PUSH(ctx->lastmark);
/* zero-width match protection */
DATA_PUSH(&ctx->u.rep->last_ptr);
ctx->u.rep->last_ptr = state->ptr;
DATA_PUSH(&temp_repeat->last_ptr);
temp_repeat->last_ptr = state->ptr;
DO_JUMP(JUMP_MAX_UNTIL_2, jump_max_until_2,
ctx->u.rep->pattern+3);
DATA_POP(&ctx->u.rep->last_ptr);
temp_repeat->pattern+3);
DATA_LOOKUP_AT(SRE_REPEAT, temp_repeat, ctx->u.rep);
DATA_POP(&temp_repeat->last_ptr);
if (ret) {
MARK_POP_DISCARD(ctx->lastmark);
RETURN_ON_ERROR(ret);
RETURN_SUCCESS;
}
MARK_POP(ctx->lastmark);
LASTMARK_RESTORE();
ctx->u.rep->count = ctx->count-1;
temp_repeat->count = ctx->count-1;
state->ptr = ctx->ptr;
}

/* cannot match more repeated items here. make sure the
tail matches */
state->repeat = ctx->u.rep->prev;
state->repeat = temp_repeat->prev;
DO_JUMP(JUMP_MAX_UNTIL_3, jump_max_until_3, ctx->pattern);
DATA_LOOKUP_AT(SRE_REPEAT, temp_repeat, ctx->u.rep);
RETURN_ON_SUCCESS(ret);
state->repeat = ctx->u.rep;
state->ptr = ctx->ptr;
Expand All @@ -1081,35 +1088,38 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel)
/* <REPEAT> <skip> <1=min> <2=max> item <MIN_UNTIL> tail */

ctx->u.rep = state->repeat;
if (!ctx->u.rep)
if (ctx->u.rep < 0)
RETURN_ERROR(SRE_ERROR_STATE);
DATA_LOOKUP_AT(SRE_REPEAT, temp_repeat, ctx->u.rep);

state->ptr = ctx->ptr;

ctx->count = ctx->u.rep->count+1;
ctx->count = temp_repeat->count+1;

TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern,
ctx->ptr, ctx->count, ctx->u.rep->pattern));
ctx->ptr, ctx->count, temp_repeat->pattern));

if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) {
if (ctx->count < (Py_ssize_t) temp_repeat->pattern[1]) {
/* not enough matches */
ctx->u.rep->count = ctx->count;
temp_repeat->count = ctx->count;
DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1,
ctx->u.rep->pattern+3);
temp_repeat->pattern+3);
DATA_LOOKUP_AT(SRE_REPEAT, temp_repeat, ctx->u.rep);
if (ret) {
RETURN_ON_ERROR(ret);
RETURN_SUCCESS;
}
ctx->u.rep->count = ctx->count-1;
temp_repeat->count = ctx->count-1;
state->ptr = ctx->ptr;
RETURN_FAILURE;
}

LASTMARK_SAVE();

/* see if the tail matches */
state->repeat = ctx->u.rep->prev;
state->repeat = temp_repeat->prev;
DO_JUMP(JUMP_MIN_UNTIL_2, jump_min_until_2, ctx->pattern);
DATA_LOOKUP_AT(SRE_REPEAT, temp_repeat, ctx->u.rep);
if (ret) {
RETURN_ON_ERROR(ret);
RETURN_SUCCESS;
Expand All @@ -1120,23 +1130,24 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel)

LASTMARK_RESTORE();

if ((ctx->count >= (Py_ssize_t) ctx->u.rep->pattern[2]
&& ctx->u.rep->pattern[2] != SRE_MAXREPEAT) ||
state->ptr == ctx->u.rep->last_ptr)
if ((ctx->count >= (Py_ssize_t) temp_repeat->pattern[2]
&& temp_repeat->pattern[2] != SRE_MAXREPEAT) ||
state->ptr == temp_repeat->last_ptr)
RETURN_FAILURE;

ctx->u.rep->count = ctx->count;
temp_repeat->count = ctx->count;
/* zero-width match protection */
DATA_PUSH(&ctx->u.rep->last_ptr);
ctx->u.rep->last_ptr = state->ptr;
DATA_PUSH(&temp_repeat->last_ptr);
temp_repeat->last_ptr = state->ptr;
DO_JUMP(JUMP_MIN_UNTIL_3,jump_min_until_3,
ctx->u.rep->pattern+3);
DATA_POP(&ctx->u.rep->last_ptr);
temp_repeat->pattern+3);
DATA_LOOKUP_AT(SRE_REPEAT, temp_repeat, ctx->u.rep);
DATA_POP(&temp_repeat->last_ptr);
if (ret) {
RETURN_ON_ERROR(ret);
RETURN_SUCCESS;
}
ctx->u.rep->count = ctx->count-1;
temp_repeat->count = ctx->count-1;
state->ptr = ctx->ptr;
RETURN_FAILURE;

Expand Down