glaurung

Fixture 41

tokenizer

C · 1 functions · 4 lanes · 4 of 4 function-lanes behave identically

All 4 lanes recompile and return the same results as the original.

A byte-level lexer driven by an explicit state variable. Character classification produces a dense switch over a small domain, and the accepted token count is written through an output pointer, so both the jump table and the store-through-parameter must survive.

tests/decompiler_fixtures/src/41_tokenizer.c source 
#include <stdint.h>

/* A byte-level lexer driven by an explicit state variable.  Character
 * classification produces a dense switch over a small domain, and the accepted
 * token count is written through an output pointer, so both the jump table and
 * the store-through-parameter must survive. */

#define TOK_MAX 16

#define TOK_STATE_BLANK 0
#define TOK_STATE_WORD 1
#define TOK_STATE_NUMBER 2

static int32_t classify(uint8_t byte) {
    if (byte >= (uint8_t)'0' && byte <= (uint8_t)'9') {
        return TOK_STATE_NUMBER;
    }
    if ((byte >= (uint8_t)'a' && byte <= (uint8_t)'z') ||
        (byte >= (uint8_t)'A' && byte <= (uint8_t)'Z') ||
        byte == (uint8_t)'_') {
        return TOK_STATE_WORD;
    }
    return TOK_STATE_BLANK;
}

__attribute__((noinline)) int32_t
tokenize(const uint8_t *text, int32_t length, int32_t *word_count,
         int32_t *number_count) {
    int32_t state = TOK_STATE_BLANK;
    int32_t words = 0;
    int32_t numbers = 0;
    int32_t index;
    if (text == 0 || word_count == 0 || number_count == 0 || length < 0 ||
        length > TOK_MAX) {
        return -1;
    }
    for (index = 0; index < length; ++index) {
        int32_t next = classify(text[index]);
        if (next != state) {
            switch (next) {
            case TOK_STATE_WORD:
                words += 1;
                break;
            case TOK_STATE_NUMBER:
                numbers += 1;
                break;
            default:
                break;
            }
            state = next;
        }
    }
    *word_count = words;
    *number_count = numbers;
    return words + numbers;
}

Recovered C

Generated by glaurung decompile --style decbench at b47f6b43. baseline.json records the result after recompiling the C and calling it beside the original with seeded inputs.

clang -O0

1/1
tokenize pass 62 lines 
// glaurung: tokenize @ 0x1100
int32_t tokenize(const uint8_t * arg0, int32_t arg1, int32_t * arg2, int32_t * arg3) {
    extern int classify(unsigned char);
    int state;
    int words;
    int numbers;
    int index;
    int next;
    int local_4;
    int local_40;
    int var3;
    // x86-64 prologue: save rbp, frame 64 bytes
    state = 0;
    words = 0;
    numbers = 0;
    if ((arg0 == 0)) {
        local_4 = -1;
        // x86-64 epilogue: restore rbp
        return (unsigned int)(local_4);
    }
    if ((arg2 == 0)) {
        local_4 = -1;
        // x86-64 epilogue: restore rbp
        return (unsigned int)(local_4);
    }
    if ((arg3 == 0)) {
        local_4 = -1;
        // x86-64 epilogue: restore rbp
        return (unsigned int)(local_4);
    }
    if (((long)(arg1) < 0)) {
        local_4 = -1;
        // x86-64 epilogue: restore rbp
        return (unsigned int)(local_4);
    }
    if (((((unsigned long)((unsigned int)(arg1)) == 16) | ((long)(arg1) < 16)) == 0)) {
        local_4 = -1;
        // x86-64 epilogue: restore rbp
        return (unsigned int)(local_4);
    }
    for (index = 0; (index < arg1); index++) {
        var3 = classify((unsigned int)((unsigned char)(arg0[index])));
        next = var3;
        if (((unsigned int)(next) != (unsigned int)(state))) {
            local_40 = next;
            if (((unsigned long)((unsigned int)(((unsigned long)((unsigned int)(next)) - 1))) == 0)) {
                words = ((unsigned int)(words) + 1);
            } else {
                if (((unsigned long)((unsigned int)(((unsigned long)((unsigned int)(local_40)) - 2))) == 0)) {
                    numbers = ((unsigned int)(numbers) + 1);
                } else {
                }
            }
            state = next;
        }
    }
    *(int *)((long)arg2) = words;
    *(int *)((long)arg3) = numbers;
    local_4 = ((unsigned int)(words) + numbers);
    // x86-64 epilogue: restore rbp
    return (unsigned int)(local_4);
}

clang -O2

1/1
tokenize pass 80 lines 
// glaurung: tokenize @ 0x1100
int32_t tokenize(const uint8_t * arg0, int32_t arg1, int32_t * arg2, int32_t * arg3) {
    int state;
    int words;
    int numbers;
    int next;
    int index;
    long ret;
    long var18;
    int var19;
    long var33;
    long var34;
    long var4;
    int var41;
    long var5;
    long var6;
    ret = 0xffffffff;
    if (((unsigned long)(16) < (unsigned long)((unsigned long)((unsigned int)(arg1))))) {
        return ret;
    }
    if ((arg0 == 0)) {
        return ret;
    }
    if ((arg2 == 0)) {
        return ret;
    }
    if ((arg3 == 0)) {
        return ret;
    }
    if (((unsigned long)((unsigned int)(arg1)) == 0)) {
        var4 = 0;
        var5 = 0;
    } else {
        var6 = (unsigned long)((unsigned int)(arg1));
        state = 0;
        words = 0;
        numbers = 0;
        var18 = 0;
        do {
            var19 = (unsigned int)((unsigned char)(*(char *)(((long)arg0 + var18))));
            next = 2;
            if (((unsigned long)(10) <= (unsigned long)((unsigned long)((unsigned char)(((unsigned long)((unsigned int)((var19 - 48))) & 255)))))) {
                next = 1;
                if (((unsigned long)(26) <= (unsigned long)((unsigned long)((unsigned char)(((unsigned long)((unsigned int)((var19 - 97))) & 255)))))) {
                    next = (unsigned int)((unsigned char)(((((unsigned long)((unsigned char)((var19 & 255))) == 95) | ((unsigned long)((unsigned long)((unsigned char)(((unsigned long)((unsigned int)((var19 - 65))) & 255)))) < (unsigned long)(26))) & 255)));
                }
            }
            var33 = (unsigned long)((unsigned int)(words));
            var34 = (unsigned long)((unsigned int)(numbers));
            if (((unsigned int)(next) != (unsigned int)(state))) {
                if (((unsigned long)((unsigned int)(next)) == 2)) {
                    var34 = (unsigned long)((unsigned int)((numbers + 1)));
                    state = 2;
                    var33 = (unsigned long)((unsigned int)(words));
                } else {
                    state = 0;
                    var33 = (unsigned long)((unsigned int)(words));
                    var34 = (unsigned long)((unsigned int)(numbers));
                    if (((unsigned long)((unsigned int)(next)) == 1)) {
                        var33 = (unsigned long)((unsigned int)((words + 1)));
                        state = 1;
                        var34 = (unsigned long)((unsigned int)(numbers));
                    }
                }
            }
            index = (var18 + 1);
            words = var33;
            numbers = var34;
            var18 = (unsigned long)((unsigned int)(index));
            var4 = var33;
            var5 = var34;
        } while ((var6 != index));
    }
    *(int *)(((long)arg2)) = var4;
    *(int *)(((long)arg3)) = var5;
    var41 = (var5 + var4);
    ret = (unsigned long)((unsigned int)(var41));
    // x86-64 epilogue: tear down frame
    return (unsigned int)(var41);
}

gcc -O0

1/1
tokenize pass 53 lines 
// glaurung: tokenize @ 0x1145
int32_t tokenize(const uint8_t * arg0, int32_t arg1, int32_t * arg2, int32_t * arg3) {
    extern int classify(unsigned char);
    int state;
    int words;
    int numbers;
    int index;
    int next;
    int var7;
    // x86-64 prologue: save rbp, frame 64 bytes
    state = 0;
    words = 0;
    numbers = 0;
    if ((arg0 == 0)) {
        // x86-64 epilogue: restore rbp
        return 0xffffffff;
    }
    if ((arg2 == 0)) {
        // x86-64 epilogue: restore rbp
        return 0xffffffff;
    }
    if ((arg3 == 0)) {
        // x86-64 epilogue: restore rbp
        return 0xffffffff;
    }
    if (((long)(arg1) < 0)) {
        // x86-64 epilogue: restore rbp
        return 0xffffffff;
    }
    if (((((unsigned long)((unsigned int)(arg1)) == 16) | ((long)(arg1) < 16)) == 0)) {
        // x86-64 epilogue: restore rbp
        return 0xffffffff;
    }
    for (index = 0; (index < arg1); index++) {
        var7 = classify((unsigned long)((unsigned int)((unsigned char)(((unsigned int)((unsigned char)(arg0[index])) & 255)))));
        next = var7;
        if (((unsigned int)(next) != (unsigned int)(state))) {
            if (((unsigned long)((unsigned int)(next)) == 1)) {
                words = (words + 1);
            } else {
                if (((unsigned long)((unsigned int)(next)) == 2)) {
                    numbers = (numbers + 1);
                } else {
                }
            }
            state = next;
        }
    }
    *(int *)((long)arg2) = words;
    *(int *)((long)arg3) = numbers;
    // x86-64 epilogue: restore rbp
    return (unsigned int)(((unsigned long)((unsigned int)(numbers)) + (unsigned long)((unsigned int)(words))));
}

gcc -O2

1/1
tokenize pass 76 lines 
// glaurung: tokenize @ 0x1100
int32_t tokenize(const uint8_t * arg0, int32_t arg1, int32_t * arg2, int32_t * arg3) {
    int state;
    int index;
    int next;
    int numbers;
    int words;
    long var0;
    long var11;
    long var12;
    long var13;
    long var14;
    long var15;
    int var17;
    long var19;
    long var22;
    var0 = (unsigned long)((unsigned int)(arg1));
    if ((arg0 == 0)) {
        return 0xffffffff;
    }
    if ((arg2 == 0)) {
        return 0xffffffff;
    }
    if ((arg3 == 0)) {
        return 0xffffffff;
    }
    if (((unsigned long)(16) < (unsigned long)((unsigned long)((unsigned int)(arg1))))) {
        return 0xffffffff;
    }
    if (((unsigned long)((unsigned int)(arg1)) == 0)) {
        *(int *)(((long)arg2)) = var0;
        *(int *)(((long)arg3)) = 0;
        return 0;
    }
    var11 = 0x3ffffff43ffffff;
    var12 = (long)((((long)arg0 + (unsigned long)((unsigned int)((arg1 - 1)))) + 1));
    var13 = 0;
    var14 = 0;
    var15 = (long)arg0;
    state = 0;
    do {
        var17 = (unsigned int)((unsigned char)(*(char *)((var15))));
        if (((unsigned long)(57) < (unsigned long)((unsigned long)((unsigned char)((var17 & 255)))))) {
            var19 = (unsigned long)((unsigned int)((var17 - 65)));
            if (((unsigned long)(57) < (unsigned long)((unsigned long)((unsigned char)((var19 & 255)))))) {
                L_115b: ;
                var22 = 0;
            } else {
                if (((((unsigned long)(var11) >> (var19 & 63)) & 1) == 0)) {
                    goto L_115b;
                } else {
                    var22 = (unsigned long)((unsigned int)(state));
                    if (((unsigned long)((unsigned int)(state)) != 1)) {
                        var14 = (unsigned long)((unsigned int)((var14 + 1)));
                        var22 = 1;
                    }
                }
            }
        } else {
            if (((unsigned long)(47) < (unsigned long)((unsigned long)((unsigned char)((var17 & 255)))))) {
                var22 = (unsigned long)((unsigned int)(state));
                if (((unsigned long)((unsigned int)(state)) != 2)) {
                    var13 = (unsigned long)((unsigned int)((var13 + 1)));
                    var22 = 2;
                }
            } else {
                goto L_115b;
            }
        }
        var15 = (var15 + 1);
        state = var22;
    } while ((var15 != var12));
    *(int *)(((long)arg2)) = var14;
    *(int *)(((long)arg3)) = var13;
    return (unsigned int)((var14 + var13));
}

← 213 fixtures