glaurung

Fixture 51

rc4

C · 1 functions · 4 lanes · 4 of 4 function-lanes behave identically

All 4 lanes recompile and return the same results as the original.

RC4 key scheduling and keystream generation. Both phases permute a 256-byte state in place with two indices that wrap modulo 256, so the recovered code must keep the byte-width truncation of j exact.

tests/decompiler_fixtures/src/51_rc4.c source 
#include <stdint.h>

/* RC4 key scheduling and keystream generation.  Both phases permute a
 * 256-byte state in place with two indices that wrap modulo 256, so the
 * recovered code must keep the byte-width truncation of `j` exact. */

#define RC4_MAX 16

__attribute__((noinline)) uint32_t
rc4_keystream_checksum(const uint8_t *key, int32_t key_length, int32_t rounds) {
    uint8_t state[256];
    uint32_t checksum = 0;
    int32_t index;
    uint8_t i;
    uint8_t j = 0;
    if (key == 0 || key_length < 1 || key_length > RC4_MAX || rounds < 0 ||
        rounds > 64) {
        return 0;
    }
    for (index = 0; index < 256; ++index) {
        state[index] = (uint8_t)index;
    }
    for (index = 0; index < 256; ++index) {
        uint8_t swap;
        j = (uint8_t)(j + state[index] + key[index % key_length]);
        swap = state[index];
        state[index] = state[j];
        state[j] = swap;
    }
    i = 0;
    j = 0;
    for (index = 0; index < rounds; ++index) {
        uint8_t swap;
        i = (uint8_t)(i + 1);
        j = (uint8_t)(j + state[i]);
        swap = state[i];
        state[i] = state[j];
        state[j] = swap;
        checksum = checksum * 33u + state[(uint8_t)(state[i] + state[j])];
    }
    return checksum;
}

Recovered C

Generated by glaurung decompile --style decbench at b47f6b43. baseline.json records the result after recompiling the C and calling it beside the original with seeded inputs.

clang -O0

1/1
rc4_keystream_checksum pass 58 lines 
// glaurung: rc4_keystream_checksum @ 0x1100
__attribute__((no_stack_protector)) uint32_t rc4_keystream_checksum(const uint8_t * arg0, int32_t arg1, int32_t arg2) {
    unsigned int checksum;
    unsigned char j;
    int index;
    unsigned char local_12b;
    unsigned char i;
    unsigned char swap;
    unsigned char local_120[256];
    int local_130;
    int local_4;
    // x86-64 prologue: save rbp, frame 176 bytes
    checksum = 0;
    j = 0;
    if ((arg0 == 0)) {
        // x86-64 epilogue: restore rbp
        return 0;
    }
    if (((long)(arg1) < 1)) {
        // x86-64 epilogue: restore rbp
        return 0;
    }
    if (((((unsigned long)((unsigned int)(arg1)) == 16) | ((long)(arg1) < 16)) == 0)) {
        // x86-64 epilogue: restore rbp
        return 0;
    }
    if (((long)(arg2) < 0)) {
        // x86-64 epilogue: restore rbp
        return 0;
    }
    if (((((unsigned long)((unsigned int)(arg2)) == 64) | ((long)(arg2) < 64)) == 0)) {
        // x86-64 epilogue: restore rbp
        return 0;
    }
    for (index = 0; ((long)(index) < 256); index++) {
        *(signed char *)((&local_120[0] + (long)(index))) = index;
    }
    for (index = 0; ((long)(index) < 256); index++) {
        local_130 = ((unsigned int)(j) + (unsigned int)((unsigned char)(*(char *)((&local_120[0] + (long)(index))))));
        j = ((unsigned long)((unsigned int)(local_130)) + (unsigned char)(arg0[(long)((int)(((int)((((long long)(int)((((unsigned long)((long)((int)((unsigned long)((unsigned int)(index))))) >> 32) & 0xffffffff)) * (((long long)1) << 32)) + (unsigned int)((unsigned long)((unsigned int)(index)))) % (int)(arg1)))))]));
        local_12b = *(char *)((&local_120[0] + (long)(index)));
        *(signed char *)((&local_120[0] + (long)(index))) = *(char *)((&local_120[0] + j));
        *(signed char *)((&local_120[0] + j)) = local_12b;
    }
    i = 0;
    j = 0;
    for (index = 0; (index < arg2); index++) {
        i = (i + 1);
        j = (j + (unsigned char)(*(char *)((&local_120[0] + i))));
        swap = *(char *)((&local_120[0] + i));
        *(signed char *)((&local_120[0] + i)) = *(char *)((&local_120[0] + j));
        *(signed char *)((&local_120[0] + j)) = swap;
        checksum = ((checksum * 33) + (unsigned int)((unsigned char)(*(char *)((&local_120[0] + (unsigned int)((unsigned char)(((unsigned long)((unsigned int)(((unsigned int)((unsigned char)(*(char *)((&local_120[0] + i)))) + (unsigned int)((unsigned char)(*(char *)((&local_120[0] + j))))))) & 255))))))));
    }
    local_4 = checksum;
    // x86-64 epilogue: restore rbp
    return (unsigned int)(local_4);
}

clang -O2

1/1
rc4_keystream_checksum pass 129 lines 
// glaurung: rc4_keystream_checksum @ 0x1100
__attribute__((no_stack_protector)) uint32_t rc4_keystream_checksum(const uint8_t * arg0, int32_t arg1, int32_t arg2) {
    int index;
    unsigned char swap;
    unsigned int checksum;
    unsigned char j;
    unsigned char local_108[264];
    long ret;
    int var106;
    int var107;
    long var3;
    long var71;
    int var81;
    long var85;
    long var87;
    int var90;
    int var92;
    int var93;
    int var94;
    ret = 0;
    if (((unsigned long)((unsigned long)((unsigned int)(arg2))) <= (unsigned long)(64))) {
        ret = 0;
        if ((arg0 == 0)) {
            return ret;
        }
        ret = 0;
        if (((unsigned long)((unsigned long)((unsigned int)((arg1 - 17)))) < (unsigned long)(0xfffffff0))) {
            return ret;
        }
        var3 = (unsigned long)((unsigned int)(arg2));
        *(int *)(&local_108[0]) = 0x3020100;
        *(int *)((&local_108[0] + 4)) = 0x7060504;
        *(int *)((&local_108[0] + 8)) = 0xb0a0908;
        *(int *)((&local_108[0] + 12)) = 0xf0e0d0c;
        *(int *)((&local_108[0] + 16)) = 0x13121110;
        *(int *)((&local_108[0] + 20)) = 0x17161514;
        *(int *)((&local_108[0] + 24)) = 0x1b1a1918;
        *(int *)((&local_108[0] + 28)) = 0x1f1e1d1c;
        *(int *)((&local_108[0] + 32)) = 0x23222120;
        *(int *)((&local_108[0] + 36)) = 0x27262524;
        *(int *)((&local_108[0] + 40)) = 0x2b2a2928;
        *(int *)((&local_108[0] + 44)) = 0x2f2e2d2c;
        *(int *)((&local_108[0] + 48)) = 0x33323130;
        *(int *)((&local_108[0] + 52)) = 0x37363534;
        *(int *)((&local_108[0] + 56)) = 0x3b3a3938;
        *(int *)((&local_108[0] + 60)) = 0x3f3e3d3c;
        *(int *)((&local_108[0] + 64)) = 0x43424140;
        *(int *)((&local_108[0] + 68)) = 0x47464544;
        *(int *)((&local_108[0] + 72)) = 0x4b4a4948;
        *(int *)((&local_108[0] + 76)) = 0x4f4e4d4c;
        *(int *)((&local_108[0] + 80)) = 0x53525150;
        *(int *)((&local_108[0] + 84)) = 0x57565554;
        *(int *)((&local_108[0] + 88)) = 0x5b5a5958;
        *(int *)((&local_108[0] + 92)) = 0x5f5e5d5c;
        *(int *)((&local_108[0] + 96)) = 0x63626160;
        *(int *)((&local_108[0] + 100)) = 0x67666564;
        *(int *)((&local_108[0] + 104)) = 0x6b6a6968;
        *(int *)((&local_108[0] + 108)) = 0x6f6e6d6c;
        *(int *)((&local_108[0] + 112)) = 0x73727170;
        *(int *)((&local_108[0] + 116)) = 0x77767574;
        *(int *)((&local_108[0] + 120)) = 0x7b7a7978;
        *(int *)((&local_108[0] + 124)) = 0x7f7e7d7c;
        *(int *)((&local_108[0] + 128)) = 0x83828180;
        *(int *)((&local_108[0] + 132)) = 0x87868584;
        *(int *)((&local_108[0] + 136)) = 0x8b8a8988;
        *(int *)((&local_108[0] + 140)) = 0x8f8e8d8c;
        *(int *)((&local_108[0] + 144)) = 0x93929190;
        *(int *)((&local_108[0] + 148)) = 0x97969594;
        *(int *)((&local_108[0] + 152)) = 0x9b9a9998;
        *(int *)((&local_108[0] + 156)) = 0x9f9e9d9c;
        *(int *)((&local_108[0] + 160)) = 0xa3a2a1a0;
        *(int *)((&local_108[0] + 164)) = 0xa7a6a5a4;
        *(int *)((&local_108[0] + 168)) = 0xabaaa9a8;
        *(int *)((&local_108[0] + 172)) = 0xafaeadac;
        *(int *)((&local_108[0] + 176)) = 0xb3b2b1b0;
        *(int *)((&local_108[0] + 180)) = 0xb7b6b5b4;
        *(int *)((&local_108[0] + 184)) = 0xbbbab9b8;
        *(int *)((&local_108[0] + 188)) = 0xbfbebdbc;
        *(int *)((&local_108[0] + 192)) = 0xc3c2c1c0;
        *(int *)((&local_108[0] + 196)) = 0xc7c6c5c4;
        *(int *)((&local_108[0] + 200)) = 0xcbcac9c8;
        *(int *)((&local_108[0] + 204)) = 0xcfcecdcc;
        *(int *)((&local_108[0] + 208)) = 0xd3d2d1d0;
        *(int *)((&local_108[0] + 212)) = 0xd7d6d5d4;
        *(int *)((&local_108[0] + 216)) = 0xdbdad9d8;
        *(int *)((&local_108[0] + 220)) = 0xdfdedddc;
        *(int *)((&local_108[0] + 224)) = 0xe3e2e1e0;
        *(int *)((&local_108[0] + 228)) = 0xe7e6e5e4;
        *(int *)((&local_108[0] + 232)) = 0xebeae9e8;
        *(int *)((&local_108[0] + 236)) = 0xefeeedec;
        *(int *)((&local_108[0] + 240)) = 0xf3f2f1f0;
        *(int *)((&local_108[0] + 244)) = 0xf7f6f5f4;
        *(int *)((&local_108[0] + 248)) = 0xfbfaf9f8;
        *(int *)((&local_108[0] + 252)) = 0xfffefdfc;
        var71 = 0;
        index = 0;
        do {
            swap = (unsigned int)((unsigned char)(*(char *)((&local_108[0] + index))));
            var71 = ((unsigned int)((unsigned char)(arg0[((int)((((long long)(int)((((unsigned long)((long)((int)((unsigned long)((unsigned int)(index))))) >> 32) & 0xffffffff)) * (((long long)1) << 32)) + (unsigned int)((unsigned long)((unsigned int)(index)))) % (int)(arg1)))])) + ((unsigned int)((unsigned char)((var71 & 255))) + (unsigned long)(swap)));
            var81 = (unsigned int)((unsigned char)((var71 & 255)));
            *(signed char *)((&local_108[0] + index)) = *(char *)((&local_108[0] + var81));
            *(signed char *)((&local_108[0] + var81)) = swap;
            index = (index + 1);
        } while ((index != 256));
        ret = 0;
        if ((((unsigned long)((unsigned int)(var3)) == 0) | ((long)((int)(var3)) < 0))) {
            return ret;
        }
        var85 = 0;
        var87 = 0;
        checksum = 0;
        do {
            var90 = ((unsigned int)((unsigned char)((var87 & 255))) + 1);
            var87 = (unsigned long)((unsigned int)(var90));
            var92 = (unsigned int)((unsigned char)(((unsigned long)((unsigned int)(var90)) & 255)));
            var93 = (unsigned int)((unsigned char)(*(char *)((&local_108[0] + var92))));
            var85 = ((unsigned int)((unsigned char)((var85 & 255))) + (unsigned long)((unsigned int)(var93)));
            var94 = (unsigned int)((unsigned char)((var85 & 255)));
            *(signed char *)((&local_108[0] + var92)) = *(char *)((&local_108[0] + var94));
            *(signed char *)((&local_108[0] + var94)) = var93;
            var106 = ((unsigned int)((unsigned char)(*(char *)((&local_108[0] + (unsigned int)((unsigned char)(((((unsigned int)((unsigned char)(*(char *)((&local_108[0] + var92)))) & 255) + (var93 & 255)) & 255))))))) + (unsigned int)(((unsigned long)((unsigned int)((checksum << 5))) + checksum)));
            ret = (unsigned long)((unsigned int)(var106));
            var107 = (var3 - 1);
            var3 = (unsigned long)((unsigned int)(var107));
            checksum = (unsigned long)((unsigned int)(var106));
        } while (((unsigned long)((unsigned int)(var107)) != 0));
    }
    return ret;
}

gcc -O0

1/1
rc4_keystream_checksum pass 47 lines 
// glaurung: rc4_keystream_checksum @ 0x1119
uint32_t rc4_keystream_checksum(const uint8_t * arg0, int32_t arg1, int32_t arg2) {
    extern __attribute__((noreturn)) void __stack_chk_fail(void);
    unsigned int checksum;
    unsigned char j;
    int index;
    unsigned char local_119;
    unsigned char i;
    unsigned char swap;
    unsigned char local_110[256];
    long local_8;
    long ret;
    // x86-64 prologue: save rbp, frame 304 bytes
    local_8 = (long)(0x28);
    checksum = 0;
    j = 0;
    if ((((((arg0 == 0) || (((unsigned long)((unsigned int)(arg1)) == 0) | ((long)(arg1) < 0))) || ((((unsigned long)((unsigned int)(arg1)) == 16) | ((long)(arg1) < 16)) == 0)) || ((long)(arg2) < 0)) || (((unsigned long)((unsigned int)(arg2)) != 64) && (64 <= (long)(arg2))))) {
        ret = 0;
    } else {
        for (index = 0; ((((unsigned long)((unsigned int)(index)) == 255) | ((long)(index) < 255)) != 0); index++) {
            *(signed char *)((&local_110[0] + (long)(index))) = index;
        }
        for (index = 0; ((((unsigned long)((unsigned int)(index)) == 255) | ((long)(index) < 255)) != 0); index++) {
            j = ((unsigned char)(arg0[(long)((int)(((int)((((long long)(int)((((unsigned long)((long)((int)((unsigned long)((unsigned int)(index))))) >> 32) & 0xffffffff)) * (((long long)1) << 32)) + (unsigned int)((unsigned long)((unsigned int)(index)))) % (int)(arg1)))))]) + (unsigned long)((unsigned int)(((unsigned int)((unsigned char)(*(char *)((&local_110[0] + (long)(index))))) + (unsigned int)(j)))));
            local_119 = *(char *)((&local_110[0] + (long)(index)));
            *(signed char *)((&local_110[0] + (long)(index))) = *(char *)((&local_110[0] + (long)(j)));
            *(signed char *)((&local_110[0] + (long)(j))) = local_119;
        }
        i = 0;
        j = 0;
        for (index = 0; (index < arg2); index++) {
            i = (i + 1);
            ret = (unsigned int)((unsigned char)(*(char *)((&local_110[0] + (long)(i)))));
            j = (j + ret);
            swap = *(char *)((&local_110[0] + (long)(i)));
            *(signed char *)((&local_110[0] + (long)(i))) = *(char *)((&local_110[0] + (long)(j)));
            *(signed char *)((&local_110[0] + (long)(j))) = swap;
            checksum = ((unsigned int)((unsigned char)(((unsigned int)((unsigned char)(*(char *)((&local_110[0] + (long)((int)((unsigned char)(((unsigned long)((unsigned int)(((unsigned int)((unsigned char)(*(char *)((&local_110[0] + (long)(j))))) + (unsigned int)((unsigned char)(*(char *)((&local_110[0] + (long)(i)))))))) & 255)))))))) & 255))) + (unsigned int)(((unsigned long)((unsigned int)((checksum << 5))) + checksum)));
        }
        ret = (unsigned long)(checksum);
    }
    if ((local_8 != 0x28)) {
        __stack_chk_fail();
    }
    // x86-64 epilogue: restore rbp
    return ret;
}

gcc -O2

1/1
rc4_keystream_checksum pass 96 lines 
// glaurung: rc4_keystream_checksum @ 0x1120
uint32_t rc4_keystream_checksum(const uint8_t * arg0, int32_t arg1, int32_t arg2) {
    extern __attribute__((noreturn)) void __stack_chk_fail(void);
    int index;
    unsigned char swap;
    unsigned int checksum;
    unsigned char i;
    unsigned char j;
    long local_10;
    unsigned char local_118[256];
    long ret;
    long var12;
    long var13;
    long var14;
    long var15;
    long var21;
    long var22;
    int var24;
    int var32;
    int var33;
    long var36;
    long var39;
    long var42;
    long var43;
    int var44;
    int var45;
    int var46;
    int var57;
    long var7;
    long var8;
    long var9;
    local_10 = (long)(0x28);
    if (((unsigned long)(15) < (unsigned long)((unsigned long)((unsigned int)((arg1 - 1)))))) {
        L_1156: ;
        ret = 0;
    } else {
        var7 = (long)arg0;
        if ((arg0 == 0)) {
            goto L_1156;
        } else {
            var8 = (unsigned long)((unsigned int)(arg2));
            if (((unsigned long)((unsigned long)((unsigned int)(arg2))) <= (unsigned long)(64))) {
                var9 = (unsigned long)((unsigned int)(arg1));
                var12 = (long)(&local_118[0]);
                var13 = 0;
                do {
                    *(signed char *)((&local_118[0] + var13)) = var13;
                    var14 = (var13 + 1);
                    var13 = var14;
                } while ((var14 != 256));
                var15 = (long)(&local_118[0]);
                index = 0;
                var21 = 0;
                do {
                    var22 = (unsigned long)((unsigned int)(index));
                    swap = (unsigned int)((unsigned char)(*(char *)((var15))));
                    var24 = (index + 1);
                    index = (unsigned long)((unsigned int)(var24));
                    var15 = (var15 + 1);
                    var32 = (var21 + (unsigned int)(((unsigned int)((unsigned char)(*(char *)((var7 + (long)((int)(((int)((((long long)(int)((((unsigned long)((long)((int)(var22))) >> 32) & 0xffffffff)) * (((long long)1) << 32)) + (unsigned int)(var22)) % (int)(var9))))))))) + (unsigned int)(swap))));
                    var21 = (unsigned long)((unsigned int)(var32));
                    var33 = (unsigned int)((unsigned char)(((unsigned long)((unsigned int)(var32)) & 255)));
                    *(signed char *)((var15 - 0x1)) = *(char *)((&local_118[0] + var33));
                    *(signed char *)((&local_118[0] + var33)) = swap;
                } while (((unsigned long)((unsigned int)(var24)) != 256));
                if (((unsigned long)((unsigned int)(var8)) == 0)) {
                    goto L_1156;
                } else {
                    var36 = (long)((&local_118[0] + 1));
                    var39 = ((var12 + (unsigned long)((unsigned int)((var8 - 1)))) + 2);
                    var42 = 0;
                    var43 = 0;
                    do {
                        var44 = (unsigned int)((unsigned char)(*(char *)((var36))));
                        var36 = (var36 + 1);
                        var45 = (var43 + var44);
                        var43 = (unsigned long)((unsigned int)(var45));
                        var46 = (unsigned int)((unsigned char)(((unsigned long)((unsigned int)(var45)) & 255)));
                        *(signed char *)((var36 - 0x1)) = *(char *)((&local_118[0] + var46));
                        *(signed char *)((&local_118[0] + var46)) = var44;
                        var57 = ((unsigned int)((var42 + (unsigned long)((unsigned int)(((unsigned long)((unsigned int)(var42)) << 5))))) + (unsigned int)((unsigned char)(*(char *)((&local_118[0] + (unsigned int)((unsigned char)((((var44 & 255) + *(char *)((var36 - 0x1))) & 255))))))));
                        ret = (unsigned long)((unsigned int)(var57));
                        var42 = (unsigned long)((unsigned int)(var57));
                    } while ((var39 != var36));
                }
            } else {
                goto L_1156;
            }
        }
    }
    if ((local_10 != 0x28)) {
        __stack_chk_fail();
    }
    // x86-64 epilogue: tear down frame
    return ret;
}

← 213 fixtures