达芬奇密码

一道windows的32位逆向题，涉及动态解密，大数运算，佩尔方程通解问题。

首先运行程序，发现字符串Wrong!,使用IDA搜索字符串搜不到，说明应该加密了，在程序中动态解密。

在程序运行时搜索内存，得如下内容

确定程序的关键函数401EA0，在IDA中反编译得如下代码

int __thiscall sub_401EA0(CWnd *this)
{
  CWnd *v1; // esi
  int v2; // eax
  WCHAR String; // [esp+Ch] [ebp-310h]
  char v5; // [esp+Eh] [ebp-30Eh]
  char v6; // [esp+20Ch] [ebp-110h]
  char v7; // [esp+20Dh] [ebp-10Fh]
  DWORD v8; // [esp+30Ch] [ebp-10h]
  CWnd *v9; // [esp+310h] [ebp-Ch]
  int v10; // [esp+314h] [ebp-8h]
  DWORD flOldProtect; // [esp+318h] [ebp-4h]

  v1 = this;
  v9 = this;
  String = 0;
  memset(&v5, 0, 0x1FEu);
  v6 = 0;
  memset(&v7, 0, 0xFFu);
  CWnd::GetDlgItemTextW(v1, 1000, &String, 20);
  if ( wcslen(&String) == 16 ) //flag长度时16字节
  {
    v2 = 0;
    while ( !(*(&String + v2) & 0xFF00) ) //输入是ASCII字符
    {
      *(&v6 + v2) = *((_BYTE *)&String + 2 * v2);//输入在内存中是宽字符，赋值给V6,V6是字节数组
      if ( ++v2 >= 16 )
      {
        v8 = 64;
        flOldProtect = 0;
        VirtualProtect(sub_4010E0, 0xD17u, 0x40u, &flOldProtect);
        if ( GetLastError() )
          return CWnd::MessageBoxW(v1, L"Wrong!", 0, 0);
        qmemcpy(sub_4010E0, &byte_5647B8, 0x330u);//前面修改内存权限，这里把数据写入函数sub_4010E0，是动态解密的
        VirtualProtect(sub_4010E0, 0xD17u, flOldProtect, &v8);
        if ( !GetLastError() )
        {
          v10 = 0;
          v10 = sub_4010E0(&v6);
          if ( v10 == 1 )
            return CWnd::MessageBoxW(v9, L"Congratulations! You are right!", 0, 0);
        }
        v1 = v9;
        return CWnd::MessageBoxW(v1, L"Wrong!", 0, 0);
      }
    }
  }
  return CWnd::MessageBoxW(v1, L"Wrong!", 0, 0);
}

在判断代码中发现FLAG是长度位16字节的ASCII码字符数组，并且函数sub_4010E0是动态解密的，且此函数返回值为1则成功。
此时需要转储内存函数sub4010E0
使用lordPE把此进程完整转存为dump.exe

随后拖入IDA中反编译函数4010E0进行分析，如下
在下面的函数中涉及到大数运算，大数表示字节数组，此时把大数看成256进制的数就很容易理解其中的代码。

signed int __cdecl sub_4010E0(int a1)
{
  signed int v1; // eax
  char v2; // cl
  signed int v3; // ecx
  signed int v4; // eax
  signed int v5; // eax
  signed int v6; // esi
  signed int v7; // ecx
  __int16 v8; // dx
  char *v9; // edi
  __int16 v10; // ax
  signed int v11; // eax
  signed int v12; // ecx
  unsigned __int16 v13; // bx
  signed int v14; // esi
  signed int v15; // ecx
  __int16 v16; // dx
  char *v17; // edi
  __int16 v18; // ax
  signed int v19; // eax
  signed int v20; // ecx
  unsigned __int16 v21; // bx
  unsigned int v22; // eax
  signed int v23; // ecx
  unsigned __int16 v24; // dx
  char v25; // dl
  signed int v26; // eax
  __int16 v27; // si
  int v28; // eax
  int v30; // [esp+8h] [ebp-90h]
  int v31; // [esp+Ch] [ebp-8Ch]
  int v32; // [esp+10h] [ebp-88h]
  int v33; // [esp+14h] [ebp-84h]
  int v34; // [esp+18h] [ebp-80h]
  int v35; // [esp+1Ch] [ebp-7Ch]
  int v36; // [esp+20h] [ebp-78h]
  int v37; // [esp+24h] [ebp-74h]
  int v38; // [esp+28h] [ebp-70h]
  int v39; // [esp+2Ch] [ebp-6Ch]
  int v40; // [esp+30h] [ebp-68h]
  int v41; // [esp+34h] [ebp-64h]
  int v42; // [esp+38h] [ebp-60h]
  int v43; // [esp+3Ch] [ebp-5Ch]
  int v44; // [esp+40h] [ebp-58h]
  int v45; // [esp+44h] [ebp-54h]
  int v46; // [esp+48h] [ebp-50h]
  int v47; // [esp+4Ch] [ebp-4Ch]
  int v48; // [esp+50h] [ebp-48h]
  int v49; // [esp+54h] [ebp-44h]
  char v50; // [esp+58h] [ebp-40h]
  int v51; // [esp+5Ch] [ebp-3Ch]
  int v52; // [esp+60h] [ebp-38h]
  int v53; // [esp+64h] [ebp-34h]
  int v54; // [esp+68h] [ebp-30h]
  char v55; // [esp+6Ch] [ebp-2Ch]
  int v56; // [esp+70h] [ebp-28h]
  int v57; // [esp+74h] [ebp-24h]
  int v58; // [esp+78h] [ebp-20h]
  int v59; // [esp+7Ch] [ebp-1Ch]
  char v60; // [esp+80h] [ebp-18h]
  int v61; // [esp+84h] [ebp-14h]
  int v62; // [esp+88h] [ebp-10h]
  int v63; // [esp+8Ch] [ebp-Ch]
  int v64; // [esp+90h] [ebp-8h]
  char v65; // [esp+94h] [ebp-4h]

  v31 = 1684969601;
  v1 = 0;
  v30 = -477325802;
  v32 = -2116286332;
  v33 = 1330138558;
  v34 = 0;
  v35 = 0;
  v36 = 0;
  v37 = 0;
  v38 = 0;
  v39 = 0;
  v44 = 0;
  v45 = 0;
  do
  {
    v2 = *((_BYTE *)&v32 + v1) ^ *(_BYTE *)(a1 + v1 + 8);
    *((_BYTE *)&v38 + v1) = *((_BYTE *)&v30 + v1) ^ *((_BYTE *)&v30 + v1 + a1 - (_DWORD)&v30);
    *((_BYTE *)&v44 + v1++) = v2;
  }
/*上面通过把输入值的前8字节和后8字节分别与固定值异或后得V44和V38*/
  while ( v1 < 8 );
  v30 = 0;
  v56 = 0;
  v57 = 0;
  v58 = 0;
  v59 = 0;
  v60 = 0;
  v61 = 0;
  v62 = 0;
  v63 = 0;
  v64 = 0;
  v65 = 0;
  v46 = 0;
  v47 = 0;
  v48 = 0;
  v49 = 0;
  v50 = 0;
  v51 = 0;
  v52 = 0;
  v53 = 0;
  v54 = 0;
  v55 = 0;
  v31 = 0;
  v32 = 0;
  v33 = 0;
  LOBYTE(v34) = 0;
  v3 = 8;
  LOBYTE(v30) = 8;
  v4 = 7;
  do
  {
    if ( *((_BYTE *)&v38 + v4) )
      break;
    --v3;
    --v4;
  }
  while ( v4 >= 0 );
  if ( v3 == 8 )
  {
    v5 = 7;
    do
    {
      if ( *((_BYTE *)&v44 + v5) )
        break;
      --v3;
      --v5;
    }
    while ( v5 >= 0 );
    if ( v3 == 8 && !(v39 & 0xF0000000) )
    {
      v6 = 0;
      do
      {
        v40 = 0;
        v41 = 0;
        v42 = 0;
        v43 = 0;
        v7 = 0;
        v8 = *((unsigned __int8 *)&v38 + v6);
        v9 = (char *)&v40 + v6;
        do
        {
          v10 = *((unsigned __int8 *)&v42 + v6) + v8 * *((unsigned __int8 *)&v38 + v7);
          v9[v7] = *((_BYTE *)&v42 + v6) + v8 * *((_BYTE *)&v38 + v7);
          ++v7;
          *((_BYTE *)&v42 + v6) = HIBYTE(v10);//取进位
        }
        while ( v7 < 8 );
        LOBYTE(v11) = 0;
        v12 = 0;
        do
        {
          v13 = (char)v11 + *((unsigned __int8 *)&v56 + v12 + v6) + (unsigned __int8)v9[v12];
          *((_BYTE *)&v56 + v12++ + v6) = v13;
          v11 = (signed int)v13 >> 8;
        }
        while ( v12 < 9 );
        ++v6;
      }
      while ( v6 < 8 );//V56 = V38 * V38
      v14 = 0;
      do
      {
        v40 = 0;
        v41 = 0;
        v42 = 0;
        v43 = 0;
        v15 = 0;
        v16 = *((unsigned __int8 *)&v44 + v14);
        v17 = (char *)&v40 + v14;
        do
        {
          v18 = *((unsigned __int8 *)&v42 + v14) + v16 * *((unsigned __int8 *)&v44 + v15);
          v17[v15] = *((_BYTE *)&v42 + v14) + v16 * *((_BYTE *)&v44 + v15);
          ++v15;
          *((_BYTE *)&v42 + v14) = HIBYTE(v18);
        }
        while ( v15 < 8 );
        LOBYTE(v19) = 0;
        v20 = 0;
        do
        {
          v21 = (char)v19 + *((unsigned __int8 *)&v61 + v20 + v14) + (unsigned __int8)v17[v20];
          *((_BYTE *)&v61 + v20++ + v14) = v21;
          v19 = (signed int)v21 >> 8;
        }
        while ( v20 < 9 );
        ++v14;
      }
      while ( v14 < 8 );//V61=V44 * V44
      LOBYTE(v22) = v50;
      v23 = 0;
      do
      {
        v24 = (unsigned __int8)v22 + 7 * *((unsigned __int8 *)&v61 + v23);
        *((_BYTE *)&v46 + v23++) = v24;
        v22 = (unsigned int)v24 >> 8;
      }
      while ( v23 < 17 );V46 = 7 * V61
      v50 = HIBYTE(v24);
      v25 = 0;
      v26 = 0;
      do
      {
        v27 = *((unsigned __int8 *)&v56 + v26) - *((unsigned __int8 *)&v46 + v26) - v25;
        *((_BYTE *)&v51 + v26) = v27;
        if ( v27 < 0 )
          v25 = 1;
        ++v26;
      }
      while ( v26 < 17 );//V56 = V38*V38 - 7 * V44 * V44
      if ( !v25 )
      {
        v28 = 0;
        while ( *((_BYTE *)&v51 + v28) == *((_BYTE *)&v30 + v28) )
        {
          if ( ++v28 >= 17 )
            return 1; //V56 == 8 返回1
        }
      }
    }
  }
  return 0;
}

在上面代码中发现V38和V44是方程 V38^2 - 7 V44^2 = 8的解，且满足
2^56<=V38<2^60
2^56<=V44<2^60
调研后得知此方程是佩尔方程，且此时有无数组解，于是只需找到满足上面条件的一组解即可。
此方程通解的递推关系为
Xn = 8 Xn-1 + 21 Yn-1
yn = 3 Xn-1 + 8 * Yn-1（n>=2)
X1 = 6
Y1 = 2
求解代码如下

#include <stdio.h>
int main()
{
    unsigned long long a = 6;
    unsigned long long b = 2;
    while(a < 0x0100000000000000 || b < 0x0100000000000000){
        unsigned long long temp = a;
        a = 8 * a + 21 * b;
        b = 3 * temp + 8 * b;
    }
    printf("%llu %llu\n",a,b);
return 0;
}

解得
V38=385044246406735194
V44=145533045678356702
继续回推可得FLAG，相关代码如下

#include <stdio.h>
int main()
{
    int a = -477325802;
    int b = 1684969601;
    int c = -2116286332;
    int d = 1330138558;
    unsigned long long p = 385044246406735194;
    unsigned long long q = 145533045678356702;
    char FLAG[17];
    FLAG[16] = 0;
    for(int i = 0;i < 8; i++) {
       FLAG[i] = (*((char *)&a + i)) ^ (*((char *)&p + i));
       FLAG[i + 8] = (*((char *)&c + i)) ^ (*((char *)&q + i));
    }
    printf("%s\n",FLAG);
    return 0;
}

最后得到FLAG为L3mZ2k9aZ0a36DMM

[看雪官方培训]《安卓高级研修班(网课)》9月班开始招生！顶尖技术、挑战极限、工资翻倍！