I am using below code to parse the SQL queries. How do i take care of parsing for japanese/ chinese characters which are multibyte.
The complete code is available at ParserTokenManager.java Source File.

private int jjMoveStringLiteralDfa0_0()
{
switch(curChar)
{
case 33:
return jjStopAtPos(0, 170);
case 35:
return jjStopAtPos(0, 171);
case 37:
jjmatchedKind = 169;
return jjMoveStringLiteralDfa1_0(0x0L, 0x0L, 0xc0000000L);
case 40:
return jjStopAtPos(0, 154);
case 41:
return jjStopAtPos(0, 155);
case 42:
jjmatchedKind = 166;
return jjMoveStringLiteralDfa1_0(0x0L, 0x0L, 0x10000000000L);
case 43:
return jjStopAtPos(0, 163);
case 44:
return jjStopAtPos(0, 156);
case 45:
return jjStartNfaWithStates_0(0, 164, 0);
case 46:
jjmatchedKind = 151;
return jjMoveStringLiteralDfa1_0(0x0L, 0x0L, 0x1000400000000L);
case 47:
return jjStartNfaWithStates_0(0, 167, 3);
case 58:
jjmatchedKind = 150;
return jjMoveStringLiteralDfa1_0(0x0L, 0x0L, 0x20000000L);
case 59:
return jjStopAtPos(0, 153);
case 60:
jjmatchedKind = 173;
return jjMoveStringLiteralDfa1_0(0x0L, 0x0L, 0x100000000L);
case 61:
jjmatchedKind = 152;
return jjMoveStringLiteralDfa1_0(0x0L, 0x0L, 0x800000000000L);
case 62:
jjmatchedKind = 172;
return jjMoveStringLiteralDfa1_0(0x0L, 0x0L, 0x200000000L);
case 64:
return jjStopAtPos(0, 174);
case 65:
case 97:
return jjMoveStringLiteralDfa1_0(0x7e0L, 0x0L, 0x0L);
case 66:
case 98:
return jjMoveStringLiteralDfa1_0(0xf800L, 0x0L, 0x0L);
case 67:
case 99:
return jjMoveStringLiteralDfa1_0(0x3fff0000L, 0x0L, 0x0L);
case 68:
case 100:
return jjMoveStringLiteralDfa1_0(0x3fc0000000L, 0x0L, 0x0L);
case 69:
case 101:
return jjMoveStringLiteralDfa1_0(0x7fc000000000L, 0x0L, 0x0L);