/************************************************************** * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * *************************************************************/ #include #include #include #if (defined(_WIN32) || defined(_MSDOS) || defined(__IBMC__)) #include #else #include #endif #include "cpp.h" /* * lexical FSM encoding * when in state state, and one of the characters * in ch arrives, enter nextstate. * States >= S_SELF are either final, or at least require special action. * In 'fsm' there is a line for each state X charset X nextstate. * List chars that overwrite previous entries later (e.g. C_ALPH * can be overridden by '_' by a later entry; and C_XX is the * the universal set, and should always be first. * States above S_SELF are represented in the big table as negative values. * S_SELF and S_SELFB encode the resulting token type in the upper bits. * These actions differ in that S_SELF doesn't have a lookahead char, * S_SELFB does. * * The encoding is blown out into a big table for time-efficiency. * Entries have * nextstate: 6 bits; ?\ marker: 1 bit; tokentype: 9 bits. */ #define MAXSTATE 32 #define ACT(tok,act) ((tok<<7)+act) #define QBSBIT 0100 #define GETACT(st) ((st>>7)&0x1ff) /* character classes */ #define C_WS 1 #define C_ALPH 2 #define C_NUM 3 #define C_EOF 4 #define C_XX 5 enum state { START = 0, NUM1, NUM2, NUM3, ID1, ST1, ST2, ST3, COM1, COM2, COM3, COM4, CC1, CC2, WS1, PLUS1, MINUS1, STAR1, SLASH1, PCT1, SHARP1, CIRC1, GT1, GT2, LT1, LT2, OR1, AND1, ASG1, NOT1, DOTS1, S_SELF = MAXSTATE, S_SELFB, S_EOF, S_NL, S_EOFSTR, S_STNL, S_COMNL, S_EOFCOM, S_COMMENT, S_EOB, S_WS, S_NAME }; int tottok; int tokkind[256]; struct fsm { int state; /* if in this state */ uchar ch[4]; /* and see one of these characters */ int nextstate; /* enter this state if +ve */ }; /*const*/ struct fsm fsm[] = { /* start state */ {START, {C_XX}, ACT(UNCLASS, S_SELF)}, {START, {' ', '\t', '\v'}, WS1}, {START, {C_NUM}, NUM1}, {START, {'.'}, NUM3}, {START, {C_ALPH}, ID1}, {START, {'L'}, ST1}, {START, {'"'}, ST2}, {START, {'\''}, CC1}, {START, {'/'}, COM1}, {START, {EOFC}, S_EOF}, {START, {'\n'}, S_NL}, {START, {'-'}, MINUS1}, {START, {'+'}, PLUS1}, {START, {'<'}, LT1}, {START, {'>'}, GT1}, {START, {'='}, ASG1}, {START, {'!'}, NOT1}, {START, {'&'}, AND1}, {START, {'|'}, OR1}, {START, {'#'}, SHARP1}, {START, {'%'}, PCT1}, {START, {'['}, ACT(SBRA, S_SELF)}, {START, {']'}, ACT(SKET, S_SELF)}, {START, {'('}, ACT(LP, S_SELF)}, {START, {')'}, ACT(RP, S_SELF)}, {START, {'*'}, STAR1}, {START, {','}, ACT(COMMA, S_SELF)}, {START, {'?'}, ACT(QUEST, S_SELF)}, {START, {':'}, ACT(COLON, S_SELF)}, {START, {';'}, ACT(SEMIC, S_SELF)}, {START, {'{'}, ACT(CBRA, S_SELF)}, {START, {'}'}, ACT(CKET, S_SELF)}, {START, {'~'}, ACT(TILDE, S_SELF)}, {START, {'^'}, CIRC1}, /* saw a digit */ {NUM1, {C_XX}, ACT(NUMBER, S_SELFB)}, {NUM1, {C_NUM, C_ALPH, '.'}, NUM1}, {NUM1, {'E', 'e'}, NUM2}, {NUM1, {'_'}, ACT(NUMBER, S_SELFB)}, /* saw possible start of exponent, digits-e */ {NUM2, {C_XX}, ACT(NUMBER, S_SELFB)}, {NUM2, {'+', '-'}, NUM1}, {NUM2, {C_NUM, C_ALPH}, NUM1}, {NUM2, {'_'}, ACT(NUMBER, S_SELFB)}, /* saw a '.', which could be a number or an operator */ {NUM3, {C_XX}, ACT(DOT, S_SELFB)}, {NUM3, {'.'}, DOTS1}, {NUM3, {C_NUM}, NUM1}, {DOTS1, {C_XX}, ACT(UNCLASS, S_SELFB)}, {DOTS1, {C_NUM}, NUM1}, {DOTS1, {'.'}, ACT(ELLIPS, S_SELF)}, /* saw a letter or _ */ {ID1, {C_XX}, ACT(NAME, S_NAME)}, {ID1, {C_ALPH, C_NUM}, ID1}, /* saw L (start of wide string?) */ {ST1, {C_XX}, ACT(NAME, S_NAME)}, {ST1, {C_ALPH, C_NUM}, ID1}, {ST1, {'"'}, ST2}, {ST1, {'\''}, CC1}, /* saw " beginning string */ {ST2, {C_XX}, ST2}, {ST2, {'"'}, ACT(STRING, S_SELF)}, {ST2, {'\\'}, ST3}, {ST2, {'\n'}, S_STNL}, {ST2, {EOFC}, S_EOFSTR}, /* saw \ in string */ {ST3, {C_XX}, ST2}, {ST3, {'\n'}, S_STNL}, {ST3, {EOFC}, S_EOFSTR}, /* saw ' beginning character const */ {CC1, {C_XX}, CC1}, {CC1, {'\''}, ACT(CCON, S_SELF)}, {CC1, {'\\'}, CC2}, {CC1, {'\n'}, S_STNL}, {CC1, {EOFC}, S_EOFSTR}, /* saw \ in ccon */ {CC2, {C_XX}, CC1}, {CC2, {'\n'}, S_STNL}, {CC2, {EOFC}, S_EOFSTR}, /* saw /, perhaps start of comment */ {COM1, {C_XX}, ACT(SLASH, S_SELFB)}, {COM1, {'='}, ACT(ASSLASH, S_SELF)}, {COM1, {'*'}, COM2}, {COM1, {'/'}, COM4}, /* saw / followed by *, start of comment */ {COM2, {C_XX}, COM2}, {COM2, {'\n'}, S_COMNL}, {COM2, {'*'}, COM3}, {COM2, {EOFC}, S_EOFCOM}, /* saw the * possibly ending a comment */ {COM3, {C_XX}, COM2}, {COM3, {'\n'}, S_COMNL}, {COM3, {'*'}, COM3}, {COM3, {'/'}, S_COMMENT}, /* // comment */ {COM4, {C_XX}, COM4}, {COM4, {'\n'}, S_NL}, {COM4, {EOFC}, S_EOFCOM}, /* saw white space, eat it up */ {WS1, {C_XX}, S_WS}, {WS1, {'\t', '\v', ' '}, WS1}, /* saw -, check --, -=, -> */ {MINUS1, {C_XX}, ACT(MINUS, S_SELFB)}, {MINUS1, {'-'}, ACT(MMINUS, S_SELF)}, {MINUS1, {'='}, ACT(ASMINUS, S_SELF)}, {MINUS1, {'>'}, ACT(ARROW, S_SELF)}, /* saw +, check ++, += */ {PLUS1, {C_XX}, ACT(PLUS, S_SELFB)}, {PLUS1, {'+'}, ACT(PPLUS, S_SELF)}, {PLUS1, {'='}, ACT(ASPLUS, S_SELF)}, /* saw <, check <<, <<=, <= */ {LT1, {C_XX}, ACT(LT, S_SELFB)}, {LT1, {'<'}, LT2}, {LT1, {'='}, ACT(LEQ, S_SELF)}, {LT2, {C_XX}, ACT(LSH, S_SELFB)}, {LT2, {'='}, ACT(ASLSH, S_SELF)}, /* saw >, check >>, >>=, >= */ {GT1, {C_XX}, ACT(GT, S_SELFB)}, {GT1, {'>'}, GT2}, {GT1, {'='}, ACT(GEQ, S_SELF)}, {GT2, {C_XX}, ACT(RSH, S_SELFB)}, {GT2, {'='}, ACT(ASRSH, S_SELF)}, /* = */ {ASG1, {C_XX}, ACT(ASGN, S_SELFB)}, {ASG1, {'='}, ACT(EQ, S_SELF)}, /* ! */ {NOT1, {C_XX}, ACT(NOT, S_SELFB)}, {NOT1, {'='}, ACT(NEQ, S_SELF)}, /* & */ {AND1, {C_XX}, ACT(AND, S_SELFB)}, {AND1, {'&'}, ACT(LAND, S_SELF)}, {AND1, {'='}, ACT(ASAND, S_SELF)}, /* | */ {OR1, {C_XX}, ACT(OR, S_SELFB)}, {OR1, {'|'}, ACT(LOR, S_SELF)}, {OR1, {'='}, ACT(ASOR, S_SELF)}, /* # */ {SHARP1, {C_XX}, ACT(SHARP, S_SELFB)}, {SHARP1, {'#'}, ACT(DSHARP, S_SELF)}, /* % */ {PCT1, {C_XX}, ACT(PCT, S_SELFB)}, {PCT1, {'='}, ACT(ASPCT, S_SELF)}, /* * */ {STAR1, {C_XX}, ACT(STAR, S_SELFB)}, {STAR1, {'='}, ACT(ASSTAR, S_SELF)}, /* ^ */ {CIRC1, {C_XX}, ACT(CIRC, S_SELFB)}, {CIRC1, {'='}, ACT(ASCIRC, S_SELF)}, {-1, "", 0} }; /* first index is char, second is state */ /* increase #states to power of 2 to encourage use of shift */ short bigfsm[256][MAXSTATE]; void expandlex(void) { /* const */ struct fsm *fp; int i, j, nstate; for (fp = fsm; fp->state >= 0; fp++) { for (i = 0; fp->ch[i]; i++) { nstate = fp->nextstate; if (nstate >= S_SELF) nstate = ~nstate; switch (fp->ch[i]) { case C_XX: /* random characters */ for (j = 0; j < 256; j++) bigfsm[j][fp->state] = (short) nstate; continue; case C_ALPH: for (j = 0; j < 256; j++) #ifdef S390 if( isalpha( j ) || (j == '_') ) #else if (('a' <= j && j <= 'z') || ('A' <= j && j <= 'Z') || j == '_') #endif bigfsm[j][fp->state] = (short) nstate; continue; case C_NUM: for (j = '0'; j <= '9'; j++) bigfsm[j][fp->state] = (short) nstate; continue; default: bigfsm[fp->ch[i]][fp->state] = (short) nstate; } } } /* * install special cases for ? (trigraphs), \ (splicing), runes, and * EOB */ for (i = 0; i < MAXSTATE; i++) { for (j = 0; j < 0xFF; j++) if (j == '?' || j == '\\' || j == '\n' || j == '\r') { if (bigfsm[j][i] > 0) bigfsm[j][i] = ~bigfsm[j][i]; bigfsm[j][i] &= ~QBSBIT; } bigfsm[EOB][i] = ~S_EOB; if (bigfsm[EOFC][i] >= 0) bigfsm[EOFC][i] = ~S_EOF; } } void fixlex(void) { /* do C++ comments? */ if ((Cplusplus == 0) || (Cflag != 0)) bigfsm['/'][COM1] = bigfsm['x'][COM1]; } /* * fill in a row of tokens from input, terminated by NL or END * First token is put at trp->lp. * Reset is non-zero when the input buffer can be "rewound." * The value is a flag indicating that possible macros have * been seen in the row. */ int gettokens(Tokenrow * trp, int reset) { register int c, state, oldstate; register uchar *ip; register Token *tp, *maxp; int runelen; Source *s = cursource; int nmac = 0; tp = trp->lp; ip = s->inp; if (reset) { s->lineinc = 0; if (ip >= s->inl) { /* nothing in buffer */ s->inl = s->inb; fillbuf(s); ip = s->inp = s->inb; } else if (ip >= s->inb + (3 * INS / 4)) { memmove(s->inb, ip, 4 + s->inl - ip); s->inl = s->inb + (s->inl - ip); ip = s->inp = s->inb; } } maxp = &trp->bp[trp->max]; runelen = 1; for (;;) { continue2: if (tp >= maxp) { trp->lp = tp; tp = growtokenrow(trp); maxp = &trp->bp[trp->max]; } tp->type = UNCLASS; tp->t = ip; tp->wslen = 0; tp->flag = 0; state = START; for (;;) { oldstate = state; c = *ip; if ((state = bigfsm[c][state]) >= 0) { ip += runelen; runelen = 1; continue; } state = ~state; reswitch: switch (state & 0177) { case S_SELF: ip += runelen; runelen = 1; case S_SELFB: tp->type = (unsigned char) GETACT(state); tp->len = ip - tp->t; tp++; goto continue2; case S_NAME: /* like S_SELFB but with nmac check */ tp->type = NAME; tp->len = ip - tp->t; nmac |= quicklook(tp->t[0], tp->len > 1 ? tp->t[1] : 0); tp++; goto continue2; case S_WS: tp->wslen = ip - tp->t; tp->t = ip; state = START; continue; default: if ((state & QBSBIT) == 0) { ip += runelen; runelen = 1; continue; } state &= ~QBSBIT; s->inp = ip; if (c == '\n') { while (s->inp + 1 >= s->inl && fillbuf(s) != EOF); if (s->inp[1] == '\r') { memmove(s->inp + 1, s->inp + 2, s->inl - s->inp + 2); s->inl -= 1; } goto reswitch; } if (c == '\r') { while (s->inp + 1 >= s->inl && fillbuf(s) != EOF); if (s->inp[1] == '\n') { memmove(s->inp, s->inp + 1, s->inl - s->inp + 1); s->inl -= 1; } else *s->inp = '\n'; state = oldstate; continue; } if (c == '?') { /* check trigraph */ if (trigraph(s)) { state = oldstate; continue; } goto reswitch; } if (c == '\\') { /* line-folding */ if (foldline(s)) { s->lineinc++; state = oldstate; continue; } goto reswitch; } error(WARNING, "Lexical botch in cpp"); ip += runelen; runelen = 1; continue; case S_EOB: s->inp = ip; fillbuf(cursource); state = oldstate; continue; case S_EOF: tp->type = END; tp->len = 0; s->inp = ip; if (tp != trp->bp && (tp - 1)->type != NL && cursource->fd != -1) error(WARNING, "No newline at end of file"); trp->lp = tp + 1; return nmac; case S_STNL: error(ERROR, "Unterminated string or char const"); case S_NL: tp->t = ip; tp->type = NL; tp->len = 1; tp->wslen = 0; s->lineinc++; s->inp = ip + 1; trp->lp = tp + 1; return nmac; case S_EOFSTR: error(FATAL, "EOF in string or char constant"); break; case S_COMNL: s->lineinc++; state = COM2; ip += runelen; runelen = 1; continue; case S_EOFCOM: error(WARNING, "EOF inside comment"); --ip; case S_COMMENT: if (!Cflag) { tp->t = ++ip; tp->t[-1] = ' '; tp->wslen = 1; state = START; continue; } else { runelen = 1; s->lineinc = 0;; tp->type = COMMENT; tp->flag |= XTWS; } } break; } ip += runelen; runelen = 1; tp->len = ip - tp->t; tp++; } } /* have seen ?; handle the trigraph it starts (if any) else 0 */ int trigraph(Source * s) { uchar c; while (s->inp + 2 >= s->inl && fillbuf(s) != EOF); ; if (s->inp[1] != '?') return 0; c = 0; switch (s->inp[2]) { case '=': c = '#'; break; case '(': c = '['; break; case '/': c = '\\'; break; case ')': c = ']'; break; case '\'': c = '^'; break; case '<': c = '{'; break; case '!': c = '|'; break; case '>': c = '}'; break; case '-': c = '~'; break; } if (c) { *s->inp = c; memmove(s->inp + 1, s->inp + 3, s->inl - s->inp + 2); s->inl -= 2; } return c; } int foldline(Source * s) { int n = 1; /* skip pending wihite spaces */ while ((s->inp[n] == ' ') || (s->inp[n] == '\t')) { n++; if ((s->inp + n >= s->inl) && (fillbuf(s) == EOF)) break; } /* refill buffer */ while (s->inp + (n + 1) >= s->inl && fillbuf(s) != EOF); /* skip DOS line ends */ if (((s->inp[n] == '\r') && (s->inp[n+1] == '\n')) || ((s->inp[n] == '\n') && (s->inp[n+1] == '\r'))) n++; if ((s->inp[n] == '\n') || (s->inp[n] == '\r')) { memmove(s->inp, s->inp + n + 1, s->inl - s->inp + n + 2); s->inl -= n + 1; return 1; } return 0; } int fillbuf(Source * s) { int n; if (s->fd < 0 || (n = read(s->fd, (char *) s->inl, INS / 8)) <= 0) n = 0; s->inl += n; s->inl[0] = s->inl[1] = s->inl[2] = s->inl[3] = EOB; if (n == 0) { s->inl[0] = s->inl[1] = s->inl[2] = s->inl[3] = EOFC; return EOF; } return 0; } /* * Push down to new source of characters. * If fd>0 and str==NULL, then from a file `name'; * if fd==-1 and str, then from the string. */ Source * setsource(char *name, int path, int fd, char *str, int wrap) { Source *s = new(Source); int len; s->line = 1; s->lineinc = 0; s->fd = fd; s->filename = name; s->next = cursource; s->ifdepth = 0; s->pathdepth = path; s->wrap = wrap; cursource = s; if (s->wrap) genwrap(0); /* slop at right for EOB */ if (str) { len = strlen(str); s->inb = domalloc(len + 4); s->inp = s->inb; strncpy((char *) s->inp, str, len); } else { s->inb = domalloc(INS + 4); s->inp = s->inb; len = 0; } s->inl = s->inp + len; s->inl[0] = s->inl[1] = EOB; return s; } void unsetsource(void) { Source *s = cursource; if (s->wrap) genwrap(1); if (s->fd >= 0) { close(s->fd); dofree(s->inb); } cursource = s->next; dofree(s); }