标记化Java源代码

/**from NowJava.com - 时  代  Java**/

import java.io.FileReader;

import java.io.IOException;

import java.io.StreamTokenizer;


public class Main {

  public static void main(String[] args) throws Exception {

    try {

      FileReader rd = new FileReader("filename.java");

      StreamTokenizer st = new StreamTokenizer(rd);


      // Prepare the tokenizer for Java-style tokenizing rules

      st.parseNumbers();

      st.wordChars('_', '_');

      st.eolIsSignificant(true);


      // If whitespace is not to be discarded, make this call

      st.ordinaryChars(0, ' ');/*nowjava.com 提供*/


      // These calls caused comments to be discarded

      st.slashSlashComments(true);

      st.slashStarComments(true);


      int token = st.nextToken();

      while (token != StreamTokenizer.TT_EOF) {

        token = st.nextToken();

        switch (token) {

        case StreamTokenizer.TT_NUMBER:

          // A number was found; the value is in nval

          double num = st.nval;

          break;

        case StreamTokenizer.TT_WORD:

          // A word was found; the value is in sval

          String word = st.sval;

          break;

        case '"':

          // A double-quoted string was found; sval contains the contents

          String dquoteVal = st.sval;

          break;

        case '\'':

          // A single-quoted string was found; sval contains the contents

          String squoteVal = st.sval;

          break;

        case StreamTokenizer.TT_EOL:

          // End of line character found

          break;

        case StreamTokenizer.TT_EOF: