Tokenizer.java


Below is the syntax highlighted version of Tokenizer.java from §5.1 Formal Languages.


/******************************************************************************
 *  Compilation:  javac Tokenizer.java In.java
 *  Execution:    java Tokenizer
 *  
 *  Takes a string as input and tokenizes it into an array of substrings
 *  using a specified delimiter character.
 *
 *  % java StringTokenizer : http://www.cs.princeton.edu/introcs/datafiles/
 *
 *  See also java.util.StringTokenizer.
 *
 ******************************************************************************/

public class Tokenizer {

    public static void main(String[] args) {
        String s = "/this is/a test of the/class// and/all//last";
        char delimiter =  '/';

        // calculate number of delimiter characters
        int n = 0;
        for (int i = 0; i < s.length(); i++)
            if (s.charAt(i) == delimiter) n++;
        String[] tokens = new String[n+1];

        // parse n+1 tokens and store in an array
        int right = 0, left = 0;
        for (int i = 0; i < n; i++) {
            while (s.charAt(right) != delimiter)
                right++;
            tokens[i] = s.substring(left, right);
            right++;
            left = right;
        }
        tokens[n] = s.substring(right, s.length());

        // print results for testing
        for (int i = 0; i < tokens.length; i++)
            StdOut.println(i + ": " + tokens[i]);
    }

}


Copyright © 2000–2017, Robert Sedgewick and Kevin Wayne.
Last updated: Fri Oct 20 14:12:12 EDT 2017.