1#!/usr/bin/env ruby
 2
 3WHITESPACE = Regexp.new('[[:blank:]]+')
 4SPLITTABLES = [';', '(', ')']
 5
 6def tokenize(code)
 7  pattern = Regexp.new("[^#{Regexp.escape(SPLITTABLES.join)}]+")
 8  output = []
 9  tokens = code.chomp.strip.split(WHITESPACE)
10  tokens.each do |token|
11    prefix, stem, suffix = token.partition(pattern)
12    output << prefix.split('') unless prefix.empty?
13    output << stem unless stem.empty?
14    output << suffix.split('') unless suffix.empty?
15  end
16
17  output.flatten
18end
19
20
21code = ARGV[0]
22tokens = tokenize(code)
23
24puts "Input: #{code.inspect}"
25puts "Tokens: #{tokens.inspect}"