type Tokenizer_Prop :
    record
        OldSent : string
        TokCurs : int
    end record

var Tokenizer : Tokenizer_Prop

fcn Tokenize (Sent, token : string) : string
    if Sent ~= Tokenizer.OldSent then
        Tokenizer.TokCurs := 1
        Tokenizer.OldSent := Sent
    end if

    var tmp : string := ""
    var ext : boolean := false

    for j : 1 .. length (token)
        if Sent (Tokenizer.TokCurs) = token (j) then
            Tokenizer.TokCurs += 1
        end if
    end for

    for i : Tokenizer.TokCurs .. length (Sent)
        for j : 1 .. length (token)
            if Sent (i) = token (j) then
                Tokenizer.TokCurs := i + 1
                ext := true
            else
                tmp += Sent (i)
            end if
            exit when ext
        end for
        exit when ext
    end for

    result tmp
end Tokenize

var sentence : string := "This is a test sentence"
var word : array 1 .. 5 of string

for i : 1 .. 5
    word (i) := Tokenize (sentence, " ")
end for

% I don't know why someone would put two similar loops right after each other,
% but this is just to show  the results

for i : 1 .. 5
    put word (i) + " "..     % reconstructs the sentence
end for

% This would output "This is a test sentence "

View.Set ("text:80;120;title:StringTokenizer Test")

var myStr : ^String
var tokenizer : ^StringTokenizer

new myStr
new tokenizer

myStr ->
    create ("ASCII stands for American Standard Code for Information Interchange. Computers can only understand numbers, so an ASCII code is the numerical representation of a character such as 'a' or '@' or an action of some sort.")

tokenizer -> createBasic (myStr, " ")

put "The string is '"..
myStr->output()
put "'"
put "#Tokens = ", tokenizer -> countTokens ()
var token : int := 1
loop
    exit when not tokenizer -> hasMoreTokens ()
    put "Token #",token," = '"..
    tokenizer -> nextToken () -> output ()
    put "'"
    token+=1
end loop