Test case : Tokens using Forth

variable tokcount

\ Count backslashes at end of string.
: count-escapes ( adr len -- n)
  tuck
  begin  dup
  while
    1-  2dup + c@  '\ <>
  until  1+
  then  nip  - ;

\ Look for ending fence.
: terminates  ( ch escapeable? --)
  begin
    over parse   2dup type   3 pick emit
    count-escapes
    over and 0=
  until  2drop ;

: toks= ( adr1 len1 adr2 len2 -- true | adr1 len1 false )
  2over compare  0= if  2drop  -1   else  0  then ;

\ We may have a fence that starts a "token" ended by another fence.
: fence ( adr1 len1 adr2 len2 char escapeable? weight)
  { ch esc weight }
  toks=
  if  space  ch esc terminates  weight tokcount +!
     ( Now exit calling word.) r> drop
  then ;

: handle-token  ( cadr len -- )
  2dup type
  s" ("         ')   0 0 fence   \  ( )
  s" \"         10   0 0 fence   \  \
  s\" .\""      '"   0 2 fence   \  ." "
  s\" s\""      '"   0 1 fence   \  s" "
  s\" abort\""  '"   0 2 fence   \  abort" "
  s\" .\\\""    '"   1 2 fence   \  .\" "
  s\" s\\\""    '"   1 1 fence   \  s\" "
  1 tokcount +!
  2drop ;

: count-tokens
  0 tokcount !
  begin
    parse-word dup
  while
    handle-token  cr
  repeat
  2drop    tokcount @ . ." tokens." ;

stdin slurp-fid  ' count-tokens execute-parsing  bye