/*

Compiling: gcc -O2 -o benchtokenize benchtokenize.c
Running:   ./benchtokenize "  aaaaa bbbbbbb  cccc"

*/

#include <stdio.h>
#include <string.h>

typedef struct token_s {
    char *value;
    size_t length;
} token_t;

#define MAX_TOKENS 6

#if 0

static size_t tokenize_command(char *command, token_t *tokens, const size_t max_tokens)  {
    char *cp;
    char *value = NULL;
    size_t length = 0;
    size_t ntokens = 0;

    cp = command;
    while(*cp != '\0' && ntokens < max_tokens - 1) {
        if(*cp == ' ') {
            // If we've accumulated a token, this is the end of it.
            if(length > 0) {
                tokens[ntokens].value = value;
                tokens[ntokens].length = length;
                ntokens++;
                length = 0;
                value = NULL;
            }
            *cp = '\0';
        } else {
            if(length == 0) {
                value = cp;
            }
            length++;
        }
        cp++;
    }

    if(ntokens < max_tokens - 1 && length > 0) {
        tokens[ntokens].value = value;
        tokens[ntokens].length = length;
        ntokens++;
    }

    /*
     * If we scanned the whole string, the terminal value pointer is null,
     * otherwise it is the first unprocessed character.
     */
    tokens[ntokens].value =  *cp == '\0' ? NULL : cp;
    tokens[ntokens].length = 0;
    ntokens++;

    return ntokens;
}


#else

static size_t tokenize_command(char *command, token_t *tokens, const size_t max_tokens) {
    char *s, *e;
    size_t ntokens = 0;

    for (s = e = command; ntokens < max_tokens - 1; ++e) {
        if (*e == ' ') {
            if (s != e) {
                tokens[ntokens].value = s;
                tokens[ntokens].length = e - s;
                ntokens++;
                *e = '\0';
            }
            s = e + 1;
        }
        else if (*e == '\0') {
            if (s != e) {
                tokens[ntokens].value = s;
                tokens[ntokens].length = e - s;
                ntokens++;
            }

            break; /* string end */
        }
    }

    /*
     * If we scanned the whole string, the terminal value pointer is null,
     * otherwise it is the first unprocessed character.
     */
    tokens[ntokens].value =  *e == '\0' ? NULL : e;
    tokens[ntokens].length = 0;
    ntokens++;

    return ntokens;
}

#endif

int 
main (int argc, char *argv[])
{
    token_t tokens[MAX_TOKENS];
    size_t n;
    char *str;
    int i;

    printf ("tokenizing: %s\n", argv[1]);

    str = strdup (argv[1]);

    for (i = 0; i < 100000000; i += 1) {
        n = tokenize_command (str, tokens, MAX_TOKENS);
        strcpy (str, argv[1]);
    }

    printf ("%d tokens:\n", n);
    for (i = 0; i < n; i += 1) {
        printf("value: %s, len: %d\n", tokens[i].value,tokens[i].length);
    }

    return 0;
}