dlib C++ Library - tokenizer.cpp

    // Copyright (C) 2005  Davis E. King (davisking@users.sourceforge.net)
    // License: Boost Software License   See LICENSE.txt for the full license.

    #include <string>
    #include <sstream>

    #include <dlib/tokenizer.h>
    #include "tester.h"

    namespace
    {
        using namespace test;
        using namespace std;
        using namespace dlib;

        logger dlog("test.tokenizer");

        template <
            typename tok
            >
        void tokenizer_kernel_test (
        )
        /*!
            requires
                - tok is an implementation of tokenizer_kernel_abstract.h
            ensures
                - runs tests on tok for compliance with the specs
        !*/
        {

            print_spinner();

            tok test;

            DLIB_CASSERT(test.numbers() == "0123456789","");
            DLIB_CASSERT(test.uppercase_letters() == "ABCDEFGHIJKLMNOPQRSTUVWXYZ","");
            DLIB_CASSERT(test.lowercase_letters() == "abcdefghijklmnopqrstuvwxyz","");

            DLIB_CASSERT(test.get_identifier_body() == "_" + test.lowercase_letters() +
                         test.uppercase_letters() + test.numbers(),"");
            DLIB_CASSERT(test.get_identifier_head() == "_" + test.lowercase_letters() +
                         test.uppercase_letters(),"");

            DLIB_CASSERT(test.stream_is_set() == false,"");
            test.clear();
            DLIB_CASSERT(test.stream_is_set() == false,"");

            DLIB_CASSERT(test.get_identifier_body() == "_" + test.lowercase_letters() +
                         test.uppercase_letters() + test.numbers(),"");
            DLIB_CASSERT(test.get_identifier_head() == "_" + test.lowercase_letters() +
                         test.uppercase_letters(),"");

            tok test2;

            ostringstream sout;
            istringstream sin;
            test2.set_stream(sin);

            DLIB_CASSERT(test2.stream_is_set(),"");
            DLIB_CASSERT(&test2.get_stream() == &sin,"");

            int type;
            string token;

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::END_OF_FILE,"");
            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::END_OF_FILE,"");
            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::END_OF_FILE,"");

            sin.clear();
            sin.str("  The cat 123asdf1234 ._ \n test.");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == "  ","");

            DLIB_CASSERT(test2.peek_type() == tok::IDENTIFIER,"");
            DLIB_CASSERT(test2.peek_token() == "The","");
            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "The","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "cat","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::NUMBER,"");
            DLIB_CASSERT(token == "123","token: " << token);

            DLIB_CASSERT(test2.peek_type() == tok::IDENTIFIER,"");
            DLIB_CASSERT(test2.peek_token() == "asdf1234","");
            DLIB_CASSERT(test2.peek_type() == tok::IDENTIFIER,"");
            DLIB_CASSERT(test2.peek_token() == "asdf1234","");
            DLIB_CASSERT(test2.peek_type() == tok::IDENTIFIER,"");
            DLIB_CASSERT(test2.peek_token() == "asdf1234","");
            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "asdf1234","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::CHAR,"");
            DLIB_CASSERT(token == ".","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "_","");

            DLIB_CASSERT(test2.peek_type() == tok::WHITE_SPACE,"");
            DLIB_CASSERT(test2.peek_token() == " ","token: \"" << token << "\"" <<
                         "\ntoken size: " << (unsigned int)token.size());

            swap(test,test2);

            DLIB_CASSERT(test2.stream_is_set() == false,"");

            DLIB_CASSERT(test.peek_type() == tok::WHITE_SPACE,"");
            DLIB_CASSERT(test.peek_token() == " ","token: \"" << token << "\"" <<
                         "\ntoken size: " << (unsigned int)token.size());
            test.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","token: \"" << token << "\"" <<
                         "\ntoken size: " << (unsigned int)token.size());

            test.get_token(type,token);
            DLIB_CASSERT(type == tok::END_OF_LINE,"token: " << token);
            DLIB_CASSERT(token == "\n","token: " << token);

            swap(test,test2);
            DLIB_CASSERT(test.stream_is_set() == false,"");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "test","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::CHAR,"");
            DLIB_CASSERT(token == ".","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::END_OF_FILE,"");

            test2.set_identifier_token("_" + test.uppercase_letters() +
                                       test.lowercase_letters(),test.numbers() + "_" + test.uppercase_letters()
                                       +test.lowercase_letters());

            sin.clear();
            sin.str("  The cat 123asdf1234 ._ \n\r test.");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == "  ","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "The","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "cat","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::NUMBER,"");
            DLIB_CASSERT(token == "123","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "asdf1234","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::CHAR,"");
            DLIB_CASSERT(token == ".","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "_","");

            swap(test,test2);

            DLIB_CASSERT(test2.stream_is_set() == false,"");

            test.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","token: \"" << token << "\"" <<
                         "\ntoken size: " << (unsigned int)token.size());

            test.get_token(type,token);
            DLIB_CASSERT(type == tok::END_OF_LINE,"token: " << token);
            DLIB_CASSERT(token == "\n","token: " << token);

            swap(test,test2);
            DLIB_CASSERT(test.stream_is_set() == false,"");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == "\r ","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "test","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::CHAR,"");
            DLIB_CASSERT(token == ".","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::END_OF_FILE,"");

            test2.set_identifier_token(test.uppercase_letters() +
                                       test.lowercase_letters(),test.numbers() + test.uppercase_letters()
                                       +test.lowercase_letters());

            sin.clear();
            sin.str("  The cat 123as_df1234 ._ \n test.");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == "  ","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "The","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "cat","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::NUMBER,"");
            DLIB_CASSERT(token == "123","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "as","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::CHAR,"");
            DLIB_CASSERT(token == "_","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "df1234","");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::CHAR,"");
            DLIB_CASSERT(token == ".","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::CHAR,"");
            DLIB_CASSERT(token == "_","");

            swap(test,test2);

            DLIB_CASSERT(test2.stream_is_set() == false,"");

            test.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","token: \"" << token << "\"" <<
                         "\ntoken size: " << (unsigned int)token.size());

            test.get_token(type,token);
            DLIB_CASSERT(type == tok::END_OF_LINE,"token: " << token);
            DLIB_CASSERT(token == "\n","token: " << token);

            swap(test,test2);
            DLIB_CASSERT(test.stream_is_set() == false,"");

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::WHITE_SPACE,"");
            DLIB_CASSERT(token == " ","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::IDENTIFIER,"");
            DLIB_CASSERT(token == "test","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::CHAR,"");
            DLIB_CASSERT(token == ".","token: " << token);

            test2.get_token(type,token);
            DLIB_CASSERT(type == tok::END_OF_FILE,"");

        }

        class tokenizer_tester : public tester
        {
        public:
            tokenizer_tester (
            ) :
                tester ("test_tokenizer",
                        "Runs tests on the tokenizer component.")
            {}

            void perform_test (
            )
            {
                dlog << LINFO << "testing kernel_1a";
                tokenizer_kernel_test<tokenizer::kernel_1a>  ();
                dlog << LINFO << "testing kernel_1a_c";
                tokenizer_kernel_test<tokenizer::kernel_1a_c>();
            }
        } a;

    }