tokenized_sentence.hpp

#ifndef TOKENIZED_SENTENCE_HDR
#define TOKENIZED_SENTENCE_HDR

#include "concordia/common/config.hpp"
#include "concordia/token_annotation.hpp"
#include "concordia/word_map.hpp"

#include <boost/shared_ptr.hpp>
#include <string>
#include <vector>
#include <list>
#include <iostream>
#include <boost/foreach.hpp>



class TokenizedSentence {
public:
    explicit TokenizedSentence(std::string sentence);

    virtual ~TokenizedSentence();

    std::string getSentence() const {
        return _sentence;
    }

    std::string getOriginalSentence() const {
        return _originalSentence;
    }

    std::string getTokenizedSentence() const;

    std::list<TokenAnnotation> getAnnotations() const {
        return _tokenAnnotations;
    }

    std::vector<INDEX_CHARACTER_TYPE> getCodes() const {
        return _codes;
    }

    std::vector<TokenAnnotation> getTokens() const {
        return _tokens;
    }

    void generateHash(boost::shared_ptr<WordMap> wordMap);

    void generateTokens();

    void toLowerCase();

    void addAnnotations(std::vector<TokenAnnotation> annotations);

    friend std::ostream & operator << (std::ostream & o,
                          const TokenizedSentence & ts) {
        int index = 0;
        BOOST_FOREACH(TokenAnnotation token, ts.getAnnotations()) {
            o << "[" << token.getStart() << "," << token.getEnd() << "]["
              << token.getType() << "][" << token.getValue() <<"]";
            if (index < ts.getAnnotations().size() - 1) {
                o << " ";
            }
            index++;
        }
        return o;
    }


private:
    std::string _sentence;

    std::string _originalSentence;

    std::list<TokenAnnotation> _tokenAnnotations;

    std::vector<INDEX_CHARACTER_TYPE> _codes;

    std::vector<TokenAnnotation> _tokens;
};

#endif