sentence_tokenizer.hpp

#ifndef SENTENCE_TOKENIZER_HDR
#define SENTENCE_TOKENIZER_HDR

#include <string>
#include <vector>
#include "concordia/common/config.hpp"
#include "concordia/tokenized_sentence.hpp"
#include "concordia/regex_rule.hpp"
#include "concordia/concordia_config.hpp"
#include "concordia/concordia_exception.hpp"
#include <boost/shared_ptr.hpp>
#include <boost/filesystem.hpp>


class SentenceTokenizer {
public:
    explicit SentenceTokenizer(boost::shared_ptr<ConcordiaConfig> config)
                                                 throw(ConcordiaException);

    virtual ~SentenceTokenizer();

    TokenizedSentence tokenize(const std::string & sentence,
                               bool byWhitespace = false);

private:
    void _createNeRules(std::string & namedEntitiesPath);

    void _createHtmlTagsRule(std::string & htmlTagsPath);

    boost::shared_ptr<RegexRule> _getMultipleRegexRule(
                                             std::string filePath,
                                             char annotationType,
                                             std::string value,
                                             bool wholeWord = false);

    std::vector<RegexRule> _namedEntities;

    boost::shared_ptr<RegexRule> _htmlTags;

    bool _stopWordsEnabled;

    boost::shared_ptr<RegexRule> _stopWords;
};

#endif