summaryrefslogtreecommitdiffstats
path: root/indexlib/tokenizer.h
blob: 9916df66ad12edb3ef827e4669c0dda819f24337 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
#ifndef LPC_TOKENIZER_H1118429480_INCLUDE_GUARD_
#define LPC_TOKENIZER_H1118429480_INCLUDE_GUARD_

#include <vector>
#include <string>
#include <memory>
#include <assert.h>

namespace indexlib { namespace detail {

class tokenizer {
	public:
		virtual ~tokenizer() { }
		std::vector<std::string> string_to_words( const char* str ) {
			assert( str );
			return do_string_to_words( str );
		}
	
	private:
		virtual std::vector<std::string> do_string_to_words( const char* ) = 0;
};

std::unique_ptr<tokenizer> get_tokenizer( std::string );
}}



#endif /* LPC_TOKENIZER_H1118429480_INCLUDE_GUARD_ */