Contents

#ifndef __simple_tokeniser_h__
#define __simple_tokeniser_h__
#include "tokeniser.h"

namespace Preprocessing {
  namespace Text {
    
    class SimpleTokeniser : public Tokeniser {
    public:
      static const uint32_t file_mark = 'simt';
      uint32_t mark() { return file_mark; }
      
      SimpleTokeniser(TextPipeline *pipeline) : Tokeniser(pipeline) {}
      void tokenise(char *text);
    };
    
  }
}

#endif

Version data entries

8 entries across 8 versions & 1 rubygems

Version	Path
thera-0.0.8	lib/quarry/src/preprocessing/text/tokeniser/simple_tokeniser.h
thera-0.0.7	lib/quarry/src/preprocessing/text/tokeniser/simple_tokeniser.h
thera-0.0.6	lib/quarry/src/preprocessing/text/tokeniser/simple_tokeniser.h
thera-0.0.5	lib/quarry/src/preprocessing/text/tokeniser/simple_tokeniser.h
thera-0.0.4	lib/quarry/src/preprocessing/text/tokeniser/simple_tokeniser.h
thera-0.0.3	lib/quarry/src/preprocessing/text/tokeniser/simple_tokeniser.h
thera-0.0.2	lib/quarry/src/preprocessing/text/tokeniser/simple_tokeniser.h
thera-0.0.1	lib/quarry/src/preprocessing/text/tokeniser/simple_tokeniser.h