Is Tokenizer::tokenize() thread-safe

paul · December 14, 2023, 6:29pm

is the tokerizer thread-safe?

OpenNMT/Tokenizer/blob/master/src/Tokenizer.cc#L502


      
                                   std::vector<std::string>& words,
                                   std::vector<std::vector<std::string> >& features,
                                   std::unordered_map<std::string, size_t>* alphabets,
                                   bool training) const
          {
            std::vector<Token> annotated_tokens;
            tokenize(text, annotated_tokens, alphabets, training);
            finalize_tokens(annotated_tokens, words, features);
          }
          
          void Tokenizer::tokenize(const std::string& text,
                                   std::vector<Token>& annotated_tokens,
                                   std::unordered_map<std::string, size_t>* alphabets,
                                   bool training) const
          {
            if (text.empty())
              return;
          
            annotated_tokens.reserve(text.size());
          
            switch (_options.mode)