LICENSE
README.md
pyproject.toml
setup.cfg
setup.py
curated_tokenizers/__init__.py
curated_tokenizers/_bbpe.pxd
curated_tokenizers/_bbpe.pyx
curated_tokenizers/_spp.pxd
curated_tokenizers/_spp.pyx
curated_tokenizers/_wordpiece.pxd
curated_tokenizers/_wordpiece.pyx
curated_tokenizers/config.h
curated_tokenizers/merges.cc
curated_tokenizers/merges.hh
curated_tokenizers/py.typed
curated_tokenizers/util.hh
curated_tokenizers/wordpiece.cc
curated_tokenizers/wordpiece.hh
curated_tokenizers.egg-info/PKG-INFO
curated_tokenizers.egg-info/SOURCES.txt
curated_tokenizers.egg-info/dependency_links.txt
curated_tokenizers.egg-info/requires.txt
curated_tokenizers.egg-info/top_level.txt
curated_tokenizers.egg-info/zip-safe
curated_tokenizers/tests/__init__.py
curated_tokenizers/tests/incorrect-merges.txt
curated_tokenizers/tests/robbert-merges-1000.txt
curated_tokenizers/tests/robbert-vocab-1000.json
curated_tokenizers/tests/test_bbpe_processor.py
curated_tokenizers/tests/test_sp.py
curated_tokenizers/tests/test_word_piece_processor.py
curated_tokenizers/tests/toy-word-pieces.txt
curated_tokenizers/tests/toy.model
sentencepiece/python/once.h
sentencepiece/src/bpe_model.cc
sentencepiece/src/bpe_model.h
sentencepiece/src/bpe_model_test.cc
sentencepiece/src/bpe_model_trainer.cc
sentencepiece/src/bpe_model_trainer.h
sentencepiece/src/bpe_model_trainer_test.cc
sentencepiece/src/builder.cc
sentencepiece/src/builder.h
sentencepiece/src/builder_test.cc
sentencepiece/src/char_model.cc
sentencepiece/src/char_model.h
sentencepiece/src/char_model_test.cc
sentencepiece/src/char_model_trainer.cc
sentencepiece/src/char_model_trainer.h
sentencepiece/src/char_model_trainer_test.cc
sentencepiece/src/common.h
sentencepiece/src/compile_charsmap_main.cc
sentencepiece/src/error.cc
sentencepiece/src/filesystem.cc
sentencepiece/src/filesystem.h
sentencepiece/src/filesystem_test.cc
sentencepiece/src/freelist.h
sentencepiece/src/freelist_test.cc
sentencepiece/src/init.h
sentencepiece/src/init_test.cc
sentencepiece/src/model_factory.cc
sentencepiece/src/model_factory.h
sentencepiece/src/model_factory_test.cc
sentencepiece/src/model_interface.cc
sentencepiece/src/model_interface.h
sentencepiece/src/model_interface_test.cc
sentencepiece/src/normalization_rule.h
sentencepiece/src/normalizer.cc
sentencepiece/src/normalizer.h
sentencepiece/src/normalizer_test.cc
sentencepiece/src/pretokenizer_for_training.cc
sentencepiece/src/pretokenizer_for_training.h
sentencepiece/src/pretokenizer_for_training_test.cc
sentencepiece/src/sentencepiece_processor.cc
sentencepiece/src/sentencepiece_processor.h
sentencepiece/src/sentencepiece_processor_test.cc
sentencepiece/src/sentencepiece_trainer.cc
sentencepiece/src/sentencepiece_trainer.h
sentencepiece/src/sentencepiece_trainer_test.cc
sentencepiece/src/spec_parser.h
sentencepiece/src/spm_decode_main.cc
sentencepiece/src/spm_encode_main.cc
sentencepiece/src/spm_export_vocab_main.cc
sentencepiece/src/spm_normalize_main.cc
sentencepiece/src/spm_train_main.cc
sentencepiece/src/test_main.cc
sentencepiece/src/testharness.cc
sentencepiece/src/testharness.h
sentencepiece/src/trainer_factory.cc
sentencepiece/src/trainer_factory.h
sentencepiece/src/trainer_factory_test.cc
sentencepiece/src/trainer_interface.cc
sentencepiece/src/trainer_interface.h
sentencepiece/src/trainer_interface_test.cc
sentencepiece/src/unicode_script.cc
sentencepiece/src/unicode_script.h
sentencepiece/src/unicode_script_map.h
sentencepiece/src/unicode_script_test.cc
sentencepiece/src/unigram_model.cc
sentencepiece/src/unigram_model.h
sentencepiece/src/unigram_model_test.cc
sentencepiece/src/unigram_model_trainer.cc
sentencepiece/src/unigram_model_trainer.h
sentencepiece/src/unigram_model_trainer_test.cc
sentencepiece/src/util.cc
sentencepiece/src/util.h
sentencepiece/src/util_test.cc
sentencepiece/src/word_model.cc
sentencepiece/src/word_model.h
sentencepiece/src/word_model_test.cc
sentencepiece/src/word_model_trainer.cc
sentencepiece/src/word_model_trainer.h
sentencepiece/src/word_model_trainer_test.cc
sentencepiece/src/builtin_pb/sentencepiece.pb.cc
sentencepiece/src/builtin_pb/sentencepiece.pb.h
sentencepiece/src/builtin_pb/sentencepiece_model.pb.cc
sentencepiece/src/builtin_pb/sentencepiece_model.pb.h
sentencepiece/third_party/absl/container/flat_hash_map.h
sentencepiece/third_party/absl/container/flat_hash_set.h
sentencepiece/third_party/absl/flags/flag.cc
sentencepiece/third_party/absl/flags/flag.h
sentencepiece/third_party/absl/flags/parse.h
sentencepiece/third_party/absl/memory/memory.h
sentencepiece/third_party/absl/strings/ascii.h
sentencepiece/third_party/absl/strings/match.h
sentencepiece/third_party/absl/strings/numbers.h
sentencepiece/third_party/absl/strings/str_cat.h
sentencepiece/third_party/absl/strings/str_format.h
sentencepiece/third_party/absl/strings/str_join.h
sentencepiece/third_party/absl/strings/str_replace.h
sentencepiece/third_party/absl/strings/str_split.h
sentencepiece/third_party/absl/strings/string_view.cc
sentencepiece/third_party/absl/strings/string_view.h
sentencepiece/third_party/absl/strings/strip.h
sentencepiece/third_party/darts_clone/darts.h
sentencepiece/third_party/protobuf-lite/arena.cc
sentencepiece/third_party/protobuf-lite/arenastring.cc
sentencepiece/third_party/protobuf-lite/bytestream.cc
sentencepiece/third_party/protobuf-lite/coded_stream.cc
sentencepiece/third_party/protobuf-lite/common.cc
sentencepiece/third_party/protobuf-lite/extension_set.cc
sentencepiece/third_party/protobuf-lite/generated_enum_util.cc
sentencepiece/third_party/protobuf-lite/generated_message_table_driven_lite.cc
sentencepiece/third_party/protobuf-lite/generated_message_util.cc
sentencepiece/third_party/protobuf-lite/implicit_weak_message.cc
sentencepiece/third_party/protobuf-lite/int128.cc
sentencepiece/third_party/protobuf-lite/io_win32.cc
sentencepiece/third_party/protobuf-lite/message_lite.cc
sentencepiece/third_party/protobuf-lite/parse_context.cc
sentencepiece/third_party/protobuf-lite/repeated_field.cc
sentencepiece/third_party/protobuf-lite/status.cc
sentencepiece/third_party/protobuf-lite/statusor.cc
sentencepiece/third_party/protobuf-lite/stringpiece.cc
sentencepiece/third_party/protobuf-lite/stringprintf.cc
sentencepiece/third_party/protobuf-lite/structurally_valid.cc
sentencepiece/third_party/protobuf-lite/strutil.cc
sentencepiece/third_party/protobuf-lite/time.cc
sentencepiece/third_party/protobuf-lite/wire_format_lite.cc
sentencepiece/third_party/protobuf-lite/zero_copy_stream.cc
sentencepiece/third_party/protobuf-lite/zero_copy_stream_impl.cc
sentencepiece/third_party/protobuf-lite/zero_copy_stream_impl_lite.cc
sentencepiece/third_party/protobuf-lite/google/protobuf/any.h
sentencepiece/third_party/protobuf-lite/google/protobuf/arena.h
sentencepiece/third_party/protobuf-lite/google/protobuf/arena_impl.h
sentencepiece/third_party/protobuf-lite/google/protobuf/arenastring.h
sentencepiece/third_party/protobuf-lite/google/protobuf/descriptor.h
sentencepiece/third_party/protobuf-lite/google/protobuf/extension_set.h
sentencepiece/third_party/protobuf-lite/google/protobuf/extension_set_inl.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_enum_reflection.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_enum_util.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/generated_message_util.h
sentencepiece/third_party/protobuf-lite/google/protobuf/has_bits.h
sentencepiece/third_party/protobuf-lite/google/protobuf/implicit_weak_message.h
sentencepiece/third_party/protobuf-lite/google/protobuf/map.h
sentencepiece/third_party/protobuf-lite/google/protobuf/map_entry_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/map_field_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/map_type_handler.h
sentencepiece/third_party/protobuf-lite/google/protobuf/message_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/metadata_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/parse_context.h
sentencepiece/third_party/protobuf-lite/google/protobuf/port.h
sentencepiece/third_party/protobuf-lite/google/protobuf/port_def.inc
sentencepiece/third_party/protobuf-lite/google/protobuf/port_undef.inc
sentencepiece/third_party/protobuf-lite/google/protobuf/repeated_field.h
sentencepiece/third_party/protobuf-lite/google/protobuf/unknown_field_set.h
sentencepiece/third_party/protobuf-lite/google/protobuf/wire_format_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/coded_stream.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/io_win32.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl.h
sentencepiece/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/bytestream.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/callback.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/casts.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/common.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/hash.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/int128.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/logging.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/macros.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/map_util.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/mutex.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/once.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/port.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/status.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/statusor.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/stl_util.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/strutil.h
sentencepiece/third_party/protobuf-lite/google/protobuf/stubs/time.h