+sentencepiece

This commit is contained in:
Max Howell 2023-03-23 11:03:15 -04:00
parent 7f68885ffa
commit 3112bc4685

View file

@ -0,0 +1,31 @@
distributable:
url: https://github.com/google/sentencepiece/archive/v{{version}}.tar.gz
strip-components: 1
versions:
github: google/sentencepiece/releases/tags
ignore: '1.0.0' # whoops, who tagged this then?
build:
dependencies:
tea.xyz/gx/cc: c99
tea.xyz/gx/make: '*'
cmake.org: ^3
working-directory: build
script: |
cmake .. -DCMAKE_INSTALL_PREFIX="{{prefix}}" -DCMAKE_BUILD_TYPE=Release
make --jobs {{ hw.concurrency }} install
provides:
- bin/spm_decode
- bin/spm_encode
- bin/spm_export_vocab
- bin/spm_normalize
- bin/spm_train
test:
dependencies:
gnu.org/wget: '*'
script: |
wget https://raw.githubusercontent.com/google/sentencepiece/master/data/botchan.txt
spm_train --input=botchan.txt --model_prefix=m --vocab_size=1000