* Implementation for calculating language statistics Impement saving code language statistics to database Implement rendering langauge stats Add primary laguage to show in repository list Implement repository stats indexer queue Add indexer test Refactor to use queue module * Do not timeout for queues
15 lines
412 B
Go
15 lines
412 B
Go
// +build flex
|
|
|
|
package tokenizer
|
|
|
|
import "github.com/src-d/enry/v2/internal/tokenizer/flex"
|
|
|
|
// Tokenize returns lexical tokens from content. The tokens returned match what
|
|
// the Linguist library returns. At most the first ByteLimit bytes of content are tokenized.
|
|
func Tokenize(content []byte) []string {
|
|
if len(content) > ByteLimit {
|
|
content = content[:ByteLimit]
|
|
}
|
|
|
|
return flex.TokenizeFlex(content)
|
|
}
|