forgejo/modules/csv/csv.go
Richard Mahn 40c8451b7d
Properly determine CSV delimiter (#17459)
* Fixes #16558 CSV delimiter determiner

* Fixes #16558 - properly determine CSV delmiiter

* Moves quoteString to a new function

* Adds big test with lots of commas for tab delimited csv

* Adds comments

* Shortens the text of the test

* Removes single quotes from regexp as only double quotes need to be searched

* Fixes spelling

* Fixes check of length as it probalby will only be 1e4, not greater

* Makes sample size a const, properly removes truncated line

* Makes sample size a const, properly removes truncated line

* Fixes comment

* Fixes comment

* tests for FormatError() function

* Adds logic to find the limiter before or after a quoted value

* Simplifies regex

* Error tests

* Error tests

* Update modules/csv/csv.go

Co-authored-by: delvh <dev.lh@web.de>

* Update modules/csv/csv.go

Co-authored-by: delvh <dev.lh@web.de>

* Adds comments

* Update modules/csv/csv.go

Co-authored-by: delvh <dev.lh@web.de>

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: delvh <dev.lh@web.de>
2021-10-30 23:50:40 +08:00

148 lines
5.4 KiB
Go

// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package csv
import (
"bytes"
stdcsv "encoding/csv"
"io"
"path/filepath"
"regexp"
"strings"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/translation"
"code.gitea.io/gitea/modules/util"
)
const maxLines = 10
const guessSampleSize = 1e4 // 10k
// CreateReader creates a csv.Reader with the given delimiter.
func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
rd := stdcsv.NewReader(input)
rd.Comma = delimiter
if delimiter != '\t' && delimiter != ' ' {
// TrimLeadingSpace can't be true when delimiter is a tab or a space as the value for a column might be empty,
// thus would change `\t\t` to just `\t` or ` ` (two spaces) to just ` ` (single space)
rd.TrimLeadingSpace = true
}
return rd
}
// CreateReaderAndDetermineDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
// Reads at most guessSampleSize bytes.
func CreateReaderAndDetermineDelimiter(ctx *markup.RenderContext, rd io.Reader) (*stdcsv.Reader, error) {
var data = make([]byte, guessSampleSize)
size, err := util.ReadAtMost(rd, data)
if err != nil {
return nil, err
}
return CreateReader(
io.MultiReader(bytes.NewReader(data[:size]), rd),
determineDelimiter(ctx, data[:size]),
), nil
}
// determineDelimiter takes a RenderContext and if it isn't nil and the Filename has an extension that specifies the delimiter,
// it is used as the delimiter. Otherwise we call guessDelimiter with the data passed
func determineDelimiter(ctx *markup.RenderContext, data []byte) rune {
extension := ".csv"
if ctx != nil {
extension = strings.ToLower(filepath.Ext(ctx.Filename))
}
var delimiter rune
switch extension {
case ".tsv":
delimiter = '\t'
case ".psv":
delimiter = '|'
default:
delimiter = guessDelimiter(data)
}
return delimiter
}
// quoteRegexp follows the RFC-4180 CSV standard for when double-quotes are used to enclose fields, then a double-quote appearing inside a
// field must be escaped by preceding it with another double quote. https://www.ietf.org/rfc/rfc4180.txt
// This finds all quoted strings that have escaped quotes.
var quoteRegexp = regexp.MustCompile(`"[^"]*"`)
// removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can reliably have each row on one line
// (quoted strings often have new lines within the string)
func removeQuotedString(text string) string {
return quoteRegexp.ReplaceAllLiteralString(text, "")
}
// guessDelimiter takes up to maxLines of the CSV text, iterates through the possible delimiters, and sees if the CSV Reader reads it without throwing any errors.
// If more than one delimiter passes, the delimiter that results in the most columns is returned.
func guessDelimiter(data []byte) rune {
delimiter := guessFromBeforeAfterQuotes(data)
if delimiter != 0 {
return delimiter
}
// Removes quoted values so we don't have columns with new lines in them
text := removeQuotedString(string(data))
// Make the text just be maxLines or less, ignoring truncated lines
lines := strings.SplitN(text, "\n", maxLines+1) // Will contain at least one line, and if there are more than MaxLines, the last item holds the rest of the lines
if len(lines) > maxLines {
// If the length of lines is > maxLines we know we have the max number of lines, trim it to maxLines
lines = lines[:maxLines]
} else if len(lines) > 1 && len(data) >= guessSampleSize {
// Even with data >= guessSampleSize, we don't have maxLines + 1 (no extra lines, must have really long lines)
// thus the last line is probably have a truncated line. Drop the last line if len(lines) > 1
lines = lines[:len(lines)-1]
}
// Put lines back together as a string
text = strings.Join(lines, "\n")
delimiters := []rune{',', '\t', ';', '|', '@'}
validDelim := delimiters[0]
validDelimColCount := 0
for _, delim := range delimiters {
csvReader := stdcsv.NewReader(strings.NewReader(text))
csvReader.Comma = delim
if rows, err := csvReader.ReadAll(); err == nil && len(rows) > 0 && len(rows[0]) > validDelimColCount {
validDelim = delim
validDelimColCount = len(rows[0])
}
}
return validDelim
}
// FormatError converts csv errors into readable messages.
func FormatError(err error, locale translation.Locale) (string, error) {
if perr, ok := err.(*stdcsv.ParseError); ok {
if perr.Err == stdcsv.ErrFieldCount {
return locale.Tr("repo.error.csv.invalid_field_count", perr.Line), nil
}
return locale.Tr("repo.error.csv.unexpected", perr.Line, perr.Column), nil
}
return "", err
}
// Looks for possible delimiters right before or after (with spaces after the former) double quotes with closing quotes
var beforeAfterQuotes = regexp.MustCompile(`([,@\t;|]{0,1}) *(?:"[^"]*")+([,@\t;|]{0,1})`)
// guessFromBeforeAfterQuotes guesses the limiter by finding a double quote that has a valid delimiter before it and a closing quote,
// or a double quote with a closing quote and a valid delimiter after it
func guessFromBeforeAfterQuotes(data []byte) rune {
rs := beforeAfterQuotes.FindStringSubmatch(string(data)) // returns first match, or nil if none
if rs != nil {
if rs[1] != "" {
return rune(rs[1][0]) // delimiter found left of quoted string
} else if rs[2] != "" {
return rune(rs[2][0]) // delimiter found right of quoted string
}
}
return 0 // no match found
}