Import CSV 3.0.2

This includes performance improvement especially writing. Writing is
about 2 times faster.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@66507 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

5 files changed, 240 insertions, 62 deletions

diff --git a/test/csv/test_csv_parsing.rb b/test/csv/test_csv_parsing.rb
index e65bbad92e..3fe1bd79e4 100755
--- a/test/csv/test_csv_parsing.rb
+++ b/test/csv/test_csv_parsing.rb
@@ -143,55 +143,52 @@ class TestCSV::Parsing < TestCSV
     end
   end
 
-  def test_malformed_csv
-    assert_raise(CSV::MalformedCSVError) do
+  def test_malformed_csv_cr_first_line
+    error = assert_raise(CSV::MalformedCSVError) do
       CSV.parse_line("1,2\r,3", row_sep: "\n")
     end
+    assert_equal("Unquoted fields do not allow \\r or \\n in line 1.",
+                 error.message)
+  end
 
-    bad_data = <<-CSV
+  def test_malformed_csv_cr_middle_line
+    csv = <<-CSV
 line,1,abc
 line,2,"def\nghi"
 
 line,4,some\rjunk
 line,5,jkl
     CSV
-    lines = bad_data.lines.to_a
-    assert_equal(6, lines.size)
-    assert_match(/\Aline,4/, lines.find { |l| l =~ /some\rjunk/ })
-
-    csv = CSV.new(bad_data)
-    begin
-      loop do
-        assert_not_nil(csv.shift)
-        assert_send([csv.lineno, :<, 4])
-      end
-    rescue CSV::MalformedCSVError
-      assert_equal( "Unquoted fields do not allow \\r or \\n in line 4.",
-                    $!.message )
+
+    error = assert_raise(CSV::MalformedCSVError) do
+      CSV.parse(csv)
     end
+    assert_equal("Unquoted fields do not allow \\r or \\n in line 4.",
+                 error.message)
+  end
 
-    assert_raise(CSV::MalformedCSVError) { CSV.parse_line('1,2,"3...') }
+  def test_malformed_csv_unclosed_quote
+    error = assert_raise(CSV::MalformedCSVError) do
+      CSV.parse_line('1,2,"3...')
+    end
+    assert_equal("Unclosed quoted field in line 1.",
+                 error.message)
+  end
 
-    bad_data = <<-CSV
+  def test_malformed_csv_illegal_quote_middle_line
+    csv = <<-CSV
 line,1,abc
 line,2,"def\nghi"
 
 line,4,8'10"
 line,5,jkl
     CSV
-    lines = bad_data.lines.to_a
-    assert_equal(6, lines.size)
-    assert_match(/\Aline,4/, lines.find { |l| l =~ /8'10"/ })
-
-    csv = CSV.new(bad_data)
-    begin
-      loop do
-        assert_not_nil(csv.shift)
-        assert_send([csv.lineno, :<, 4])
-      end
-    rescue CSV::MalformedCSVError
-      assert_equal("Illegal quoting in line 4.", $!.message)
+
+    error = assert_raise(CSV::MalformedCSVError) do
+      CSV.parse(csv)
     end
+    assert_equal("Illegal quoting in line 4.",
+                 error.message)
   end
 
   def test_the_parse_fails_fast_when_it_can_for_unquoted_fields
@@ -239,6 +236,24 @@ line,5,jkl
                  CSV.parse("a b  d", col_sep: " "))
   end
 
+  def test_row_sep_auto_cr
+    assert_equal([["a"]], CSV.parse("a\r"))
+  end
+
+  def test_row_sep_auto_lf
+    assert_equal([["a"]], CSV.parse("a\n"))
+  end
+
+  def test_row_sep_auto_cr_lf
+    assert_equal([["a"]], CSV.parse("a\r\n"))
+  end
+
+  def test_headers_empty_line
+    assert_equal(CSV::Table.new([CSV::Row.new(["header1"], [])],
+                                headers: ["header1"]),
+                 CSV.parse("\n", headers: "header1"))
+  end
+
   private
 
   def assert_parse_errors_out(*args)
diff --git a/test/csv/test_data_converters.rb b/test/csv/test_data_converters.rb
index 114049d66f..8b3163da18 100755
--- a/test/csv/test_data_converters.rb
+++ b/test/csv/test_data_converters.rb
@@ -243,23 +243,35 @@ class TestCSV::DataConverters < TestCSV
                   CSV.parse_line(@data, converters: [:numeric, @custom]) )
   end
 
-  def test_unconverted_fields
-    [ [ @data,
-        ["Numbers", :integer, 1, :float, 3.015],
-        %w{Numbers :integer 1 :float 3.015} ],
-      ["\n", Array.new, Array.new] ].each do |test, fields, unconverted|
-      row = nil
-      assert_nothing_raised(Exception) do
-        row = CSV.parse_line( test,
-                              converters:         [:numeric, @custom],
-                              unconverted_fields: true )
-      end
-      assert_not_nil(row)
-      assert_equal(fields, row)
-      assert_respond_to(row, :unconverted_fields)
-      assert_equal(unconverted, row.unconverted_fields)
-    end
+  def test_unconverted_fields_number
+    row = CSV.parse_line(@data,
+                         converters:         [:numeric, @custom],
+                         unconverted_fields: true)
+    assert_equal([
+                   ["Numbers", :integer, 1, :float, 3.015],
+                   ["Numbers", ":integer", "1", ":float", "3.015"],
+                 ],
+                 [
+                   row,
+                   row.unconverted_fields,
+                 ])
+  end
 
+  def test_unconverted_fields_empty_line
+    row = CSV.parse_line("\n",
+                         converters:         [:numeric, @custom],
+                         unconverted_fields: true)
+    assert_equal([
+                   [],
+                   [],
+                 ],
+                 [
+                   row,
+                   row.unconverted_fields,
+                 ])
+  end
+
+  def test_unconverted_fields
     data = <<-CSV
 first,second,third
 1,2,3
diff --git a/test/csv/test_features.rb b/test/csv/test_features.rb
index 45d937e037..53b513d0fa 100755
--- a/test/csv/test_features.rb
+++ b/test/csv/test_features.rb
@@ -58,26 +58,37 @@ line,4,jkl
   end
 
   def test_row_sep
-    assert_raise(CSV::MalformedCSVError) do
-        CSV.parse_line("1,2,3\n,4,5\r\n", row_sep: "\r\n")
+    error = assert_raise(CSV::MalformedCSVError) do
+      CSV.parse_line("1,2,3\n,4,5\r\n", row_sep: "\r\n")
     end
+    assert_equal("Unquoted fields do not allow \\r or \\n in line 1.",
+                 error.message)
     assert_equal( ["1", "2", "3\n", "4", "5"],
                   CSV.parse_line(%Q{1,2,"3\n",4,5\r\n}, row_sep: "\r\n"))
   end
 
   def test_quote_char
     TEST_CASES.each do |test_case|
-      assert_equal( test_case.last.map { |t| t.tr('"', "'") unless t.nil? },
-                    CSV.parse_line( test_case.first.tr('"', "'"),
-                                    quote_char: "'" ) )
+      assert_equal(test_case.last.map {|t| t.tr('"', "'") unless t.nil?},
+                   CSV.parse_line(test_case.first.tr('"', "'"),
+                                  quote_char: "'" ))
     end
   end
 
-  def test_bug_8405
+  def test_quote_char_special_regexp_char
     TEST_CASES.each do |test_case|
-      assert_equal( test_case.last.map { |t| t.tr('"', "|") unless t.nil? },
-                    CSV.parse_line( test_case.first.tr('"', "|"),
-                                    quote_char: "|" ) )
+      assert_equal(test_case.last.map {|t| t.tr('"', "|") unless t.nil?},
+                   CSV.parse_line(test_case.first.tr('"', "|"),
+                                  quote_char: "|"))
+    end
+  end
+
+  def test_quote_char_special_regexp_char_liberal_parsing
+    TEST_CASES.each do |test_case|
+      assert_equal(test_case.last.map {|t| t.tr('"', "|") unless t.nil?},
+                   CSV.parse_line(test_case.first.tr('"', "|"),
+                                  quote_char: "|",
+                                  liberal_parsing: true))
     end
   end
 
@@ -157,27 +168,68 @@ line,4,jkl
     assert_equal(3, count)
   end
 
-  def test_liberal_parsing
+  def test_liberal_parsing_middle_quote_start
     input = '"Johnson, Dwayne",Dwayne "The Rock" Johnson'
-    assert_raise(CSV::MalformedCSVError) do
+    error = assert_raise(CSV::MalformedCSVError) do
         CSV.parse_line(input)
     end
+    assert_equal("Illegal quoting in line 1.",
+                 error.message)
     assert_equal(["Johnson, Dwayne", 'Dwayne "The Rock" Johnson'],
                  CSV.parse_line(input, liberal_parsing: true))
+  end
 
+  def test_liberal_parsing_middle_quote_end
     input = '"quoted" field'
-    assert_raise(CSV::MalformedCSVError) do
+    error = assert_raise(CSV::MalformedCSVError) do
         CSV.parse_line(input)
     end
+    assert_equal("Do not allow except col_sep_split_separator " +
+                 "after quoted fields in line 1.",
+                 error.message)
     assert_equal(['"quoted" field'],
                  CSV.parse_line(input, liberal_parsing: true))
+  end
 
-    assert_raise(CSV::MalformedCSVError) do
+  def test_liberal_parsing_quote_after_column_separator
+    error = assert_raise(CSV::MalformedCSVError) do
       CSV.parse_line('is,this "three," or four,fields', liberal_parsing: true)
     end
+    assert_equal("Unclosed quoted field in line 1.",
+                 error.message)
+  end
 
+  def test_liberal_parsing_quote_before_column_separator
     assert_equal(["is", 'this "three', ' or four"', "fields"],
-      CSV.parse_line('is,this "three, or four",fields', liberal_parsing: true))
+                 CSV.parse_line('is,this "three, or four",fields',
+                                liberal_parsing: true))
+  end
+
+  def test_liberal_parsing_backslash_quote
+    assert_equal([
+                   "1",
+                   "\"Hamlet says, \\\"Seems",
+                   "\\\" madam! Nay it is; I know not \\\"seems.\\\"\"",
+                 ],
+                 CSV.parse_line('1,' +
+                                '"Hamlet says, \"Seems,' +
+                                '\" madam! Nay it is; I know not \"seems.\""',
+                                liberal_parsing: true))
+  end
+
+  def test_liberal_parsing_space_quote
+    input = <<~CSV
+      Los Angeles,   34°03'N,    118°15'W
+      New York City, 40°42'46"N, 74°00'21"W
+      Paris,         48°51'24"N, 2°21'03"E
+    CSV
+    assert_equal(
+                 [
+                   ["Los Angeles", "   34°03'N", "    118°15'W"],
+                   ["New York City", " 40°42'46\"N", " 74°00'21\"W"],
+                   ["Paris", "         48°51'24\"N", " 2°21'03\"E"],
+                 ],
+                 CSV.parse(input, liberal_parsing: true))
   end
 
   def test_csv_behavior_readers
@@ -338,9 +390,31 @@ line,4,jkl
 
   def test_requires_skip_lines_to_call_match
     regex_stub = RegexStub.new
+    csv = CSV.new(@sample_data, :skip_lines => regex_stub)
     assert_raise_with_message(ArgumentError, /skip_lines/) do
-      CSV.new(@sample_data, :skip_lines => regex_stub)
+      csv.shift
+    end
+  end
+
+  class Matchable
+    def initialize(pattern)
+      @pattern = pattern
     end
+
+    def match(line)
+      @pattern.match(line)
+    end
+  end
+
+  def test_skip_lines_match
+    csv = <<-CSV.chomp
+1
+# 2
+3
+# 4
+    CSV
+    assert_equal([["1"], ["3"]],
+                 CSV.parse(csv, :skip_lines => Matchable.new(/\A#/)))
   end
 
   def test_comment_rows_are_ignored
@@ -375,4 +449,48 @@ line,4,jkl
   def test_table_nil_equality
     assert_nothing_raised(NoMethodError) { CSV.parse("test", headers: true) == nil }
   end
+
+  # non-seekable input stream for testing https://github.com/ruby/csv/issues/44
+  class DummyIO
+    extend Forwardable
+    def_delegators :@io, :gets, :read, :pos, :eof?  # no seek or rewind!
+    def initialize(data)
+      @io = StringIO.new(data)
+    end
+  end
+
+  def test_line_separator_autodetection_for_non_seekable_input_lf
+    c = CSV.new(DummyIO.new("one,two,three\nfoo,bar,baz\n"))
+    assert_equal [["one", "two", "three"], ["foo", "bar", "baz"]], c.each.to_a
+  end
+
+  def test_line_separator_autodetection_for_non_seekable_input_cr
+    c = CSV.new(DummyIO.new("one,two,three\rfoo,bar,baz\r"))
+    assert_equal [["one", "two", "three"], ["foo", "bar", "baz"]], c.each.to_a
+  end
+
+  def test_line_separator_autodetection_for_non_seekable_input_cr_lf
+    c = CSV.new(DummyIO.new("one,two,three\r\nfoo,bar,baz\r\n"))
+    assert_equal [["one", "two", "three"], ["foo", "bar", "baz"]], c.each.to_a
+  end
+
+  def test_line_separator_autodetection_for_non_seekable_input_1024_over_lf
+    table = (1..10).map { |row| (1..200).map { |col| "row#{row}col#{col}" }.to_a }.to_a
+    input = table.map { |line| line.join(",") }.join("\n")
+    c = CSV.new(DummyIO.new(input))
+    assert_equal table, c.each.to_a
+  end
+
+  def test_line_separator_autodetection_for_non_seekable_input_1024_over_cr_lf
+    table = (1..10).map { |row| (1..200).map { |col| "row#{row}col#{col}" }.to_a }.to_a
+    input = table.map { |line| line.join(",") }.join("\r\n")
+    c = CSV.new(DummyIO.new(input))
+    assert_equal table, c.each.to_a
+  end
+
+  def test_line_separator_autodetection_for_non_seekable_input_many_cr_only
+    # input with lots of CRs (to make sure no bytes are lost due to look-ahead)
+    c = CSV.new(DummyIO.new("foo\r" + "\r" * 9999 + "bar\r"))
+    assert_equal [["foo"]] + [[]] * 9999 + [["bar"]], c.each.to_a
+  end
 end
diff --git a/test/csv/test_interface.rb b/test/csv/test_interface.rb
index 912e2ec7f5..309fbbb87b 100755
--- a/test/csv/test_interface.rb
+++ b/test/csv/test_interface.rb
@@ -139,6 +139,18 @@ class TestCSV::Interface < TestCSV
     assert_equal(Array.new, CSV.parse_line("\n1,2,3"))
   end
 
+  def test_parse_header_only
+    table = CSV.parse("a,b,c", headers: true)
+    assert_equal([
+                   ["a", "b", "c"],
+                   [],
+                 ],
+                 [
+                   table.headers,
+                   table.each.to_a,
+                 ])
+  end
+
   def test_read_and_readlines
     assert_equal( @expected,
                   CSV.read(@path, col_sep: "\t", row_sep: "\r\n") )
@@ -236,7 +248,7 @@ class TestCSV::Interface < TestCSV
     CSV.open(@path, "w", headers: true) do |csv|
       csv << headers
       csv << %w{1 2 3}
-      assert_equal(headers, csv.instance_variable_get(:@headers))
+      assert_equal(headers, csv.headers)
     end
   end
 
diff --git a/test/csv/test_table.rb b/test/csv/test_table.rb
index d99b7d2932..a5ae8e0381 100755
--- a/test/csv/test_table.rb
+++ b/test/csv/test_table.rb
@@ -21,6 +21,8 @@ class TestCSV::Table < TestCSV
     @header_table = CSV::Table.new(
       [CSV::Row.new(%w{A B C}, %w{A B C}, true)] + @rows
     )
+    
+    @header_only_table = CSV::Table.new([], headers: %w{A B C})
   end
 
   def test_initialze
@@ -63,6 +65,10 @@ class TestCSV::Table < TestCSV
     assert_equal Array.new, t.headers
   end
 
+  def test_headers_only
+    assert_equal(%w[A B C], @header_only_table.headers)
+  end
+
   def test_index
     ##################
     ### Mixed Mode ###
@@ -471,6 +477,21 @@ A
     CSV
   end
 
+  def test_delete_headers_only
+    ###################
+    ### Column Mode ###
+    ###################
+    @header_only_table.by_col!
+
+    # delete by index
+    assert_equal([], @header_only_table.delete(0))
+    assert_equal(%w[B C], @header_only_table.headers)
+
+    # delete by header
+    assert_equal([], @header_only_table.delete("C"))
+    assert_equal(%w[B], @header_only_table.headers)
+  end
+
   def test_values_at
     ##################
     ### Mixed Mode ###