diff options
Diffstat (limited to 'lib/rubygems/package')
| -rw-r--r-- | lib/rubygems/package/f_sync_dir.rb | 24 | ||||
| -rw-r--r-- | lib/rubygems/package/tar_header.rb | 245 | ||||
| -rw-r--r-- | lib/rubygems/package/tar_input.rb | 219 | ||||
| -rw-r--r-- | lib/rubygems/package/tar_output.rb | 143 | ||||
| -rw-r--r-- | lib/rubygems/package/tar_reader.rb | 86 | ||||
| -rw-r--r-- | lib/rubygems/package/tar_reader/entry.rb | 99 | ||||
| -rw-r--r-- | lib/rubygems/package/tar_writer.rb | 180 |
7 files changed, 996 insertions, 0 deletions
diff --git a/lib/rubygems/package/f_sync_dir.rb b/lib/rubygems/package/f_sync_dir.rb new file mode 100644 index 0000000000..3e2e4a59a8 --- /dev/null +++ b/lib/rubygems/package/f_sync_dir.rb @@ -0,0 +1,24 @@ +#++ +# Copyright (C) 2004 Mauricio Julio Fern�ndez Pradier +# See LICENSE.txt for additional licensing information. +#-- + +require 'rubygems/package' + +module Gem::Package::FSyncDir + + private + + ## + # make sure this hits the disc + + def fsync_dir(dirname) + dir = open dirname, 'r' + dir.fsync + rescue # ignore IOError if it's an unpatched (old) Ruby + ensure + dir.close if dir rescue nil + end + +end + diff --git a/lib/rubygems/package/tar_header.rb b/lib/rubygems/package/tar_header.rb new file mode 100644 index 0000000000..c194cc0530 --- /dev/null +++ b/lib/rubygems/package/tar_header.rb @@ -0,0 +1,245 @@ +#++ +# Copyright (C) 2004 Mauricio Julio Fern�ndez Pradier +# See LICENSE.txt for additional licensing information. +#-- + +require 'rubygems/package' + +## +#-- +# struct tarfile_entry_posix { +# char name[100]; # ASCII + (Z unless filled) +# char mode[8]; # 0 padded, octal, null +# char uid[8]; # ditto +# char gid[8]; # ditto +# char size[12]; # 0 padded, octal, null +# char mtime[12]; # 0 padded, octal, null +# char checksum[8]; # 0 padded, octal, null, space +# char typeflag[1]; # file: "0" dir: "5" +# char linkname[100]; # ASCII + (Z unless filled) +# char magic[6]; # "ustar\0" +# char version[2]; # "00" +# char uname[32]; # ASCIIZ +# char gname[32]; # ASCIIZ +# char devmajor[8]; # 0 padded, octal, null +# char devminor[8]; # o padded, octal, null +# char prefix[155]; # ASCII + (Z unless filled) +# }; +#++ + +class Gem::Package::TarHeader + + FIELDS = [ + :checksum, + :devmajor, + :devminor, + :gid, + :gname, + :linkname, + :magic, + :mode, + :mtime, + :name, + :prefix, + :size, + :typeflag, + :uid, + :uname, + :version, + ] + + PACK_FORMAT = 'a100' + # name + 'a8' + # mode + 'a8' + # uid + 'a8' + # gid + 'a12' + # size + 'a12' + # mtime + 'a7a' + # chksum + 'a' + # typeflag + 'a100' + # linkname + 'a6' + # magic + 'a2' + # version + 'a32' + # uname + 'a32' + # gname + 'a8' + # devmajor + 'a8' + # devminor + 'a155' # prefix + + UNPACK_FORMAT = 'A100' + # name + 'A8' + # mode + 'A8' + # uid + 'A8' + # gid + 'A12' + # size + 'A12' + # mtime + 'A8' + # checksum + 'A' + # typeflag + 'A100' + # linkname + 'A6' + # magic + 'A2' + # version + 'A32' + # uname + 'A32' + # gname + 'A8' + # devmajor + 'A8' + # devminor + 'A155' # prefix + + attr_reader(*FIELDS) + + def self.from(stream) + header = stream.read 512 + empty = (header == "\0" * 512) + + fields = header.unpack UNPACK_FORMAT + + name = fields.shift + mode = fields.shift.oct + uid = fields.shift.oct + gid = fields.shift.oct + size = fields.shift.oct + mtime = fields.shift.oct + checksum = fields.shift.oct + typeflag = fields.shift + linkname = fields.shift + magic = fields.shift + version = fields.shift.oct + uname = fields.shift + gname = fields.shift + devmajor = fields.shift.oct + devminor = fields.shift.oct + prefix = fields.shift + + new :name => name, + :mode => mode, + :uid => uid, + :gid => gid, + :size => size, + :mtime => mtime, + :checksum => checksum, + :typeflag => typeflag, + :linkname => linkname, + :magic => magic, + :version => version, + :uname => uname, + :gname => gname, + :devmajor => devmajor, + :devminor => devminor, + :prefix => prefix, + + :empty => empty + + # HACK unfactor for Rubinius + #new :name => fields.shift, + # :mode => fields.shift.oct, + # :uid => fields.shift.oct, + # :gid => fields.shift.oct, + # :size => fields.shift.oct, + # :mtime => fields.shift.oct, + # :checksum => fields.shift.oct, + # :typeflag => fields.shift, + # :linkname => fields.shift, + # :magic => fields.shift, + # :version => fields.shift.oct, + # :uname => fields.shift, + # :gname => fields.shift, + # :devmajor => fields.shift.oct, + # :devminor => fields.shift.oct, + # :prefix => fields.shift, + + # :empty => empty + end + + def initialize(vals) + unless vals[:name] && vals[:size] && vals[:prefix] && vals[:mode] then + raise ArgumentError, ":name, :size, :prefix and :mode required" + end + + vals[:uid] ||= 0 + vals[:gid] ||= 0 + vals[:mtime] ||= 0 + vals[:checksum] ||= "" + vals[:typeflag] ||= "0" + vals[:magic] ||= "ustar" + vals[:version] ||= "00" + vals[:uname] ||= "wheel" + vals[:gname] ||= "wheel" + vals[:devmajor] ||= 0 + vals[:devminor] ||= 0 + + FIELDS.each do |name| + instance_variable_set "@#{name}", vals[name] + end + + @empty = vals[:empty] + end + + def empty? + @empty + end + + def ==(other) + self.class === other and + @checksum == other.checksum and + @devmajor == other.devmajor and + @devminor == other.devminor and + @gid == other.gid and + @gname == other.gname and + @linkname == other.linkname and + @magic == other.magic and + @mode == other.mode and + @mtime == other.mtime and + @name == other.name and + @prefix == other.prefix and + @size == other.size and + @typeflag == other.typeflag and + @uid == other.uid and + @uname == other.uname and + @version == other.version + end + + def to_s + update_checksum + header + end + + def update_checksum + header = header " " * 8 + @checksum = oct calculate_checksum(header), 6 + end + + private + + def calculate_checksum(header) + header.unpack("C*").inject { |a, b| a + b } + end + + def header(checksum = @checksum) + header = [ + name, + oct(mode, 7), + oct(uid, 7), + oct(gid, 7), + oct(size, 11), + oct(mtime, 11), + checksum, + " ", + typeflag, + linkname, + magic, + oct(version, 2), + uname, + gname, + oct(devmajor, 7), + oct(devminor, 7), + prefix + ] + + header = header.pack PACK_FORMAT + + header << ("\0" * ((512 - header.size) % 512)) + end + + def oct(num, len) + "%0#{len}o" % num + end + +end + diff --git a/lib/rubygems/package/tar_input.rb b/lib/rubygems/package/tar_input.rb new file mode 100644 index 0000000000..2ed3d6b772 --- /dev/null +++ b/lib/rubygems/package/tar_input.rb @@ -0,0 +1,219 @@ +#++ +# Copyright (C) 2004 Mauricio Julio Fern�ndez Pradier +# See LICENSE.txt for additional licensing information. +#-- + +require 'rubygems/package' + +class Gem::Package::TarInput + + include Gem::Package::FSyncDir + include Enumerable + + attr_reader :metadata + + private_class_method :new + + def self.open(io, security_policy = nil, &block) + is = new io, security_policy + + yield is + ensure + is.close if is + end + + def initialize(io, security_policy = nil) + @io = io + @tarreader = Gem::Package::TarReader.new @io + has_meta = false + + data_sig, meta_sig, data_dgst, meta_dgst = nil, nil, nil, nil + dgst_algo = security_policy ? Gem::Security::OPT[:dgst_algo] : nil + + @tarreader.each do |entry| + case entry.full_name + when "metadata" + @metadata = load_gemspec entry.read + has_meta = true + when "metadata.gz" + begin + # if we have a security_policy, then pre-read the metadata file + # and calculate it's digest + sio = nil + if security_policy + Gem.ensure_ssl_available + sio = StringIO.new(entry.read) + meta_dgst = dgst_algo.digest(sio.string) + sio.rewind + end + + gzis = Zlib::GzipReader.new(sio || entry) + # YAML wants an instance of IO + @metadata = load_gemspec(gzis) + has_meta = true + ensure + gzis.close unless gzis.nil? + end + when 'metadata.gz.sig' + meta_sig = entry.read + when 'data.tar.gz.sig' + data_sig = entry.read + when 'data.tar.gz' + if security_policy + Gem.ensure_ssl_available + data_dgst = dgst_algo.digest(entry.read) + end + end + end + + if security_policy then + Gem.ensure_ssl_available + + # map trust policy from string to actual class (or a serialized YAML + # file, if that exists) + if String === security_policy then + if Gem::Security::Policy.key? security_policy then + # load one of the pre-defined security policies + security_policy = Gem::Security::Policy[security_policy] + elsif File.exist? security_policy then + # FIXME: this doesn't work yet + security_policy = YAML.load File.read(security_policy) + else + raise Gem::Exception, "Unknown trust policy '#{security_policy}'" + end + end + + if data_sig && data_dgst && meta_sig && meta_dgst then + # the user has a trust policy, and we have a signed gem + # file, so use the trust policy to verify the gem signature + + begin + security_policy.verify_gem(data_sig, data_dgst, @metadata.cert_chain) + rescue Exception => e + raise "Couldn't verify data signature: #{e}" + end + + begin + security_policy.verify_gem(meta_sig, meta_dgst, @metadata.cert_chain) + rescue Exception => e + raise "Couldn't verify metadata signature: #{e}" + end + elsif security_policy.only_signed + raise Gem::Exception, "Unsigned gem" + else + # FIXME: should display warning here (trust policy, but + # either unsigned or badly signed gem file) + end + end + + @tarreader.rewind + @fileops = Gem::FileOperations.new + + raise Gem::Package::FormatError, "No metadata found!" unless has_meta + end + + def close + @io.close + @tarreader.close + end + + def each(&block) + @tarreader.each do |entry| + next unless entry.full_name == "data.tar.gz" + is = zipped_stream entry + + begin + Gem::Package::TarReader.new is do |inner| + inner.each(&block) + end + ensure + is.close if is + end + end + + @tarreader.rewind + end + + def extract_entry(destdir, entry, expected_md5sum = nil) + if entry.directory? then + dest = File.join(destdir, entry.full_name) + + if File.dir? dest then + @fileops.chmod entry.header.mode, dest, :verbose=>false + else + @fileops.mkdir_p dest, :mode => entry.header.mode, :verbose => false + end + + fsync_dir dest + fsync_dir File.join(dest, "..") + + return + end + + # it's a file + md5 = Digest::MD5.new if expected_md5sum + destdir = File.join destdir, File.dirname(entry.full_name) + @fileops.mkdir_p destdir, :mode => 0755, :verbose => false + destfile = File.join destdir, File.basename(entry.full_name) + @fileops.chmod 0600, destfile, :verbose => false rescue nil # Errno::ENOENT + + open destfile, "wb", entry.header.mode do |os| + loop do + data = entry.read 4096 + break unless data + # HACK shouldn't we check the MD5 before writing to disk? + md5 << data if expected_md5sum + os.write(data) + end + + os.fsync + end + + @fileops.chmod entry.header.mode, destfile, :verbose => false + fsync_dir File.dirname(destfile) + fsync_dir File.join(File.dirname(destfile), "..") + + if expected_md5sum && expected_md5sum != md5.hexdigest then + raise Gem::Package::BadCheckSum + end + end + + # Attempt to YAML-load a gemspec from the given _io_ parameter. Return + # nil if it fails. + def load_gemspec(io) + Gem::Specification.from_yaml io + rescue Gem::Exception + nil + end + + ## + # Return an IO stream for the zipped entry. + # + # NOTE: Originally this method used two approaches, Return a GZipReader + # directly, or read the GZipReader into a string and return a StringIO on + # the string. The string IO approach was used for versions of ZLib before + # 1.2.1 to avoid buffer errors on windows machines. Then we found that + # errors happened with 1.2.1 as well, so we changed the condition. Then + # we discovered errors occurred with versions as late as 1.2.3. At this + # point (after some benchmarking to show we weren't seriously crippling + # the unpacking speed) we threw our hands in the air and declared that + # this method would use the String IO approach on all platforms at all + # times. And that's the way it is. + + def zipped_stream(entry) + if defined? Rubinius then + zis = Zlib::GzipReader.new entry + dis = zis.read + is = StringIO.new(dis) + else + # This is Jamis Buck's Zlib workaround for some unknown issue + entry.read(10) # skip the gzip header + zis = Zlib::Inflate.new(-Zlib::MAX_WBITS) + is = StringIO.new(zis.inflate(entry.read)) + end + ensure + zis.finish if zis + end + +end + diff --git a/lib/rubygems/package/tar_output.rb b/lib/rubygems/package/tar_output.rb new file mode 100644 index 0000000000..b22f7dd86b --- /dev/null +++ b/lib/rubygems/package/tar_output.rb @@ -0,0 +1,143 @@ +#++ +# Copyright (C) 2004 Mauricio Julio Fern�ndez Pradier +# See LICENSE.txt for additional licensing information. +#-- + +require 'rubygems/package' + +## +# TarOutput is a wrapper to TarWriter that builds gem-format tar file. +# +# Gem-format tar files contain the following files: +# [data.tar.gz] A gzipped tar file containing the files that compose the gem +# which will be extracted into the gem/ dir on installation. +# [metadata.gz] A YAML format Gem::Specification. +# [data.tar.gz.sig] A signature for the gem's data.tar.gz. +# [metadata.gz.sig] A signature for the gem's metadata.gz. +# +# See TarOutput::open for usage details. + +class Gem::Package::TarOutput + + ## + # Creates a new TarOutput which will yield a TarWriter object for the + # data.tar.gz portion of a gem-format tar file. + # + # See #initialize for details on +io+ and +signer+. + # + # See #add_gem_contents for details on adding metadata to the tar file. + + def self.open(io, signer = nil, &block) # :yield: data_tar_writer + tar_outputter = new io, signer + tar_outputter.add_gem_contents(&block) + tar_outputter.add_metadata + tar_outputter.add_signatures + + ensure + tar_outputter.close + end + + ## + # Creates a new TarOutput that will write a gem-format tar file to +io+. If + # +signer+ is given, the data.tar.gz and metadata.gz will be signed and + # the signatures will be added to the tar file. + + def initialize(io, signer) + @io = io + @signer = signer + + @tar_writer = Gem::Package::TarWriter.new @io + + @metadata = nil + + @data_signature = nil + @meta_signature = nil + end + + ## + # Yields a TarWriter for the data.tar.gz inside a gem-format tar file. + # The yielded TarWriter has been extended with a #metadata= method for + # attaching a YAML format Gem::Specification which will be written by + # add_metadata. + + def add_gem_contents + @tar_writer.add_file "data.tar.gz", 0644 do |inner| + sio = @signer ? StringIO.new : nil + Zlib::GzipWriter.wrap(sio || inner) do |os| + + Gem::Package::TarWriter.new os do |data_tar_writer| + def data_tar_writer.metadata() @metadata end + def data_tar_writer.metadata=(metadata) @metadata = metadata end + + yield data_tar_writer + + @metadata = data_tar_writer.metadata + end + end + + # if we have a signing key, then sign the data + # digest and return the signature + if @signer then + digest = Gem::Security::OPT[:dgst_algo].digest sio.string + @data_signature = @signer.sign digest + inner.write sio.string + end + end + + self + end + + ## + # Adds metadata.gz to the gem-format tar file which was saved from a + # previous #add_gem_contents call. + + def add_metadata + return if @metadata.nil? + + @tar_writer.add_file "metadata.gz", 0644 do |io| + begin + sio = @signer ? StringIO.new : nil + gzos = Zlib::GzipWriter.new(sio || io) + gzos.write @metadata + ensure + gzos.flush + gzos.finish + + # if we have a signing key, then sign the metadata digest and return + # the signature + if @signer then + digest = Gem::Security::OPT[:dgst_algo].digest sio.string + @meta_signature = @signer.sign digest + io.write sio.string + end + end + end + end + + ## + # Adds data.tar.gz.sig and metadata.gz.sig to the gem-format tar files if + # a Gem::Security::Signer was sent to initialize. + + def add_signatures + if @data_signature then + @tar_writer.add_file 'data.tar.gz.sig', 0644 do |io| + io.write @data_signature + end + end + + if @meta_signature then + @tar_writer.add_file 'metadata.gz.sig', 0644 do |io| + io.write @meta_signature + end + end + end + + ## + # Closes the TarOutput. + + def close + @tar_writer.close + end + +end + diff --git a/lib/rubygems/package/tar_reader.rb b/lib/rubygems/package/tar_reader.rb new file mode 100644 index 0000000000..8359399207 --- /dev/null +++ b/lib/rubygems/package/tar_reader.rb @@ -0,0 +1,86 @@ +#++ +# Copyright (C) 2004 Mauricio Julio Fern�ndez Pradier +# See LICENSE.txt for additional licensing information. +#-- + +require 'rubygems/package' + +class Gem::Package::TarReader + + include Gem::Package + + class UnexpectedEOF < StandardError; end + + def self.new(io) + reader = super + + return reader unless block_given? + + begin + yield reader + ensure + reader.close + end + + nil + end + + def initialize(io) + @io = io + @init_pos = io.pos + end + + def close + end + + def each + loop do + return if @io.eof? + + header = Gem::Package::TarHeader.from @io + return if header.empty? + + entry = Gem::Package::TarReader::Entry.new header, @io + size = entry.header.size + + yield entry + + skip = (512 - (size % 512)) % 512 + + if @io.respond_to? :seek then + # avoid reading... + @io.seek(size - entry.bytes_read, IO::SEEK_CUR) + else + pending = size - entry.bytes_read + + while pending > 0 do + bread = @io.read([pending, 4096].min).size + raise UnexpectedEOF if @io.eof? + pending -= bread + end + end + + @io.read skip # discard trailing zeros + + # make sure nobody can use #read, #getc or #rewind anymore + entry.close + end + end + + alias each_entry each + + ## + # NOTE: Do not call #rewind during #each + + def rewind + if @init_pos == 0 then + raise Gem::Package::NonSeekableIO unless @io.respond_to? :rewind + @io.rewind + else + raise Gem::Package::NonSeekableIO unless @io.respond_to? :pos= + @io.pos = @init_pos + end + end + +end + diff --git a/lib/rubygems/package/tar_reader/entry.rb b/lib/rubygems/package/tar_reader/entry.rb new file mode 100644 index 0000000000..dcc66153d8 --- /dev/null +++ b/lib/rubygems/package/tar_reader/entry.rb @@ -0,0 +1,99 @@ +#++ +# Copyright (C) 2004 Mauricio Julio Fern�ndez Pradier +# See LICENSE.txt for additional licensing information. +#-- + +require 'rubygems/package' + +class Gem::Package::TarReader::Entry + + attr_reader :header + + def initialize(header, io) + @closed = false + @header = header + @io = io + @orig_pos = @io.pos + @read = 0 + end + + def check_closed # :nodoc: + raise IOError, "closed #{self.class}" if closed? + end + + def bytes_read + @read + end + + def close + @closed = true + end + + def closed? + @closed + end + + def eof? + check_closed + + @read >= @header.size + end + + def full_name + if @header.prefix != "" then + File.join @header.prefix, @header.name + else + @header.name + end + end + + def getc + check_closed + + return nil if @read >= @header.size + + ret = @io.getc + @read += 1 if ret + + ret + end + + def directory? + @header.typeflag == "5" + end + + def file? + @header.typeflag == "0" + end + + def pos + check_closed + + bytes_read + end + + def read(len = nil) + check_closed + + return nil if @read >= @header.size + + len ||= @header.size - @read + max_read = [len, @header.size - @read].min + + ret = @io.read max_read + @read += ret.size + + ret + end + + def rewind + check_closed + + raise Gem::Package::NonSeekableIO unless @io.respond_to? :pos= + + @io.pos = @orig_pos + @read = 0 + end + +end + diff --git a/lib/rubygems/package/tar_writer.rb b/lib/rubygems/package/tar_writer.rb new file mode 100644 index 0000000000..6e11440e22 --- /dev/null +++ b/lib/rubygems/package/tar_writer.rb @@ -0,0 +1,180 @@ +#++ +# Copyright (C) 2004 Mauricio Julio Fern�ndez Pradier +# See LICENSE.txt for additional licensing information. +#-- + +require 'rubygems/package' + +class Gem::Package::TarWriter + + class FileOverflow < StandardError; end + + class BoundedStream + + attr_reader :limit, :written + + def initialize(io, limit) + @io = io + @limit = limit + @written = 0 + end + + def write(data) + if data.size + @written > @limit + raise FileOverflow, "You tried to feed more data than fits in the file." + end + @io.write data + @written += data.size + data.size + end + + end + + class RestrictedStream + + def initialize(io) + @io = io + end + + def write(data) + @io.write data + end + + end + + def self.new(io) + writer = super + + return writer unless block_given? + + begin + yield writer + ensure + writer.close + end + + nil + end + + def initialize(io) + @io = io + @closed = false + end + + def add_file(name, mode) + check_closed + + raise Gem::Package::NonSeekableIO unless @io.respond_to? :pos= + + name, prefix = split_name name + + init_pos = @io.pos + @io.write "\0" * 512 # placeholder for the header + + yield RestrictedStream.new(@io) if block_given? + + size = @io.pos - init_pos - 512 + + remainder = (512 - (size % 512)) % 512 + @io.write "\0" * remainder + + final_pos = @io.pos + @io.pos = init_pos + + header = Gem::Package::TarHeader.new :name => name, :mode => mode, + :size => size, :prefix => prefix + + @io.write header + @io.pos = final_pos + + self + end + + def add_file_simple(name, mode, size) + check_closed + + name, prefix = split_name name + + header = Gem::Package::TarHeader.new(:name => name, :mode => mode, + :size => size, :prefix => prefix).to_s + + @io.write header + os = BoundedStream.new @io, size + + yield os if block_given? + + min_padding = size - os.written + @io.write("\0" * min_padding) + + remainder = (512 - (size % 512)) % 512 + @io.write("\0" * remainder) + + self + end + + def check_closed + raise IOError, "closed #{self.class}" if closed? + end + + def close + check_closed + + @io.write "\0" * 1024 + flush + + @closed = true + end + + def closed? + @closed + end + + def flush + check_closed + + @io.flush if @io.respond_to? :flush + end + + def mkdir(name, mode) + check_closed + + name, prefix = split_name(name) + + header = Gem::Package::TarHeader.new :name => name, :mode => mode, + :typeflag => "5", :size => 0, + :prefix => prefix + + @io.write header + + self + end + + def split_name(name) # :nodoc: + raise Gem::Package::TooLongFileName if name.size > 256 + + if name.size <= 100 then + prefix = "" + else + parts = name.split(/\//) + newname = parts.pop + nxt = "" + + loop do + nxt = parts.pop + break if newname.size + 1 + nxt.size > 100 + newname = nxt + "/" + newname + end + + prefix = (parts + [nxt]).join "/" + name = newname + + if name.size > 100 or prefix.size > 155 then + raise Gem::Package::TooLongFileName + end + end + + return name, prefix + end + +end + |