From 5eaae8c746f0d24b7145d5a8afc1112550a37d4f Mon Sep 17 00:00:00 2001 From: Brian Hawkins Date: Sat, 22 Aug 2015 07:42:55 -0600 Subject: [PATCH 1/6] Added common download method --- lib/librarian/puppet/cli.rb | 7 +++++++ lib/librarian/puppet/source/forge/repo.rb | 9 +++------ .../puppet/source/githubtarball/repo.rb | 14 +------------- lib/librarian/puppet/util.rb | 16 ++++++++++++++++ 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/librarian/puppet/cli.rb b/lib/librarian/puppet/cli.rb index afb8e88c..447618c4 100644 --- a/lib/librarian/puppet/cli.rb +++ b/lib/librarian/puppet/cli.rb @@ -76,6 +76,13 @@ def update(*names) super(*names.map{|n| normalize_name(n)}) end + desc "download", "Download puppet modules to specified path" + option "verbose", :type => :boolean, :default => false + option "path", :type => :string + def download + + end + desc "package", "Cache the puppet modules in vendor/puppet/cache." option "quiet", :type => :boolean, :default => false option "verbose", :type => :boolean, :default => false diff --git a/lib/librarian/puppet/source/forge/repo.rb b/lib/librarian/puppet/source/forge/repo.rb index cd4ea9e9..85af0718 100644 --- a/lib/librarian/puppet/source/forge/repo.rb +++ b/lib/librarian/puppet/source/forge/repo.rb @@ -142,13 +142,10 @@ def check_puppet_module_options def vendor_cache(name, version) url = url(name, version) path = vendored_path(name, version).to_s - debug { "Downloading #{url} into #{path}"} environment.vendor! - File.open(path, 'wb') do |f| - open(url, "rb") do |input| - f.write(input.read) - end - end + + download_file(url, path) + end end diff --git a/lib/librarian/puppet/source/githubtarball/repo.rb b/lib/librarian/puppet/source/githubtarball/repo.rb index c53bf186..8ed9694a 100644 --- a/lib/librarian/puppet/source/githubtarball/repo.rb +++ b/lib/librarian/puppet/source/githubtarball/repo.rb @@ -74,19 +74,7 @@ def vendor_cache(name, version) add_api_token_to_url(url) environment.vendor! - File.open(vendored_path(vendored_name(name), version).to_s, 'wb') do |f| - begin - debug { "Downloading <#{url}> to <#{f.path}>" } - open(url, - "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}") do |res| - while buffer = res.read(8192) - f.write(buffer) - end - end - rescue OpenURI::HTTPError => e - raise e, "Error requesting <#{url}>: #{e.to_s}" - end - end + download_file(url, vendored_path(vendored_name(name), version).to_s) end def clean_up_old_cached_versions(name) diff --git a/lib/librarian/puppet/util.rb b/lib/librarian/puppet/util.rb index bfbbea96..7c058f4c 100644 --- a/lib/librarian/puppet/util.rb +++ b/lib/librarian/puppet/util.rb @@ -57,6 +57,22 @@ def module_name(name) name.rpartition('-').last end + def download_file(url, path) + File.open(path, 'wb') do |f| + begin + debug { "Downloading <#{url}> to <#{f.path}>" } + open(url, + "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}") do |res| + while buffer = res.read(8192) + f.write(buffer) + end + end + rescue OpenURI::HTTPError => e + raise e, "Error requesting <#{url}>: #{e.to_s}" + end + end + end + # deprecated alias :organization_name :module_name end From 22338d546504f0a3549db77aed867dddaae8fae4 Mon Sep 17 00:00:00 2001 From: Brian Hawkins Date: Mon, 24 Aug 2015 09:57:01 -0600 Subject: [PATCH 2/6] removing puppet references --- Rakefile | 7 ++-- lib/librarian/puppet.rb | 22 ----------- lib/librarian/puppet/cli.rb | 7 ---- lib/librarian/puppet/source/forge/repo.rb | 48 ++--------------------- librarian-puppet.gemspec | 2 +- spec/librarian_puppet_spec.rb | 14 ------- 6 files changed, 9 insertions(+), 91 deletions(-) delete mode 100644 spec/librarian_puppet_spec.rb diff --git a/Rakefile b/Rakefile index e2a9bbe1..4d2aaf90 100644 --- a/Rakefile +++ b/Rakefile @@ -10,10 +10,11 @@ CLOBBER.include('Gemfile.lock') RSpec::Core::RakeTask.new Cucumber::Rake::Task.new(:features) do |t| - require 'puppet' - puppet_version = Puppet::version.gsub("~>","").split(".").first.to_i - tags = (2..4).select {|i| i != puppet_version}.map{|i| "--tags @puppet#{puppet_version},~@puppet#{i}"} + #require 'puppet' + #puppet_version = Puppet::version.gsub("~>","").split(".").first.to_i + #tags = (2..4).select {|i| i != puppet_version}.map{|i| "--tags @puppet#{puppet_version},~@puppet#{i}"} # don't run githubtarball scenarios in Travis, they easily fail with rate limit exceeded + tags = [] tags << "--tags ~@github" if ENV['TRAVIS']=='true' t.cucumber_opts = tags.join(" ") end diff --git a/lib/librarian/puppet.rb b/lib/librarian/puppet.rb index 22886032..365537ca 100644 --- a/lib/librarian/puppet.rb +++ b/lib/librarian/puppet.rb @@ -8,29 +8,7 @@ module Librarian module Puppet - @@puppet_version = nil - # Output of puppet --version, typically x.y.z - # For Puppet Enterprise it contains the PE version too, ie. 3.4.3 (Puppet Enterprise 3.2.1) - def puppet_version - return @@puppet_version unless @@puppet_version.nil? - - begin - @@puppet_version = Librarian::Posix.run!(%W{puppet --version}).strip - rescue Errno::ENOENT, Librarian::Posix::CommandFailure => error - msg = "Unable to load puppet. Please install it using native packages for your platform (eg .deb, .rpm, .dmg, etc)." - msg += "\npuppet --version returned #{error.status}" if error.respond_to? :status - msg += "\n#{error.message}" unless error.message.nil? - $stderr.puts msg - exit 1 - end - return @@puppet_version - end - - # Puppet version x.y.z translated as a Gem version - def puppet_gem_version - Gem::Version.create(puppet_version.split(' ').first.strip.gsub('-', '.')) - end end end diff --git a/lib/librarian/puppet/cli.rb b/lib/librarian/puppet/cli.rb index 447618c4..afb8e88c 100644 --- a/lib/librarian/puppet/cli.rb +++ b/lib/librarian/puppet/cli.rb @@ -76,13 +76,6 @@ def update(*names) super(*names.map{|n| normalize_name(n)}) end - desc "download", "Download puppet modules to specified path" - option "verbose", :type => :boolean, :default => false - option "path", :type => :string - def download - - end - desc "package", "Cache the puppet modules in vendor/puppet/cache." option "quiet", :type => :boolean, :default => false option "verbose", :type => :boolean, :default => false diff --git a/lib/librarian/puppet/source/forge/repo.rb b/lib/librarian/puppet/source/forge/repo.rb index 85af0718..0704cc54 100644 --- a/lib/librarian/puppet/source/forge/repo.rb +++ b/lib/librarian/puppet/source/forge/repo.rb @@ -73,46 +73,13 @@ def cache_version_unpacked!(version) path = version_unpacked_cache_path(version) return if path.directory? - # The puppet module command is only available from puppet versions >= 2.7.13 - # - # Specifying the version in the gemspec would force people to upgrade puppet while it's still usable for git - # So we do some more clever checking - # - # Executing older versions or via puppet-module tool gives an exit status = 0 . - # - check_puppet_module_options - path.mkpath - target = vendored?(name, version) ? vendored_path(name, version).to_s : name - - # can't pass the default v3 forge url (http://forgeapi.puppetlabs.com) - # to clients that use the v1 API (https://forge.puppetlabs.com) - # nor the other way around - module_repository = source.to_s - - if Forge.client_api_version() > 1 and module_repository =~ %r{^http(s)?://forge\.puppetlabs\.com} - module_repository = "https://forgeapi.puppetlabs.com" - warn { "Replacing Puppet Forge API URL to use v3 #{module_repository} as required by your client version #{Librarian::Puppet.puppet_version}" } - end - - m = module_repository.match(%r{^http(s)?://forge(api)?\.puppetlabs\.com}) - if Forge.client_api_version() == 1 and m - ssl = m[1] - # Puppet 2.7 can't handle the 302 returned by the https url, so stick to http - if ssl and Librarian::Puppet::puppet_gem_version < Gem::Version.create('3.0.0') - warn { "Using plain http as your version of Puppet #{Librarian::Puppet::puppet_gem_version} can't download from forge.puppetlabs.com using https" } - ssl = nil - end - module_repository = "http#{ssl}://forge.puppetlabs.com" - end - - command = %W{puppet module install --version #{version} --target-dir} - command.push(*[path.to_s, "--module_repository", module_repository, "--modulepath", path.to_s, "--module_working_dir", path.to_s, "--ignore-dependencies", target]) - debug { "Executing puppet module install for #{name} #{version}: #{command.join(" ")}" } + download_file(url(name, version), "#{path.to_s}/#{name}-#{version}.tar.gz") begin - Librarian::Posix.run!(command) + Librarian::Posix.run!(%W{tar -xf #{path.to_s}/#{name}-#{version}.tar.gz -C #{path.to_s}}) + FileUtils::mv "#{path.to_s}/#{name}-#{version}", "#{path.to_s}/#{name.split('-')[1]}" rescue Posix::CommandFailure => e # Rollback the directory if the puppet module had an error begin @@ -126,18 +93,11 @@ def cache_version_unpacked!(version) file = tar.first msg = " (looks like an incomplete download of #{file})" end - raise Error, "Error executing puppet module install#{msg}. Check that this command succeeds:\n#{command.join(" ")}\nError:\n#{e.message}" + raise Error, "Error extracting module #{msg}." end end - def check_puppet_module_options - min_version = Gem::Version.create('2.7.13') - - if Librarian::Puppet.puppet_gem_version < min_version - raise Error, "To get modules from the forge, we use the puppet faces module command. For this you need at least puppet version 2.7.13 and you have #{Librarian::Puppet.puppet_version}" - end - end def vendor_cache(name, version) url = url(name, version) diff --git a/librarian-puppet.gemspec b/librarian-puppet.gemspec index e920b90b..66ebe5dd 100644 --- a/librarian-puppet.gemspec +++ b/librarian-puppet.gemspec @@ -34,7 +34,7 @@ Gem::Specification.new do |s| s.add_development_dependency "rspec" s.add_development_dependency "cucumber" s.add_development_dependency "aruba" - s.add_development_dependency "puppet", ENV["PUPPET_VERSION"] + #s.add_development_dependency "puppet", ENV["PUPPET_VERSION"] s.add_development_dependency "minitest", "~> 5" s.add_development_dependency "mocha" s.add_development_dependency "simplecov", ">= 0.9.0" diff --git a/spec/librarian_puppet_spec.rb b/spec/librarian_puppet_spec.rb deleted file mode 100644 index cfbed4fe..00000000 --- a/spec/librarian_puppet_spec.rb +++ /dev/null @@ -1,14 +0,0 @@ -describe Librarian::Puppet do - it 'exits with error if puppet is not installed' do - error = Librarian::Posix::CommandFailure.new 'puppet not installed' - error.status = 42 - - expect(Librarian::Posix).to receive(:run!).and_raise(error) - expect($stderr).to receive(:puts) do |message| - expect(message).to match /42/ - expect(message).to match /puppet not installed/ - end - - expect { Librarian::Puppet::puppet_version }.to raise_error(SystemExit) - end -end From 4ad5ba4be80b8c5e411bf4bbbc92ecd494f06acd Mon Sep 17 00:00:00 2001 From: Brian Hawkins Date: Mon, 24 Aug 2015 15:08:43 -0600 Subject: [PATCH 3/6] fixing unit tests --- lib/librarian/puppet/source/forge.rb | 19 ++++++++++++------- lib/librarian/puppet/source/local.rb | 2 +- librarian-puppet.gemspec | 2 +- spec/source/forge_spec.rb | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/librarian/puppet/source/forge.rb b/lib/librarian/puppet/source/forge.rb index b885f445..a4bab139 100644 --- a/lib/librarian/puppet/source/forge.rb +++ b/lib/librarian/puppet/source/forge.rb @@ -61,11 +61,6 @@ def client_api_version() def initialize(environment, uri, options = {}) self.environment = environment - if uri =~ %r{^http(s)?://forge\.puppetlabs\.com} - uri = "https://forgeapi.puppetlabs.com" - warn { "Replacing Puppet Forge API URL to use v3 #{uri}. You should update your Puppetfile" } - end - @uri = URI::parse(uri) @cache_path = nil end @@ -158,8 +153,18 @@ def repo(name) @repo ||= {} unless @repo[name] - # if we are using the official Forge then use API v3, otherwise stick to v1 for now - if uri.hostname =~ /\.puppetlabs\.com$/ || !environment.use_v1_api + use_version_3 = true + # Use v3 of the api unless the url is the old one or v1 is explicitly set + if uri.hostname =~ /forge\.puppetlabs\.com$/ || environment.use_v1_api + use_version_3 = false + end + + #Override the above if they have specified the forgeapi (v3) endpoint + if uri.hostname =~ /forgeapi\.puppetlabs\.com$/ + use_version_3 = true + end + + if use_version_3 @repo[name] = RepoV3.new(self, name) else @repo[name] = RepoV1.new(self, name) diff --git a/lib/librarian/puppet/source/local.rb b/lib/librarian/puppet/source/local.rb index 4a3744ce..ccb7be4a 100644 --- a/lib/librarian/puppet/source/local.rb +++ b/lib/librarian/puppet/source/local.rb @@ -88,7 +88,7 @@ def evaluate_modulefile(modulefile) # Puppet 4 does not have the class unless defined? ::Puppet::ModuleTool::ModulefileReader - warn { "Can't parse Modulefile in Puppet >= 4.0 and you are using #{Librarian::Puppet::puppet_version}. Ignoring dependencies in #{modulefile}" } + warn { "Can't parse Modulefile in Puppet >= 4.0. Ignoring dependencies in #{modulefile}" } return metadata end diff --git a/librarian-puppet.gemspec b/librarian-puppet.gemspec index 66ebe5dd..e920b90b 100644 --- a/librarian-puppet.gemspec +++ b/librarian-puppet.gemspec @@ -34,7 +34,7 @@ Gem::Specification.new do |s| s.add_development_dependency "rspec" s.add_development_dependency "cucumber" s.add_development_dependency "aruba" - #s.add_development_dependency "puppet", ENV["PUPPET_VERSION"] + s.add_development_dependency "puppet", ENV["PUPPET_VERSION"] s.add_development_dependency "minitest", "~> 5" s.add_development_dependency "mocha" s.add_development_dependency "simplecov", ">= 0.9.0" diff --git a/spec/source/forge_spec.rb b/spec/source/forge_spec.rb index cbb9630f..748f3d20 100644 --- a/spec/source/forge_spec.rb +++ b/spec/source/forge_spec.rb @@ -7,7 +7,7 @@ describe Forge do let(:environment) { Librarian::Puppet::Environment.new } - let(:uri) { "https://forge.puppetlabs.com" } + let(:uri) { "https://forgeapi.puppetlabs.com" } let(:puppet_version) { "3.6.0" } subject { Forge.new(environment, uri) } From 860238aad0dfe86382316ed97adcd092adea2175 Mon Sep 17 00:00:00 2001 From: Brian Hawkins Date: Mon, 24 Aug 2015 15:51:17 -0600 Subject: [PATCH 4/6] fixed unit tests pointing at old forge url --- features/update.feature | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/features/update.feature b/features/update.feature index f11cce26..59e52528 100644 --- a/features/update.feature +++ b/features/update.feature @@ -17,14 +17,14 @@ Feature: cli/update And a file named "Puppetfile.lock" with: """ FORGE - remote: http://forge.puppetlabs.com + remote: https://forgeapi.puppetlabs.com specs: puppetlabs/stdlib (3.1.0) DEPENDENCIES puppetlabs/stdlib (~> 3.0) """ - When I run `librarian-puppet update puppetlabs/stdlib` + When I run `librarian-puppet update puppetlabs-stdlib` Then the exit status should be 0 And the file "Puppetfile" should not exist And the file "Puppetfile.lock" should match /puppetlabs.stdlib \(3\.1\.1\)/ @@ -40,7 +40,7 @@ Feature: cli/update And a file named "Puppetfile.lock" with: """ FORGE - remote: http://forge.puppetlabs.com + remote: https://forgeapi.puppetlabs.com specs: puppetlabs/stdlib (3.1.0) From 031f42f447d29d6e58716bf8a175d09011d22fba Mon Sep 17 00:00:00 2001 From: Brian Hawkins Date: Mon, 24 Aug 2015 16:07:39 -0600 Subject: [PATCH 5/6] cleaned up unused code --- .idea/uiDesigner.xml | 124 ++++++++++++++++++++++++++++++++++++++++ Rakefile | 4 -- lib/librarian/puppet.rb | 1 - 3 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 .idea/uiDesigner.xml diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 00000000..e96534fb --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Rakefile b/Rakefile index 4d2aaf90..5f6ffe28 100644 --- a/Rakefile +++ b/Rakefile @@ -10,10 +10,6 @@ CLOBBER.include('Gemfile.lock') RSpec::Core::RakeTask.new Cucumber::Rake::Task.new(:features) do |t| - #require 'puppet' - #puppet_version = Puppet::version.gsub("~>","").split(".").first.to_i - #tags = (2..4).select {|i| i != puppet_version}.map{|i| "--tags @puppet#{puppet_version},~@puppet#{i}"} - # don't run githubtarball scenarios in Travis, they easily fail with rate limit exceeded tags = [] tags << "--tags ~@github" if ENV['TRAVIS']=='true' t.cucumber_opts = tags.join(" ") diff --git a/lib/librarian/puppet.rb b/lib/librarian/puppet.rb index 365537ca..4a412d69 100644 --- a/lib/librarian/puppet.rb +++ b/lib/librarian/puppet.rb @@ -9,6 +9,5 @@ module Librarian module Puppet - end end From 6bf1c000ee9befcabe4668b7f241abd9e11653bb Mon Sep 17 00:00:00 2001 From: Brian Hawkins Date: Mon, 24 Aug 2015 16:09:38 -0600 Subject: [PATCH 6/6] accidently added intellij file --- .idea/uiDesigner.xml | 124 ------------------------------------------- 1 file changed, 124 deletions(-) delete mode 100644 .idea/uiDesigner.xml diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml deleted file mode 100644 index e96534fb..00000000 --- a/.idea/uiDesigner.xml +++ /dev/null @@ -1,124 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file