How to Develop Puppet Modules: From Source to the Forge With Zero Clicks

How to develop Puppet ModulesFrom source to the Forge with zero clicks

Carlos Sanchez@csanchezhttp://csanchez.orghttp://maestrodev.com

@csanchez Apache Maven

ASF Member

Eclipse Foundation

csanchez.orgmaestrodev.com

Modules

We use 50 modules

DEV QA OPS

Modules ARE software

oh my

VersionsDependenciesIncompabilities

Specs

RSpec-Puppet

Gemfile

source 'https://rubygems.org'

group :rake do gem 'puppet' gem 'rake' gem 'puppet-lint' gem 'rspec-puppet'end

modules

{module}/ spec/ spec_helper.rb classes/ {class}_spec.pp definitions/ fixtures/ hosts/

maven::maven

class maven::maven( $version = '3.0.5', $repo = { #url => 'http://repo1.maven.org/maven2', #username => '', #password => '', } ) {

if "x${repo['url']}x" != 'xx' { wget::authfetch { 'fetch-maven': source => "${repo['url']}/.../$version/apache-maven-${version}-bin.tar.gz", destination => $archive, user => $repo['username'], password => $repo['password'], before => Exec['maven-untar'], } } else { wget::fetch { 'fetch-maven': source => "http://archive.apache.org/.../apache-maven-${version}-bin.tar.gz", destination => $archive, before => Exec['maven-untar'], } }

rspec-puppet

require 'spec_helper'

describe 'maven::maven' do

context "when downloading maven from another repo" do let(:params) { { :repo => { 'url' => 'http://repo1.maven.org/maven2', 'username' => 'u', 'password' => 'p' } } }

it 'should fetch maven with username and password' do should contain_wget__authfetch('fetch-maven').with( 'source' => 'http://repo1.maven.org/...ven-3.0.5-bin.tar.gz', 'user' => 'u', 'password' => 'p') end endend

hosts

node 'agent' inherits 'parent' { include wget

include maestro::test::dependencies include maestro_nodes::agentrvm}

hosts

require 'spec_helper'

describe 'agent' do it do should contain_class('maestro::agent').with( 'agent_name' => 'agent-01', 'stomp_host' => 'maestro.maestrodev.net') end it { should_not contain_service('maestro') } it { should_not contain_service('activemq') } it { should_not contain_service('jenkins') } it { should_not contain_service('postgresqld') } it { should_not contain_service('maestro-test-hub') } it { should_not contain_service('sonar') } it { should_not contain_service('archiva') }end

rspec-puppet with facts

require 'spec_helper'

describe 'wget' do

context 'running on OS X' do let(:facts) { {:operatingsystem => 'Darwin'} } it { should_not contain_package('wget') } end

context 'running on CentOS' do let(:facts) { {:operatingsystem => 'CentOS'} } it { should contain_package('wget') } end

context 'no version specified' do it { should contain_package('wget').with_ensure('installed') } end

context 'version is 1.2.3' do let(:params) { {:version => '1.2.3'} } it { should contain_package('wget').with_ensure('1.2.3') } endend

shared_context

shared_context :centos do

let(:facts) {{ :operatingsystem => 'CentOS', :kernel => 'Linux', :osfamily => 'RedHat' }}

end

describe 'maestro::maestro' do include_context :centos ...end

extending for reuse

describe 'maestro::maestro' do include_context :centos

let(:facts) { super().merge({ :operatingsystem => 'RedHat' })}end

shared_examples

require 'spec_helper'

describe 'nginx::package' do

shared_examples 'redhat' do |operatingsystem| let(:facts) {{ :operatingsystem => operatingsystem }} it { should contain_package('nginx') } it { should contain_package('gd') } it { should contain_package('libXpm') } it { should contain_package('libxslt') } it { should contain_yumrepo('nginx-release').with_enabled('1') } end

shared_examples 'debian' do |operatingsystem| let(:facts) {{ :operatingsystem => operatingsystem }} it { should contain_file('/etc/apt/sources.list.d/nginx.list') } end

shared_examples (cont.)

context 'RedHat' do it_behaves_like 'redhat', 'centos' it_behaves_like 'redhat', 'fedora' it_behaves_like 'redhat', 'rhel' it_behaves_like 'redhat', 'redhat' it_behaves_like 'redhat', 'scientific' end

context 'debian' do it_behaves_like 'debian', 'debian' it_behaves_like 'debian', 'ubuntu' end

context 'other' do let(:facts) {{ :operatingsystem => 'xxx' }} it { expect { subject }.to raise_error(Puppet::Error, /Module nginx is not supported on xxx/) } endend

puppetlabs_spec_helper

Gemfile

source 'https://rubygems.org'

group :rake do gem 'puppet' gem 'rspec-puppet' gem 'rake' gem 'puppet-lint' gem 'puppetlabs_spec_helper'end

Rakefile

require 'puppetlabs_spec_helper/rake_tasks'

build # Build puppet module package

clean # Clean a built module package

coverage # Generate code coverage information

lint # Check puppet manifests with puppet-lint

spec # Run spec tests in a clean fixtures directory

spec_clean # Clean up the fixtures directory

spec_prep # Create the fixtures directory

spec_standalone # Run spec tests on an existing fixtures directory

spec/spec_helper.rb

require 'puppetlabs_spec_helper/module_spec_helper'

RSpec.configure do |c|

c.before(:each) do Puppet::Util::Log.level = :warning Puppet::Util::Log.newdestination(:console) end

end

.fixtures

# bring modules into spec/fixturesfixtures: repositories: firewall: "git://github.com/puppetlabs/puppetlabs-firewall" stdlib: repo: "git://github.com/puppetlabs/puppetlabs-stdlib" ref: "2.6.0" symlinks: my_module: "#{source_dir}"

librarian-puppet

Gemfile

source 'https://rubygems.org'

group :rake do gem 'puppet' gem 'rspec-puppet' gem 'rake' gem 'puppet-lint' gem 'puppetlabs_spec_helper' gem 'librarian-puppet-maestrodev'end

Puppetfile

forge 'http://forge.puppetlabs.com'

mod 'maestrodev/activemq', '>=1.0'mod 'saz/limits', ">=2.0.1"mod 'maestrodev/maestro_nodes', '>=1.1.0'mod 'maestrodev/maestro_demo', '>=1.0.2'mod 'maestrodev', :path => './private_modules/maestrodev'mod 'nginx', :git => 'https://github.com/jfryman/puppet-nginx.git'

Puppetfile.lock

FORGE remote: http://forge.puppetlabs.com specs: jfryman/nginx (0.0.2) puppetlabs/stdlib (>= 0.1.6) maestrodev/activemq (1.2.0) maestrodev/wget (>= 1.0.0) maestrodev/android (1.1.0) maestrodev/wget (>= 1.0.0) maestrodev/ant (1.0.4) maestrodev/wget (>= 0.0.1) maestrodev/archiva (1.1.0) maestrodev/wget (>= 1.0.0) maestrodev/git (1.0.1) maestrodev/jenkins (1.0.1) maestrodev/maestro (1.2.13) maestrodev/maven (>= 1.0.0) maestrodev/wget (>= 1.0.0) puppetlabs/postgresql (= 2.0.1) puppetlabs/stdlib (>= 2.5.1) maestrodev/maestro_demo (1.0.5) maestrodev/android (>= 1.1.0) maestrodev/maestro (>= 1.2.0) puppetlabs/postgresql (= 2.0.1) maestrodev/maestro_nodes (1.3.0) jfryman/nginx (>= 0.0.0) maestrodev/activemq (>= 1.0.0) maestrodev/ant (>= 1.0.3) maestrodev/archiva (>= 1.0.0) maestrodev/git (>= 1.0.0) maestrodev/jenkins (>= 1.0.0) maestrodev/maestro (>= 1.1.0) maestrodev/maven (>= 0.0.2) maestrodev/rvm (>= 1.0.0) maestrodev/sonar (>= 1.0.0) maestrodev/ssh_keygen (>= 1.0.0) maestrodev/statsd (>= 0.0.0) maestrodev/svn (>= 1.0.0)

puppetlabs/java (>= 0.3.0) puppetlabs/mongodb (>= 0.1.0) puppetlabs/nodejs (>= 0.3.0) puppetlabs/ntp (>= 0.0.0) stahnma/epel (>= 0.0.0) maestrodev/maven (1.1.2) maestrodev/wget (>= 1.0.0) maestrodev/rvm (1.1.5) maestrodev/sonar (1.0.0) maestrodev/maven (>= 0.0.2) maestrodev/wget (>= 0.0.1) puppetlabs/stdlib (>= 2.3.0) maestrodev/ssh_keygen (1.0.0) maestrodev/statsd (1.0.3) puppetlabs/nodejs (>= 0.2.0) maestrodev/svn (1.1.0) maestrodev/wget (1.2.0) puppetlabs/apt (1.2.0) puppetlabs/stdlib (>= 2.2.1) puppetlabs/firewall (0.4.0) puppetlabs/java (1.0.1) puppetlabs/stdlib (>= 0.1.6) puppetlabs/mongodb (0.1.0) puppetlabs/apt (>= 0.0.2) puppetlabs/nodejs (0.3.0) puppetlabs/apt (>= 0.0.3) puppetlabs/stdlib (>= 2.0.0) puppetlabs/ntp (1.0.1) puppetlabs/stdlib (>= 0.1.6) puppetlabs/postgresql (2.0.1) puppetlabs/apt (< 2.0.0, >= 1.1.0) puppetlabs/firewall (>= 0.0.4) puppetlabs/stdlib (< 4.0.0, >= 3.2.0) puppetlabs/stdlib (3.2.0) saz/limits (2.0.1) stahnma/epel (0.0.5)

GIT remote: https://github.com/jfryman/puppet-nginx.git ref: master sha: fd4e3c5a3719132bacabe6238ad2ad31fa3ba48c specs: nginx (0.0.2) puppetlabs/stdlib (>= 0.1.6)

PATH remote: ./private_modules/maestrodev specs: maestrodev (0.0.1)

DEPENDENCIES maestrodev (>= 0) maestrodev/activemq (>= 1.0) maestrodev/maestro_demo (>= 1.0.2) maestrodev/maestro_nodes (>= 1.1.0) nginx (>= 0) saz/limits (>= 2.0.1)

librarian-puppet

clean # Cleans out the cache and install paths.init # Initializes the current directoryinstall # Resolves and installs all of the dependencies you specifyoutdated # Lists outdated dependencies.package # Cache the puppet modules in vendor/puppet/cacheshow # Shows dependenciesupdate # Updates and installs the dependencies you specify

librarian-puppet for fixtures

# use librarian-puppet to manage fixtures instead of .fixtures.yml. Offers more possibilities like explicit version management, forge downloads,...

task :librarian_spec_prep do sh "librarian-puppet install --path=spec/fixtures/modules/"end

task :spec_prep => :librarian_spec_prep

.fixtures

fixtures: symlinks: my_module: "#{source_dir}"

Vagrant

tests/init.pp

stage { 'epel': before => Stage['rvm-install']}

class { 'epel': stage => 'epel' } ->class { 'rvm': }

Vagrantfile

Vagrant.configure("2") do |config|

config.vm.synced_folder ".", "/etc/puppet/modules/rvm"

# install the epel module needed for rvm in CentOS config.vm.provision :shell, :inline => "test -d /etc/puppet/modules/epel || puppet module install stahnma/epel -v 0.0.3"

config.vm.provision :puppet do |puppet| puppet.manifests_path = "tests" puppet.manifest_file = "init.pp" end

config.vm.define :centos63 do |config| config.vm.box = "CentOS-6.3-x86_64-minimal" config.vm.box_url = "https://repo.maestrodev.com/archiva/repository/public-releases/com/maestrodev/vagrant/CentOS/6.3/CentOS-6.3-x86_64-minimal.box" end

config.vm.define :centos64 do |config| config.vm.box = "CentOS-6.4-x86_64-minimal" config.vm.box_url = "https://repo.maestrodev.com/archiva/repository/public-releases/com/maestrodev/vagrant/CentOS/6.4/CentOS-6.4-x86_64-minimal.box" end

end

Rakefile

desc "Integration test with Vagrant"task :integration do sh %{vagrant destroy --force} sh %{vagrant up} sh %{vagrant destroy --force}end

Rakefile

# start one at a timedesc "Integration test with Vagrant"task :integration do sh %{vagrant destroy --force} ["centos63", "centos64"].each do |vm| sh %{vagrant up #{vm}} sh %{vagrant destroy --force #{vm}} end sh %{vagrant destroy --force}end

Blacksmith

gem 'puppet-blacksmith'

Rakefile

require 'puppet_blacksmith/rake_tasks'

Rake

module:bump # Bump module version to the next minormodule:bump_commit # Bump version and git commitmodule:clean # Runs clean againmodule:push # Push module to the Puppet Forgemodule:release # Release the Puppet module, doing a clean, build, tag, push, bump_commit and git pushmodule:tag # Git tag with the current module version

~/.puppetforge.yml

---forge: https://forge.puppetlabs.comusername: myusernamepassword: mypassword

just remember

create project in the Forge first(not yet implemented)

2.0.0 is built as a library to be reused

All together

Maven module

http://github.com/maestrodev/puppet-maven

Modulefile

name 'maestrodev-maven'version '1.1.3'

author 'maestrodev'license 'Apache License, Version 2.0'project_page 'http://github.com/maestrodev/puppet-maven'source 'http://github.com/maestrodev/puppet-maven'summary 'Apache Maven module for Puppet'description 'A Puppet module to download artifacts from Maven repositories'

dependency 'maestrodev/wget', '>=1.0.0'

Gemfile

source 'https://rubygems.org'

group :rake do gem 'puppet', '>=2.7.20' gem 'rspec-puppet', '>=0.1.3' gem 'rake', '>=0.9.2.2' gem 'puppet-lint', '>=0.1.12' gem 'puppetlabs_spec_helper' gem 'puppet-blacksmith', '>=1.0.5' gem 'librarian-puppet-maestrodev', '>=0.9.8'end

Rakefile

require 'bundler'Bundler.require(:rake)require 'rake/clean'

CLEAN.include('spec/fixtures/', 'doc', 'pkg')CLOBBER.include('.tmp', '.librarian')

require 'puppetlabs_spec_helper/rake_tasks'require 'puppet_blacksmith/rake_tasks'

task :librarian_spec_prep do sh "librarian-puppet install --path=spec/fixtures/modules/"endtask :spec_prep => :librarian_spec_prep

task :default => [:clean, :spec]

Rakefile (cont.)

desc "Integration test with Vagrant"task :integration do sh %{vagrant destroy --force} failed = [] ["centos64", "debian6"].each do |vm| sh %{vagrant up #{vm}} do |ok| if ok sh %{vagrant destroy --force #{vm}} else failed << vm end end end fail("Machines failed to start: #{failed.join(', ')}")end

.fixtures.yml

fixtures: symlinks: maven: "#{source_dir}"

Puppetfile

forge 'http://forge.puppetlabs.com'

mod 'maestrodev/wget', '>=1.0.0'

Integrating modules

modules

PREVIEW

code

DEV

DEMO

EVAL

CLIENT

modulesmodulesmodules

codecodecode

manifests

Automate!

librarian-puppet to fetch modulesVagrant box

Integration testscucumber

junitselenium

...

Vagrant integration tests

Use local Puppet files and modules

config.vm.share_folder "puppet", "/etc/puppet", ".", :create => true, :owner => "puppet", :group => "puppet"

Share logs

config.vm.share_folder "jenkins-logs", "/var/log/jenkins", "target/logs/jenkins", :create => true, :extra => "dmode=777,fmode=666"

Save downloaded files in host

config.vm.share_folder "repo2", "/var/lib/jenkins/.m2/repository",

File.expand_path("~/.m2/repository"), :extra => "dmode=777,fmode=666"

config.vm.share_folder "yum", "/var/cache/yum", File.expand_path("~/.maestro/yum"), :owner => "root", :group => "root

Provision

config.vm.provision :puppet do |puppet| puppet.manifests_path = "manifests" puppet.manifest_file = "site.pp" puppet.pp_path = "/etc/puppet" puppet.options = ["--verbose"] puppet.facter = {} end

Run!

vagrant destroy --force vagrant uprake integration

Forward looking

Auto update

Automatically update all the modules and tell me if it’s broken

bonus point: automatically edit the Gemfile, Puppetfile, Modulefile constraints

csanchez@maestrodev.comcarlos@apache.org@csanchez

Thanks!

http://csanchez.orghttp://maestrodev.com

Photo Credits

Brick wall - Luis Argerichhttp://www.flickr.com/photos/lrargerich/4353397797/

Agile vs. Iterative flow - Christopher Littlehttp://en.wikipedia.org/wiki/File:Agile-vs-iterative-flow.jpg

DevOps - Rajiv.Panthttp://en.wikipedia.org/wiki/File:Devops.png

Pimientos de Padron - Howard Walfishhttp://www.flickr.com/photos/h-bomb/4868400647/

Compiling - XKCDhttp://xkcd.com/303/

Printer in 1568 - Meggs, Philip Bhttp://en.wikipedia.org/wiki/File:Printer_in_1568-ce.png

Relativity - M. C. Escherhttp://en.wikipedia.org/wiki/File:Escher%27s_Relativity.jpg

Teacher and class - Herald Posthttp://www.flickr.com/photos/heraldpost/5169295832/