Index: trunk/server/doc/install-howto.sh
===================================================================
--- trunk/server/doc/install-howto.sh	(revision 2111)
+++ trunk/server/doc/install-howto.sh	(revision 2112)
@@ -128,4 +128,7 @@
     systemctl enable syslog-ng.service
 
+# Temporarily remove ghc-cgi-devel and ghc-cgi
+    rpm -e --nodeps ghc-cgi-devel ghc-cgi
+
 # Install the full list of RPMs that users expect to be on the
 # scripts.mit.edu servers.
@@ -156,4 +159,10 @@
 #                      SPHEROID SHENANIGANS
 
+# Install the Python eggs and Ruby gems and PEAR/PECL doohickeys that are on
+# the other scripts.mit.edu servers and do not have RPMs.
+# The general mode of operation will be to run the "list" command
+# on both servers, see what the differences are, check if those diffs
+# are packaged up as rpms, and install them (rpm if possible, native otherwise)
+
 # Note: Since ultimately we'd like to move away from using per-language
 # package manager and all of these be RPMs, it is of questionable
@@ -163,4 +172,21 @@
 # packaged up the RPM.  Unfortunately we don't really have good incants
 # for this.
+
+# Warning: If you're installing a new server mid-lifecycle (or even if
+# this is the start of a cycle, but you've been staggering the
+# installation of servers), upstream may have moved on.  Because we
+# don't normally upgrade spheroid projects, that means executing these
+# instructions directly means that you will have mismatched versions
+# (the new servers will have newer versions.)  Please follow the
+# UPGRADE commentary attached to each of these.
+
+# Warning: The package lists that are generated are inconsistent on
+# the question of whether or not they contain all packages (locally
+# installed as well as distro packaged), or if they just contain locally
+# installed packages.  Check this carefully; many of the install incants
+# filter out already installed packages.
+
+# PERL CPAN
+# ---------
 
 # Install the full list of perl modules that users expect to be on the
@@ -174,11 +200,20 @@
 perldoc -u perllocal | grep head2 | cut -f 3 -d '<' | cut -f 1 -d '|' | sort -u | perl -ne 'chomp; print "notest install $_\n" if system("rpm -q --whatprovides \"perl($_)\" >/dev/null 2>/dev/null")' > perl-packages.txt
 # arrange for perl-packages.txt to be transferred to server
+    # Package list only contains new packages
     cat perl-packages.txt | perl -MCPAN -e shell
-
-# Install the Python eggs and Ruby gems and PEAR/PECL doohickeys that are on
-# the other scripts.mit.edu servers and do not have RPMs.
-# The general mode of operation will be to run the "list" command
-# on both servers, see what the differences are, check if those diffs
-# are packaged up as rpms, and install them (rpm if possible, native otherwise)
+# These are in /usr/local
+
+# UPGRADE: Installing old versions of CPAN modules requires you to
+# specify the full path of a module, e.g.
+# M/MS/MSCHWERN/Test-Simple-0.62.tar.gz.  It is not currently clear how
+# to get this information programatically.  Furthermore, we have a lot
+# of CPAN managed modules.  Since CPAN is the only thing
+# placed in /usr/local at this point, it may be easier to simple tar and
+# cp the Perl modules from one server to another, to keep them
+# consistent.  But doing this is fiddly XXX
+
+# PYTHON EGGS
+# -----------
+
 # - Look at /usr/lib/python2.7/site-packages and
 #           /usr/lib64/python2.7/site-packages for Python eggs and modules.
@@ -192,5 +227,16 @@
 # 'easy_install AuthKit jsonlib2 pygit'
 cat /usr/lib/python2.7/site-packages/easy-install.pth | grep "^./" | cut -c3- | cut -f1 -d- > egg.txt
+    # Package list only contains new packages
     cat egg.txt | xargs easy_install -Z
+# These are in /usr
+
+# UPGRADE: Use 'easy_install -n' to see what new versions are installed, and if there
+# are updates validate them and upgrade them on the old servers.  Since
+# we have a really small package list (around 4) checking these manually
+# should be fine.  Note that dry run is slightly buggy and may fail
+# midway processing files on account of a missing build directory.
+
+# RUBY GEMS
+# ---------
 
 # - Look at `gem list` for Ruby gems.
@@ -200,7 +246,31 @@
 # XXX This doesn't do the right thing for old version gems
 gem list --no-version > gem.txt
+    # Package list contains distro gems too
     gem install $(gem list --no-version | grep -Fxvf - gem.txt)
     # Also, we need to install the old rails version
     gem install -v=2.3.5 rails
+# These are in /usr
+
+# UPGRADE:  You can either upgrade out-of-date gems, or leave them at
+# the old version.  We recommend the latter (see below for the
+# rationale), but note that the install script described here doesn't
+# pin against version, so you'll need to supply the -v parameters
+# manually (the gems we install manually don't move too quickly, so this
+# is fairly tractable if you check 'gem outdated'.)
+#
+# If you want to upgrade, do NOT use wildcard 'gem update'; use 'gem
+# outdated' to find out all gems that are out of date, and verify this
+# against our locally installed gems (there will be a lot of out of date
+# gems, but this is simply because Fedora packaging lags behind the
+# canonical versions (this is a good thing).  Manually upgrade just
+# those gems.  Note that this doesn't save you from having to install
+# old gems on the servers that are being installed out-of-cycle,
+# because Ruby supports pinning against old versions, and if those gems
+# then mysteriously disappear, things will be sad (note that this isn't
+# a *huge* problem, because usually when you pin gems it's in
+# conjunction with rvm, so they have their local copy of the gem.)
+
+# PHP PEAR
+# --------
 
 # - Look at `pear list` for Pear fruits (or whatever they're called).
@@ -209,7 +279,12 @@
 #   (you might get complaints about the php_scripts module; ignore them)
 pear list | tail -n +4 | cut -f 1 -d " " > pear.txt
+    # Package list contains distro packages
     pear config-set preferred_state beta
     pear channel-update pear.php.net
     pear install $(pear list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pear.txt)
+# These are in /usr
+
+# PHP PECL
+# --------
 
 # - Look at `pecl list` for PECL things.  'yum search', and if you must,
@@ -217,13 +292,20 @@
 #   pecl/foo' or 'pecl install foo-beta' or those two combined.
 pecl list | tail -n +4 | cut -f 1 -d " " > pecl.txt
+    # Package list contains distro packages
     pecl install --nodeps $(pecl list | tail -n +4 | cut -f 1 -d " " | grep -Fxvf - pecl.txt)
+# These are in /usr
 
 # ----------------------------->8--------------------------------------
 #                       INFINITE CONFIGURATION
 
-# Create fedora-ds user (needed for credit-card)
+# [PROD] Create fedora-ds user (needed for credit-card)
 useradd -u 103 -r -d /var/lib/dirsrv fedora-ds
 
 # Run credit-card to clone in credentials and make things runabble
+# NOTE: You may be tempted to run credit-card earlier in the install
+# process in order, for example, to be able to SSH in to the servers
+# with Kerberos.  However, it is better to install the credentials
+# *after* we have run a boatload untrusted code as part of the
+# spheroids objects process.  So don't move this step earlier!
 python host.py push $server
 
@@ -238,22 +320,20 @@
 #   #   [TESTSERVER] daemon.scripts-test
 
+# Test that zephyr is working
+    systemctl enable zhm.service
+    systemctl start zhm.service
+    echo 'Test!' | zwrite -d -c scripts -i test
+
+# Check out the scripts /usr/vice/etc configuration
+    cd /root/vice
+    \cp -a etc /usr/vice
 # [TESTSERVER] If you're installing a test server, this needs to be
 # much smaller; the max filesize on XVM is 10GB.  Pick something like
 # 500000. Also, some of the AFS parameters are kind of retarded (and if
 # you're low on disk space, will actually exhaust our inodes).  Edit
-# these parameters in /etc/sysconfig/openafs (This doesn't work in the
-# new systemd world order: try editing the unit file instead.)
+# these parameters in /etc/sysconfig/openafs (I just chopped a zero
+# off of all of our parameters)
     echo "/afs:/usr/vice/cache:500000" > /usr/vice/etc/cacheinfo
-# XXX This is out of date in the systemd world.
     vim /etc/sysconfig/openafs
-
-# Test that zephyr is working
-    systemctl enable zhm.service
-    systemctl start zhm.service
-    echo 'Test!' | zwrite -d -c scripts -i test
-
-# Check out the scripts /usr/vice/etc configuration
-    cd /root/vice
-    \cp -a etc /usr/vice
 
 # [PRODUCTION] Set up replication (see ./install-ldap).
@@ -365,6 +445,7 @@
 #     or do SSL.  Generate with:
     openssl req -new -x509 -keyout /etc/pki/tls/private/scripts.key -out /etc/pki/tls/certs/scripts.cert -nodes
-#     Also make /etc/pki/tls/certs/ca.pem match up (XXX what's the
-#     incant for that?)
+    ln -s /etc/pki/tls/private/scripts.key /etc/pki/tls/private/scripts-1024.key
+#     Also make /etc/pki/tls/certs/ca.pem match up
+    openssl rsa -in /etc/pki/tls/private/scripts.key -pubout > /etc/pki/tls/certs/ca.pem
 
 # [TESTSERVER] More stuff for test servers