#!/opt/bin/python
# alternately
# /bin/sh
# PATH="$PATH:/opt/bin" env python - <<EIEIEOF
"""
$Id: SdiMaint.py,v 1.25.2.1 2007/11/27 17:49:34 rayg Exp $
Copyright 2006, University of Wisconsin Space Science & Engineering Center

Principal Author: Ray Garcia <rayg@ssec.wisc.edu>

This is a library of update and maintenance functions for the SDI-104 system.

Current functionality includes:
    - locate packges in a directory, or a .tar file (DirWithPackages, TarWithPackages)
    - compare package revisions (sdiMergePackageLists)
    - identify ingestor satellite type (sdiIngestorType)
    - selectively update 
    - incremental update of compact flash memory (PartitionTarget)
    - incremental update of RAMdisk running image (RootTarget)
    - substantial error checking and safety
    - cross-grade from one satellite type to another, if multiple are available
    - simple dependency tree for packages
    
Features not yet implemented:
    - use optarg or optparse to provide command line options
    - e-mail notification of configuration change
    - selective downgrade package replacement
    - download updates from WWW
    - SSH key management
    - system configuration summary e-mail
    - improved logging
    - system reinstallation (currently handled by sdi104-updateusb script)
    
The default operating mode is to look for removable disks on the system (USB drives), 
prompt the user for a removable device or directory name holding a cache of new packages,
create an update plan, allow the user to review the update plan, and execute the update 
plan on memory and compact flash.


TEST_ROOT='test_data/base' TEST_TGT=test_data/base TEST_SRC=test_data/update3 TEST_RUNLEVEL=3 python SdiMaint.py
TEST_ROOT='test_data/base' TEST_TGT=test_data/base TEST_SRC=test_data/update4 TEST_RUNLEVEL=3 python SdiMaint.py
TEST_ROOT='test_data/base' TEST_TGT=test_data/base TEST_SRC=test_data/update5 TEST_RUNLEVEL=3 python SdiMaint.py

"""
#####################################
# python standard library imports

import logging
import os,sys,time,re,sets,glob
import shutil,subprocess,md5
import tarfile
from exceptions import *

#####################################
# globals

LOG_FILENAME_FORMAT='/tmp/SdiMaint-%Y%m%d%H%M%S.log'
TMP_DIR='/tmp' # directory to put temporary files
MNT_DIR='/mnt' # directory to put temporary mount points

# root logger instance for all to use
log = logging.getLogger()

env = {}

# break package name into base (e.g. adde-gvar) and version (e.g. 20060412)
re_pkgname = re.compile(r'([-\.\w]*?)-(\d+[a-z]?)\.tar\.gz')
# a package bunch
# i.e. a tar file holding a bunch of .tar.gz files
re_tarname = re.compile(r'([-\w]*?)-(\d+[a-z]?)\.tar')
# a flag file present in a directory, is_______
# used to flag system packges in a system-YYYYMMDD.tar with isSystem
re_flagname = re.compile(r'is(\w+)')

re_removable_partition = re.compile(r'^\s*\d+\s+\d+\s+(\d+)\s(sd\w\d+)',re.MULTILINE)


# regex matching inge-____-*.tar.gz or adde-____-*.tar.gz - SDI-104 specific
re_sat_specific_pkg = re.compile( r'(?P<app>inge|adde)-(?P<sat>[-\w]+)' )
# re_adde_pkg = re.compile( r'adde-(?P<sat>[-\w]+)' )
# re_inge_pkg = re.compile( r'inge-(?P<sat>[-\w]+)' )


FLAG_SYSTEM = 'System'


# workarounds
try:
    import zlib
    NOZLIB = False
except:
    NOZLIB = True


#####################################
# utility wrappers

class ExternCommandError(EnvironmentError):
    pass

def _system(cmd):
    pip = subprocess.Popen( cmd, stdout=subprocess.PIPE )
    output,error = pip.communicate()
    rc = pip.wait()
    if rc != 0: raise ExternCommandError('bad result code from "%s"' % ' '.join(cmd))
    return output,error

def mount(dev,mnt,writable=False,remount=False):
    opts = [ writable and 'rw' or 'ro' ]
    if remount: opts.append('remount')
    cmd = ['mount',dev,mnt,'-o',','.join(opts)]
    return _system(cmd)

def remount(mnt,writable=False):
    opts = [ 'remount', writable and 'rw' or 'ro' ]
    cmd = ['mount',mnt,'-o',','.join(opts)]
    return _system(cmd)

def umount(mnt):
    return _system(['umount',mnt])
    
def sync():
    return _system(['sync'])
    
def runlevel():
    test_runlevel = os.environ.get('TEST_RUNLEVEL',None)
    if test_runlevel: return int(test_runlevel)
    
    out,err = _system(['/sbin/runlevel'])
    n,rlev = out.split(' ')
    return int(rlev)

def removable_devices():
    """return [ (blocks,devname), (blocks,devname).. ] for /dev/sd[a-z]#
    """
    prt = file('/proc/partitions','r').read()
    return re_removable_partition.findall(prt)

def targz_filelist(tarfilename):    
    """return list of files in a tar.gz file
    """
    if NOZLIB: # prefer not doing it this way
        tgz = _system( ['tar', 'ztf', tarfilename] )
        return tgz[0].split()
    else:
        tf = tarfile.open(tarfilename,"r:gz")
        tall = tf.getmembers()
        return [ x.name for x in tall ]

def targz_unpack(tarfilename,tgt_dir):
    if NOZLIB:
        _system( ['tar', 'zxf', tarfilename, '-C', tgt_dir ] )
    else:
        tf = tarfile.open(tarfilename,"r:gz")
        for each in tf.getmembers():
            tf.extract(each,tgt_dir)


#####################################
# classes

def _version_val(sval): # version value for a string version number
    if sval[-1].isalpha(): return float(sval[0:-1]) + float(ord(sval[-1].lower())-ord('a')+1)/27.0
    else: return int(sval)

class Package(object):
    """A simple package base class
    """
    def __init__(self,name,version):
        self._name = name
        self._version = version
        self._flags = []
    def setFlags(self,flags):
        self._flags = flags
    def flags(self):
        return self._flags
    def name(self):
        return self._name
    def split(self):
        "split into 3 components - base, satellite (or None), version-value"
        vv = _version_val(self._version)
        missat = re_sat_specific_pkg.match(self._name)
        if not missat: return self._name, None, vv
        D = missat.groupdict()
        return D['app'], D['sat'], vv
    def version(self):
        return self._version
    def value(self):
        return _version_val(self._version)
    def __repr__(self):
        return '<Package %(_name)s version %(_version)s flags %(_flags)s>' % self.__dict__

class FilePackage(Package):
    """A simple package file, .tar.gz
    """
    def __init__(self,path,name=None,version=None):
        if not name or not version:
            name,version = re_pkgname.match(os.path.split(path)[1]).groups()
        Package.__init__(self,name,version)
        self._path = path
        _,self._filename = os.path.split(path)
        self._filelist = targz_filelist(self._path)
    def __repr__(self):
        return '<FilePackage %(_name)s version %(_version)s flags %(_flags)s from %(_path)s>' % self.__dict__
    def path(self):
        return self._path
    def files(self):
        return self._filelist
    def copyToDir(self,tgt_dir):
        """write the file to a target directory
        returns the new package resulting
        """
        assert(os.path.isfile(self._path))
        dn,fn=os.path.split(self._path)
        newpath = os.path.join(tgt_dir,fn)
        shutil.copy2(self._path,newpath)
        return FilePackage(newpath,self._name,self._version)
    def unpackToDir(self,tgt_dir):
        targz_unpack(self._path,tgt_dir)

class TarredPackage(Package):
    """A .tar.gz package file inside a .tar file
    """
    def __init__(self,tar,path,name=None,version=None):
        if not name or not version:
            name,version = re_pkgname.match(os.path.split(path)[1]).groups()
        Package.__init__(self,name,version)
        self._path = path
        _,self._filename = os.path.split(path)
        self._tarpath = tar.name
        _,self._tarname = os.path.split(self._tarpath)
        self._tar = tar        
    def __repr__(self):
        return '<TarredPackage %(_name)s version %(_version)s flags %(_flags)s in %(_path)s>' % self.__dict__
    def copyToDir(self,tgt_dir):
        """write the file to a target directory
        returns resultant package
        """
        self._tar.extract(self._path,tgt_dir)
        fn = os.path.split(self._path)[1]
        return FilePackage(os.path.join(tgt_dir,fn),self._name,self._version)
    def files(self):
        log.debug("temporarily copying to tmp: %s" % self)
        fpkg = self.copyToDir(TMP_DIR)
        #all = fpkg.files()
        os.unlink(fpkg.path())
        return all
    def unpackToDir(self,tgt_dir):
        log.debug("temporarily copying to tmp for extraction: %s" % self)
        fpkg = self.copyToDir(TMP_DIR)
        fpkg.unpackToDir(tgt_dir)
        os.unlink(fpkg.path())
        
    
class TarWithPackages(object):
    """A tar file containing multiple packages
    """
    def __init__(self,path):
        self._tarfile = tarfile.open(path,'r')
        self._path = path
    def findPackages(self):
        tall = self._tarfile.getmembers()
        packages = []
        flags = []
        for path in [ x.name for x in tall if x.isfile() ]:
            base,fn = os.path.split(path)
            f = re_flagname.match(fn)
            m = re_pkgname.match(fn)
            if m:
                pkgname, pkgversion = m.groups()
                packages.append(TarredPackage(self._tarfile,path,pkgname,pkgversion))
                log.info("found package %s version %s in %s" % (pkgname,pkgversion,path))
            elif f:
                flags += f.groups()
            else:
                log.debug("file %s not a package" % path)        
        for each in packages:
            each.setFlags(flags)
        return packages
    def __iter__(self):
        tall = self.findPackages()
        for each in tall: yield each
    def __repr__(self):
        return "<TarWithPackages\n" + '\n'.join(['    %s' % x for x in self.findPackages()]) + '\n>\n'

class DirWithPackages(object):
    """A directory tree holding one or more packages
    """
    def __init__(self,path,trashpath=None):
        self._path = path
        self._trashpath = trashpath
        if not os.path.isdir(self._path): os.makedirs(self._path)
        if self._trashpath and not os.path.isdir(self._trashpath): os.makedirs(self._trashpath)
    def findPackages(self):
        packages = []
        flags = []
        for (base,subdirs,files) in os.walk(self._path):
            for fn in files:
                path = os.path.join(base,fn)
                m = re_pkgname.match(fn)
                tm = re_tarname.match(fn)
                f = re_flagname.match(fn)
                if m:
                    pkgname, pkgversion = m.groups()
                    packages.append(FilePackage(path,pkgname,pkgversion))
                    log.info("found package %s version %s in %s" % (pkgname,pkgversion,path))
                elif tm:
                    packages += TarWithPackages(path).findPackages()
                elif f:
                    flags += f.groups()
                else:
                    log.debug("file %s not a package" % path)
        for each in packages:
            each.setFlags(flags)
        return packages
    def __iter__(self):
        tall = self.findPackages()
        for each in tall: yield each
    def __repr__(self):
        return "<DirWithPackages\n" + '\n'.join(['    %s' % x for x in self.findPackages()]) + '\n>\n'
    def removePackage(self,pkg):
        log.info("removing %s version %s from %s" % (pkg.name(),pkg.version(),self._path))
        if not self._trashpath: 
            log.info("unlinking %s" % pkg.path())
            os.unlink(pkg.path())
        else: 
            fn_pkg = os.path.split(pkg.path())[1]
            newpath = os.path.join(self._trashpath,fn_pkg)
            log.info("moving %s to %s" % (pkg.path(),newpath))
            shutil.move(pkg.path(),newpath)
    def addPackage(self,pkg):
        log.info("adding %s version %s to %s" % (pkg.name(),pkg.version(),self._path))
        pkg.copyToDir(self._path)

class DevSource(DirWithPackages):
    """A directory or mount point holding a set of packages, e.g. mounted USB key or remounted CF disk
    """
    def __init__(self,devpath,mntpath=None):
        self._mnt = mntpath or os.tempnam('/mnt')
        if not os.path.isdir(self._mnt): os.makedirs(self._mnt)
        mount(devpath,self._mnt)
        log.debug("mounted %s to %s" % (devpath,self._mnt))
        DirWithPackages.__init__(self,self._mnt)
    def __del__(self):
        log.debug("unmounting %s" % self._mnt)
        umount(self._mnt)
        os.rmdir(self._mnt)        

class UrlSource(object):
    """A URL containing downloadable package information
    """
    pass
    
    
class RootTarget(object):
    """The active root filesystem
    """
    def __init__(self,root='/',keys='root/.ssh/authorized_keys2'):
        self._root = root
        self._keypath=os.path.join(root,keys)
    def _purgeFiles(self,filelist):
        for filename in filelist:
            path = os.path.join(self._root,filename)
            if not os.path.islink(path) or (os.path.exists(path) and not os.path.isdir(path)):
                log.debug("removing package file %s" % path)
                os.unlink(path)
            else:
                log.info("%s does not exist or is a directory, will not remove it" % path)
    def getKeyRing(self):
        return SshKeyRing(self._keypath)
    def removePackage(self,pkg):
        log.info("removing files from %s version %s" % (pkg.name(),pkg.version()))
        self._purgeFiles(pkg.files())
    def addPackage(self,pkg):
        log.info("adding files from %s version %s" % (pkg.name(),pkg.version()))
        pkg.unpackToDir(self._root)    
    
class PartitionTarget(DirWithPackages):
    """A DOS partition holding the packages/ and other folders for the system
    holds active packages in a packages/ subdir, and moves old packages to packages.1/
    """
    def __init__(self,dev=None,mnt=None,pkgdir="packages",trashdir="packages.1",keys='root/.ssh/authorized_keys2'):
        self._mnt = mnt or os.tempnam(MNT_DIR)
        self._pkgdir = pkgdir
        self._trashdir = trashdir
        self._keypath = os.path.join(self._mnt,keys)
        ok = False
        try:
            remount(self._mnt,writable=True)
            ok = True
            self._remounted = True
        except:
            pass
        if not ok:
            mount(dev,self._mnt,writable=True)
            ok = True
            self._remounted = False
        DirWithPackages.__init__(self,os.path.join(self._mnt,self._pkgdir),self._trashdir and os.path.join(self._mnt,self._trashdir) or None)
        if not ok:
            raise EnvironmentError('partition %s could not be mounted' % self._mnt)
            self._mnt = None
    def getKeyRing(self):
        return SshKeyRing(self._keypath)
    def __del__(self):
        if self._remounted: 
            log.info('remounting %s read-only' % self._mnt)
            remount(self._mnt,writable=False)
        else: 
            log.info('unmounting %s' % self._mnt)
            umount(self._mnt)


class SshKeyRing(object):
    def __init__(self,auth_keys_path):
        self._ring = sets.Set( x.strip() for x in file(auth_keys_path,'rt').readlines() )
        self._path=auth_keys_path
    def __repr__(self):
        return '<SSH file %s with public keys: %s>' % (self._path,' '.join( x.split()[3] for x in self._ring) )
    def save(self,path=None):
        if not path: path = self._path
        file(path,'wt').write( '\n'.join(self._ring) + '\n' )
    def add(self,keystring):
        self._ring.add(keystring.strip()) 
    def remove(self,keystring):
        self._ring.remove(keystring.strip())       


#####################################
# SDI-104 Specific Knowledge Base

class requires(object):
    package = None
    satellite = None
    base = None
    base_satellite = None
    newer_than = None
    
    def __init__(self,base,base_satellite=None,package=None,satellite=None,newer_than=None):
        "note that newer_than should be a number and not a string"
        self.__dict__.update(locals())
        
    def __repr__(self):
        name = lambda b,s: s and ('%s-%s' % (b,s)) or b
        base = name(self.base,self.base_satellite)
        need = name(self.package,self.satellite)
        newer = self.newer_than and (' >=%f' % self.newer_than) or ''
        return "<requires: package %s depends on %s%s>" % (base,need,newer)
    
    def pick(self,available):
        "from available packages, list all that fit dependencies"
        ret = []
        for each in available:
            app,sat,ver = each.split()
            if app==self.package and sat==self.satellite:
                if not self.newer_than or ver>=self.newer_than: ret.append(each)
        return ret
        
    def applies_to(self,pkg):
        "return boolean whether this rule applies to this package"
        app,sat,ver = pkg.split()
        return (app==self.base) and (self.base_satellite==sat)
        
            
# Simple, non-recursive dependency list. This is mostly needed when we change satellites.
# FUTURE: add a dependency ruleset to invididual packages.
SDI_PACKAGE_DEPENDENCIES = [ 
    requires('adde','gvar',package='libg2c'),
    requires('adde','poes',package='libg2c'),
    requires('adde','mtsat-hirid',package='libg2c'),
    requires('inge','gvar',package='libg2c'),
    requires('inge','poes',package='libg2c'),
    requires('inge','mtsat-hirid',package='libg2c'),
    requires('inge','mtsat-hrit',package='jre1.5_jai')
    ]
    
# The principal directory where user configuration file are kept. 
# When we change satellites, most of the files in this directory get moved to a save directory.
SDI_DATA_OPT_ETC = '/mnt/HDD/data/opt/etc'

# These globs are used to identify file list of data files to purge when changing satellite types.
SDI_PURGE_DATA_GLOBS = ['/mnt/HDD/data/*','/mnt/HDD/data/sounder/*', '/mnt/HDD/data/area/*', '/mnt/HDD/data/l1b/*']


def sdiAddPackageDependencies(installed, to_install, available, deps=SDI_PACKAGE_DEPENDENCIES):
    """Given a list of packages to be installed, 
    returns [packages-installed-which-are-needed],[needed-packages-to-pick-newest-from]
    returns None,None if a dependency can't be met! 
    packages already scheduled to be installed take priority over anything in the available list,
        though under normal usage the to_install is a subset of the available

    >>> tgt = DirWithPackages('test_data/base')
    >>> src = sdiNewestPackages(DirWithPackages('test_data/update4').findPackages())
    >>> src2 = sdiNewestPackages(DirWithPackages('test_data/update5').findPackages())
    >>> print sdiAddPackageDependencies(tgt.findPackages(), src, src)
    (None, None)
    >>> print sdiAddPackageDependencies(tgt.findPackages(), src, src2)
    ([], [<FilePackage jre1.5_jai version 20061104 flags [] from test_data/update5/jre1.5_jai-20061104.tar.gz>])
    """
    must_keep = []  # these packages must not be removed from the system
    install_newest_from = []
    for pkg in to_install:
        for rule in deps: 
            if rule.applies_to(pkg):
                # make lists of candidate packages to use from different locations
                candi = rule.pick(installed)
                canda = rule.pick(available)
                candt = rule.pick(to_install)                
                if not candi and not canda and not candt: 
                    log.error("Cannot satisfy dependency: %s" % rule)
                    return None,None
                elif canda: 
                    # see if we already have it in the to_install list
                    if not candt:
                        log.info('satisfying dependency %s with new package/s %s' % (rule,canda))
                        install_newest_from += canda
                    else:
                        log.info('dependency %s satisfied with package/s %s' % (rule,candt))
                        must_keep += candt
                else: 
                    log.debug('dependency %s satisfied by installed package/s %s' % (rule,candi))
                    must_keep += candi
    return must_keep, install_newest_from

def sdiIngestorType(pkglist):
    """ returns single-type-if-exists, dictionary of (satellite,list-of-packages-for-satellite) entries
    satellite is "gvar", "mtsat", other satellite type or None
    single-type-if-exists is 

    >>> tgt = DirWithPackages('test_data/base')
    >>> src = DirWithPackages('test_data/update1')
    >>> print sdiIngestorType(tgt.findPackages())
    ('poes-relay', {'poes-relay': [<FilePackage adde-poes-relay version 20061012 flags [] from test_data/base/adde-poes-relay-20061012.tar.gz>, <FilePackage inge-poes-relay version 20061127 flags [] from test_data/base/inge-poes-relay-20061127.tar.gz>]})
    >>> print sdiIngestorType(src.findPackages())
    ('gvar', {'gvar': [<FilePackage adde-gvar version 20060317 flags [] from test_data/update1/adde-gvar-20060317.tar.gz>, <FilePackage inge-gvar version 20060314 flags [] from test_data/update1/inge-gvar-20060314.tar.gz>]})
    >>> print sdiIngestorType(DirWithPackages('test_data/update2').findPackages())
    ('gvar', {'gvar': [<FilePackage adde-gvar version 20060410 flags [] from test_data/update2/adde-gvar-20060410.tar.gz>, <TarredPackage adde-gvar version 20060317 flags [] in adde-gvar-20060317.tar.gz>]})
    """
    tall = {}
    for each in pkglist: 
        matchlist = re_sat_specific_pkg.findall(each.name())
        for app,sat in matchlist:
            log.debug("found application %s for satellite %s" % (app,sat))
            L = tall.get(sat,[])
            L.append(each)
            tall[sat] = L
    single = (len(tall)==1) and tall.keys()[0] or None
    return single,tall

def _sdiAllSatSpec(instdict):
    """ return remove-list and app-name list for anything satellite-specific
    """
    to_remove = []
    apps = []
    for name,pkg in instdict.items():
        missat = re_sat_specific_pkg.match(name)
        if not missat: continue
        to_remove.append(pkg)
        apps.append( missat.groupdict()['app'] )
    return to_remove,apps

def _sdiMergeOrdinaryPkg(instdict,pkg):
    """ returns to_remove list,to_add list
    """
    remo = instdict.get(pkg.name(),None)
    if not remo:
        log.info("ignoring %s version %s, no previous version installed" % (pkg.name(),pkg.version()))
        return [],[]
    else:
        if remo.value() < pkg.value():
            log.info("replacing %s version %s with version %s" % (pkg.name(),remo.version(),pkg.version()))
            return [remo],[pkg]
        else:
            log.debug("%s version %s is older than installed version %s" % (pkg.name(),pkg.version(),remo.version()) )
            return [],[]

def _sdiMergeSatSpecPkg(satellite,instdict,pkg,redict):
    """ returns satellite,to_remove list,to_add list
    """
    if satellite and satellite != redict['sat']:
        log.info("%s version %s is wrong satellite type (%s, want %s) - ignoring" % (pkg.name(),pkg.version(),redict['sat'],satellite))
        return satellite,[],[]
    elif not satellite:
        log.warning("%s version %s will be installed, establishing system as a %s ingestor" % (pkg.name(),pkg.version(),redict['sat']))
        return redict['sat'],[],[pkg]
    elif satellite and satellite==redict['sat']:
        if instdict.get(pkg.name(),None): 
            return (satellite,) + _sdiMergeOrdinaryPkg(instdict,pkg)
        else: # not already installed, install it fresh
            return satellite,[],[pkg]

def sdiNewestPackages(available):
    """find newest versions of the packages in the input list, returning revised list
    
    >>> print sdiNewestPackages(DirWithPackages('test_data/update2').findPackages())
    [<FilePackage adde-gvar version 20060410 flags [] from test_data/update2/adde-gvar-20060410.tar.gz>, <FilePackage flibber version 20060421 flags [] from test_data/update2/flibber-20060421.tar.gz>, <FilePackage libg2c version 20060317 flags [] from test_data/update2/libg2c-20060317.tar.gz>, <TarredPackage ntp version 20060309a flags [] in ntp-20060309a.tar.gz>]
    >>> print sdiNewestPackages(DirWithPackages('test_data/update3').findPackages())
    [<FilePackage new version 20060508 flags [] from test_data/update3/new-20060508.tar.gz>, <FilePackage adde-mtsat-hirid version 20060614 flags [] from test_data/update3/adde-mtsat-hirid-20060614.tar.gz>, <FilePackage inge-mtsat-hirid version 20060614 flags [] from test_data/update3/inge-mtsat-hirid-20060614.tar.gz>]
    """
    dnu = {}
    for each in available:
        name = each.name()
        cur = dnu.get(name,None)
        if (not cur) or (cur.value() < each.value()):
            dnu[name] = each
    return dnu.values()
    
def _sdiNonRemovableConfigFiles(target=SDI_DATA_OPT_ETC,etc='/etc'):
    """Files which are in the user configuration directory that are linked into /etc 
    """
    etc_files = [os.path.split(x)[1] for x in glob.glob(os.path.join(etc,'*')) if os.path.islink(x) and os.readlink(x)[0:len(target)]==target]
    # these files cannot be softlinked and as such have to be hardcoded
    etc_files += [x for x in ['rc.local','passwd','shadow'] if os.path.isfile(os.path.join(target,x))]
    return etc_files

def sdiPlanPurge(satellite_being_removed=None,data_globs=SDI_PURGE_DATA_GLOBS,data_opt_etc=SDI_DATA_OPT_ETC,etc='/etc'):
    """return (to-remove-list,to-archive-and-remove-list,where-to-archive-to) 
    """
    # remove all files in /data that aren't directories
    from operator import add
    # get a list of all the directory entries in the data_dirs
    tall = reduce(add, [glob.glob(dg) for dg in data_globs], [])
    # reduce it just to files
    data_files = [x for x in tall if os.path.isfile(x) or os.path.islink(x)]
    nonremovable_config_files = _sdiNonRemovableConfigFiles(data_opt_etc,etc)
    config_files = [x for x in glob.glob(os.path.join(data_opt_etc,'*')) if os.path.isfile(x) and os.path.split(x)[1] not in nonremovable_config_files]    
    arch_dir = ''
    n = 0
    while not arch_dir or os.path.exists(arch_dir):
        arch_dir = os.path.join(data_opt_etc, 'saved_'+time.strftime('%Y%m%d')) + '_%02d'%n
        n += 1
    return data_files,config_files,arch_dir

def executePurgePlan(to_remove,to_archive,arch_dir,dryrun=False):
    os.makedirs(arch_dir)
    from pprint import pprint
    log.info('purging %d files and moving %d config files to %s' % (len(to_remove),len(to_archive),arch_dir))
    if dryrun: 
        link=lambda x,y: pprint("%s -> %s" % (x,y))
        unlink=lambda x: pprint("X %s" % x)
    else: 
        link = os.link
        unlink = os.unlink
    for pn in to_archive:
        tgt = os.path.join(arch_dir,os.path.split(pn)[1])
        link(pn,tgt)
        log.info('%s -> %s' % (pn,tgt))
        print "archiving %s.." % tgt
        unlink(pn)
    print "done archiving %d files." % len(to_archive)
    for pn in to_remove:
        print "removing %s..                    \r" % pn,
        unlink(pn)
    log.info("done with purge.")

def sdiMergePackageLists(satellite,installed,available):
    """for packages available, see which ones get removed and which get installed
    ingestor_type is "gvar", other satellite name, or None
    available should probably be sorted to latest versions using sdiNewestPackages
    returns (to_remove,to_install,to_reinstall,resulting_package_list)
    
    >>> tgt = DirWithPackages('test_data/base'); src = DirWithPackages('test_data/update3'); sat,_ = sdiIngestorType(src.findPackages())
    >>> to_remove,to_add,_,result_config= sdiMergePackageLists(sat,sdiNewestPackages(tgt.findPackages()),sdiNewestPackages(src.findPackages()))
    >>> print to_remove
    [<FilePackage adde-poes-relay version 20061012 flags [] from test_data/base/adde-poes-relay-20061012.tar.gz>, <FilePackage inge-poes-relay version 20061127 flags [] from test_data/base/inge-poes-relay-20061127.tar.gz>]
    >>> print to_add
    [<FilePackage adde-mtsat-hirid version 20060614 flags [] from test_data/update3/adde-mtsat-hirid-20060614.tar.gz>, <FilePackage inge-mtsat-hirid version 20060614 flags [] from test_data/update3/inge-mtsat-hirid-20060614.tar.gz>]
    >>> print result_config
    [<FilePackage hdd-tools version 20060317b flags [] from test_data/base/hdd-tools-20060317b.tar.gz>, <FilePackage adde-poes-relay version 20061012 flags [] from test_data/base/adde-poes-relay-20061012.tar.gz>, <FilePackage ntp version 20060309 flags [] from test_data/base/ntp-20060309.tar.gz>, <FilePackage sib-driver version 20060504 flags [] from test_data/base/sib-driver-20060504.tar.gz>, <FilePackage inge-mtsat-hirid version 20060614 flags [] from test_data/update3/inge-mtsat-hirid-20060614.tar.gz>, <FilePackage python2.4 version 20060216 flags [] from test_data/base/python2.4-20060216.tar.gz>, <FilePackage vsftpd version 20060309 flags [] from test_data/base/vsftpd-20060309.tar.gz>, <FilePackage inge-poes-relay version 20061127 flags [] from test_data/base/inge-poes-relay-20061127.tar.gz>, <FilePackage libg2c version 20060217 flags [] from test_data/base/libg2c-20060217.tar.gz>, <FilePackage adde-mtsat-hirid version 20060614 flags [] from test_data/update3/adde-mtsat-hirid-20060614.tar.gz>, <FilePackage SDI104-TEST version 20060606 flags [] from test_data/base/SDI104-TEST-20060606.tar.gz>]
    """
    dinst = dict([ (x.name(),x) for x in installed ])
    # print dinst
    to_remove = []
    to_add = []
    to_reinstall = []
    oldsat,_ = sdiIngestorType(installed)
    if satellite!=oldsat: # then put all the old satellite packages in the remove list
        to_remove,oldsat_apps = _sdiAllSatSpec(dinst)
        log.info("planning to remove %s for old satellite %s." % (','.join(oldsat_apps),oldsat))
    for each in available:
        name = each.name()
        msatspec = re_sat_specific_pkg.match(name)
        if msatspec: # then it's a satellite-specific package
            satellite,rems,adds = _sdiMergeSatSpecPkg(satellite,dinst,each,msatspec.groupdict())
        elif FLAG_SYSTEM in each.flags():
            log.debug("implement system package installation!")
            log.info("ignoring system package %s version %s" % (each.name(),each.version()))
            # to_reinstall = _sdiMergeSystemPkg(dinst,each)
        else:
            rems,adds = _sdiMergeOrdinaryPkg(dinst,each)
        for add in adds: # update the install dictionary
            dinst[add.name()] = add
        to_remove += rems   
        to_add += adds
    return to_remove,to_add,to_reinstall,dinst.values()


#####################################
# Tier 1: functions callable by main()

def configLogger():
    """
    """
    logging.basicConfig(level=logging.INFO)
    
    fn_log = time.strftime(LOG_FILENAME_FORMAT)
    logformat = '[%(name)s %(asctime)s %(levelname)s] %(message)s'
    
    # the error log needed to have it's own filehandler. 
    rootBucket = logging.FileHandler(fn_log,mode='at')
    rootFormatter = logging.Formatter(logformat,'%Y%m%d %H:%M:%S')
    rootBucket.setFormatter(rootFormatter)
    rootBucket.setLevel(logging.DEBUG)
    logging.getLogger().addHandler(rootBucket)
    return fn_log
    
def findPackages(loc):
    if not loc: return []
    if os.path.isdir(loc): return DirWithPackages(loc)
    elif os.path.isfile(loc): return TarWithPackages(loc)
    else: return []

def readArguments(args):
    """
    """
    pass
    
def attachSource(name):
    """Attach the named source and provide a directory in return.
    """
    if not name:
        tall = removable_devices()
        devs = [ '/dev/%s' % x[1] for x in tall ]
        print '-' * 72
        print 'Upgrade package source:'
        print 'Found these removable storage devices:'
        print '\n'.join( ['  %s' % x for x in devs] )
        name = raw_input('Please enter device or directory name to retrieve new packages from, or "exit": ')
        if (name == 'exit'): 
            log.info("user exited")
            sys.exit(5)
    if os.path.isdir(name): return DirWithPackages(name)
    else: return DevSource(name,'/mnt/sdi_packages')
    
def attachTarget(name):
    """Attach the named target and provide a directory in return.
    This generally means mounting as read-write
    """
    if name and os.path.isdir(name) and glob.glob(name + '/*.tar.gz'): return DirWithPackages(name)
    if not name: name = '/mnt/CFD'
    return PartitionTarget(mnt=name)

def confirmPurgeSat(to_remove, to_archive, save_dir):
    to_remove_size = sum( os.stat(x).st_size for x in to_remove if not os.path.islink(x) )
    to_archive_size = sum( os.stat(x).st_size for x in to_archive if not os.path.islink(x) )
    
    print "To change satellites, it is necessary to purge out old satellite data and manually review configuration files."
    print "There are %d megabytes of data in %d files which should be removed." % (float(to_remove_size)/(1024**2),len(to_remove))
    print "There are %d configuration files in which will be moved to %s." % (len(to_archive),save_dir)
    print "Enter 'yes' to confirm the deletion of this data and the archival of the configuration files."
    print "If you wish to do this purge manually, answer 'no'."
    print "Upgrade process will continue regardless of your choice; however,"
    print "choosing 'yes' is strongly recommended."
    yesno = ''
    while yesno.lower() not in ['yes','no']:
        yesno = raw_input('Delete old satellite data and archive config files? (yes/no): ')        
    return (yesno.lower()=='yes')    
    
def confirmPlan(to_remove,to_add,to_reinstall,result_pkgs):
    print '-'*72
    print 'SDI-104 Upgrade'
    print '-'*72
    print 'The following packages will be REMOVED from the system:'
    print '\n'.join( [' - %s' % x for x in to_remove] )
    log.debug( "packages to remove: %s" % to_remove )
    print '-'*72
    print 'The following packages will be ADDED to the system:'
    print '\n'.join( [' + %s' % x for x in to_add] )
    log.debug( "packages to add: %s" % to_add )
    print '-'*72
    response = raw_input('Confirm this action (yes/no)? ')
    return response.lower() == 'yes'


#####################################
# Tier 0: test routines

def test_dryrun(dirpath,newdirpath,sat=None):
    logging.basicConfig(level=logging.DEBUG)
    twp = findPackages(dirpath)
    print twp
    twn = findPackages(newdirpath)
    best = sdiNewestPackages(twn)
    insat = sdiIngestorType(twp)[0]
    print twn
    print "satellite type"
    print sdiIngestorType(twp)[0] or "-None Found-"
    print "old packages"
    print '\n'.join( ['%s' % pkg for pkg in twp] )
    print '-' * 40
    print "available packages"
    print '\n'.join( ['%s' % pkg for pkg in best] )
    print '-' * 40
    old,gnu,reinst,outcome = sdiMergePackageLists(sat and sat or insat, twp, best)
    print "will remove"
    print '\n'.join(['%s' % pkg for pkg in old] )
    print '-' * 40
    print "will add"
    print '\n'.join(['%s' % pkg for pkg in gnu] )
    print '-' * 40
    print "resultant configuration"
    print '\n'.join(['%s' % pkg for pkg in outcome] )

def test_findPackagesInLoc(path):
    twp = findPackages(path)
    print '\n'.join( ['%s' % pkg for pkg in twp] )
    print '-' * 40
    print sdiIngestorType(twp)
    
def test_simple():
    logging.basicConfig(level=logging.DEBUG)
    tgt = PartitionTarget(mnt='/mnt/CFD')
    src = DevSource('/dev/sda1','/mnt/usb')
    installable_packages = src.findPackages()
    existing_packages = tgt.findPackages()
    satellite,sat_pkgs = sdiIngestorType(existing_packages)
    print "this is a %s ingestor" % satellite
    newest_packages = sdiNewestPackages(installable_packages)
    
    to_remove,to_add,to_reinstall,result_pkgs = sdiMergePackageLists(satellite,existing_packages,newest_packages)
    if not confirmPlan(to_remove,to_add,to_reinstall,result_pkgs): 
        log.warning( "plan not approved, exiting.." )
        sys.exit(5)
    for each in to_add: tgt.addPackage(each)
    for each in to_remove: tgt.removePackage(each)
    print "<<DONE>>"

def test_live():
    if 3 != runlevel():
        print "Please move to maintenance mode by running 'init 3', then re-run this script."
        log.error("runlevel was not 3 - exiting")
        sys.exit(7)
    logging.basicConfig(level=logging.DEBUG)
    tgt = PartitionTarget('/mnt/CFD')
    src = DevSource('/dev/sda1','/mnt/usb')
    root = RootTarget('/')
    installable_packages = src.findPackages()
    existing_packages = tgt.findPackages()
    newest_packages = sdiNewestPackages(installable_packages)
    satellite,sat_pkgs = sdiIngestorType(existing_packages)
    to_remove,to_add,to_reinstall,result_pkgs = sdiMergePackageLists(satellite,existing_packages,newest_packages)
    if not confirmPlan(to_remove,to_add,to_reinstall,result_pkgs): 
        log.warning( "plan not approved, exiting.." )
        sys.exit(5)
    actually_added = []
    actually_removed = []
    try:
        for each in to_remove:
            phase = 'removing packages from memory'
            root.removePackage(each)    # remove from memory
        for each in to_add:
            phase = 'adding packages'
            root.addPackage(each)       # unpack to memory
            tgt.addPackage(each)        # copy package tarball to flash
            actually_added.append(each)
        for each in to_remove:
            phase = 'removing packages from permanent storage'
            tgt.removePackage(each)     # remove old package from flash
            actually_removed.append(each)
    except:
        log.error("an error occurred while %s" % phase)
        log.error("added %s" % actually_added)
        log.error("removed %s" % actually_removed)
    sync()
    print "<<DONE>>"


#####################################
# Tier 0: MAIN



def main(args):
    """
    Main routine -- FUTURE: can we break this down into smaller functions?
    """
    # global env
    # configure the logger
    configLogger()
    #logging.basicConfig(level=logging.INFO)
    
    if 3 != runlevel():
        print "Please move to maintenance mode by running 'init 3', then re-run this script."
        log.error("runlevel was not 3 - exiting")
        sys.exit(7)
    # read the command line arguments, separating into options dictionary and immediate parameters
    # env,src_name,tgt_name,root_path = readArguments(args)
    # for now, go with default/discovery
    src_name = os.environ.get('TEST_SRC',None)
    tgt_name = os.environ.get('TEST_TGT',None)
    root_path = os.environ.get('TEST_ROOT','/') 
    # get the source directory, mounting it if necessary
    src = attachSource(src_name)
    # get the package listing from the source directory
    installable_packages = src.findPackages()
    # attach the target directory
    tgt = attachTarget(tgt_name)
    # attach the root directory for live updates
    root = RootTarget(root_path)
    # find the already installed packages
    existing_packages = tgt.findPackages()
    # find satellite type and which packages are satellite-specific
    satellite,sat_pkgs = sdiIngestorType(existing_packages)
    print "Found ingestor type: %s" % satellite
    # get old packages out of the update list
    newest_packages = sdiNewestPackages(installable_packages)
    
    # find out what satellites we could convert to from the installable list
    purge = None
    avail_satellites,avail_sat_pkgs = sdiIngestorType(installable_packages)
    satset = sets.Set(sat_pkgs.keys() + avail_sat_pkgs.keys())
    if len(satset) > 1:
        oldsat = satellite
        print "Please choose which satellite type to install the packages of."
        print "Available: " + ','.join([x for x in satset])
        print "Default: " + satellite
        newsat = raw_input('Enter satellite type: ')
        if not newsat: newsat = satellite
        else: satellite = newsat
        if newsat!=oldsat:
            purge = sdiPlanPurge(oldsat)
            if not confirmPurgeSat(*purge):
                print "WARNING: Disk may fill if old satellite data is not purged."
                purge = None
            else:
                print "Purge will take place at the end of installation."
        
    # filter packages to the versions we want
    to_remove,to_add,to_reinstall,result_pkgs = sdiMergePackageLists(satellite,existing_packages,newest_packages)
    # satisfy any package dependencies
    _,additional_needed_packages = sdiAddPackageDependencies(existing_packages,to_add,newest_packages)
    if None==additional_needed_packages: # then we couldn't satisfy a dependency
        print "Dependency check failed. Please review errors and obtain needed package. Aborting."
        sys.exit(5)
    # filter it to get rid of old versions
    additional_needed_newest_packages = sdiNewestPackages(additional_needed_packages)
    log.info( "adding these packages to satisfy dependencies: %s" % additional_needed_newest_packages )
    to_add += additional_needed_newest_packages
    
    # present update plan
    if not confirmPlan(to_remove,to_add,to_reinstall,result_pkgs): 
        log.warning( "plan not approved, exiting.." )
        sys.exit(5)    
        
    # make sure target has space for the change
    # FIXME tgt.checkAvailableSpace(to_remove,to_add,to_reinstall)
    success = False
    reboot_required = False
    if to_reinstall: # replace system level packages
        # tgt.buildSystem(to_reinstall,result_pkgs)
        log.error("system reinstall not yet supported")
    else: # incremental update
        # take contents of package files and remove them from working copy in RAM
        actually_added = []
        actually_removed = []
        try:
            for each in to_add:
                phase = 'adding packages to permanent storage'
                tgt.addPackage(each)        # copy package tarball to flash
                actually_added.append(each)
                reboot_required = True
            for each in to_remove:
                phase = 'removing packages from permanent storage'
                tgt.removePackage(each)     # remove old package from flash
                actually_removed.append(each)
            success = True
        except:
            log.error("an error occurred while %s" % phase)
            log.error("added %s" % actually_added)
            log.error("removed %s" % actually_removed)
        try:
            for each in to_remove:
                phase = 'removing packages from memory'
                root.removePackage(each)    # remove from memory
            for each in to_add:
                phase = 'adding packages to memory'
                root.addPackage(each)       # unpack to memory
            reboot_required = False
        except:
            log.warning("an error occurred while %s" % phase)
        sync()
        
        final = tgt.findPackages()
        
        del tgt  # detaching
        del src  # detaching

        if purge:
            executePurgePlan(*purge)

        log.info("resultant configuration:\n" + '\n'.join(['    %s' % pkg for pkg in final]))

        if success and not reboot_required:
            log.info("Successfully wrote flash memory and updated ramdisk. Return to runlevel 4 to resume operation.")
        elif success and reboot_required: 
            log.info("Successfully wrote flash memory.")
            log.warning("Memory could not be updated: a reboot will be required to load the new packages.")
            sys.exit(2)
        else:
            log.error("Update did not finish successfully! Please send log to UW SSEC")
            sys.exit(9)



def _unittest():
    import doctest
    print "running unit tests.."
    logging.basicConfig(level=logging.CRITICAL)
    doctest.testmod()
    sys.exit(2) # exit with an error code nonetheless

    
if __name__=='__main__':
    # FIXME: use optparse
    if len(sys.argv)==1: 
        main(sys.argv)
    elif len(sys.argv)>1 and sys.argv[1]=='--test': 
        _unittest()
    elif len(sys.argv)==2:
        test_findPackagesInLoc(sys.argv[1])
    elif len(sys.argv)==3:
        test_dryrun(sys.argv[1],sys.argv[2])
    elif len(sys.argv)==4:
        test_dryrun(sys.argv[1],sys.argv[2],sys.argv[3])