Basic slave-master stateful encrypted interaction

* Slave can "take" jobs from the master now
    - Asymmetrical encryption is used via GPG
    - models.Slave stores the gpg fingerprint in models.GPGFingerprintField
    - Slave imports the master's GPG key (/slave_api/autotua_master.asc)
        * Currently, Slave registration is manual (./manage.py shell)
    - Slave does fancy encrypted pickle talking (autotua.talk()) :)
        * "Take" jobs via autotua.Jobs().takejob(maintainer, job_name)
    - slave/autotua/crypt/__init__.py:
        * Implements the glue with `gpg` (maybe pygnupg later?)
        * Crypto() object. Has encrypt() and decrypt()
            - Also see autotua.decrypt_if_required()
    - GNUPGHOME for the slave is /var/tmp/autotua
        * => Job().fetch() requires root access (userpriv/sandbox later)

* Phases store state to allow pausing/stopping and resuming of jobs
    - Future feature, not really used ATM
    - Job().everything() has prelim support for "resume"

* Various small bug fixes and tweaks
    - Yes, I know I need to make this stuff more atomic :p

12 files changed, 294 insertions, 38 deletions

diff --git a/master/TODO b/master/TODO
index 0944b7f..8c21c35 100644
--- a/master/TODO
+++ b/master/TODO
@@ -1,8 +1,7 @@
 TODO:
  * Implement input via the webinterface (manual via command line atm)
  * Better jobuild dependency resolution for atom list (basic right now)
- * Job status tracking and management (none right now)
- * Asymmetrical key authentication for accepting jobs
+ * Job status tracking and management (skeleton right now)
 
 setup-master.py:
  * Should prompt for super-user
diff --git a/master/master/.gitignore b/master/master/.gitignore
new file mode 100644
index 0000000..c6bc574
--- /dev/null
+++ b/master/master/.gitignore
@@ -0,0 +1 @@
+gnupg/
diff --git a/master/master/const.py b/master/master/const.py
index abdd4e4..a01eef2 100644
--- a/master/master/const.py
+++ b/master/master/const.py
@@ -7,10 +7,13 @@
 #
 
 from autotua import config
+import os
 
 # FIXME: insecure tmpdir.
 TMPDIR      = '/tmp/master'
 JOBTAGE     = config.JOBTAGE_DIR
+MASTER_DIR  = os.path.abspath(os.path.dirname(__file__))
+GPGHOME     = MASTER_DIR+'/gnupg'
 
 STAGE_TYPES = ( ('stage1', 'stage1'),
                 ('stage2', 'stage2'),
diff --git a/master/master/models.py b/master/master/models.py
index 6cf9330..6e2cd22 100644
--- a/master/master/models.py
+++ b/master/master/models.py
@@ -16,18 +16,30 @@ import const, random, urllib2
 ### Models begin ###
 ####################
 
+class GPGFingerprintField(models.CharField):
+    def __init__(self, *args, **kwargs):
+        kwargs['max_length'] = 41
+        kwargs['unique'] = True
+        kwargs['editable'] = False
+        super(GPGFingerprintField, self).__init__(*args, **kwargs)
+
 class Slave(models.Model):
-    name = models.CharField(max_length=30, unique=True)
+    name = models.CharField(max_length=30)
+    # GPG Key Fingerprint
+    gpg_fp = GPGFingerprintField()
     # User which owns this slave
     owner = models.ForeignKey(User)
-    # Groups which can use this slave
+    # Groups which can offer jobs to this slave
     users = models.ManyToManyField(Group)
     # Status of slave
     state = models.CharField(max_length=30, default='U',
                              choices=const.SLAVE_STATES)
 
+    class Meta:
+        unique_together = ['owner', 'name']
+
     def __unicode__(self):
-        return "%s (%s)" % (self.name, self.state)
+        return "%s/%s (%s)" % (self.owner, self.name, self.state)
 
 class Provider(models.Model):
     # Identifier for the mirror provider.
@@ -88,9 +100,9 @@ class Mirror(models.Model):
         return '%s/%s (%s)' % (self.server, self.prefix, self.owner.name)
 
     def save(self):
-        if not self.server.endswith('/'):
+        if self.server[-1] != '/':
             self.server += '/'
-        if not self.prefix.endswith('/'):
+        if self.prefix[-1] != '/':
             self.prefix += '/'
         super(Mirror, self).save()
 
@@ -108,7 +120,7 @@ class Job(models.Model):
     # Release (gives implicit info about provider)
     release = models.ForeignKey(Release)
     # i686, amd64, g4, etc.
-    arch = models.ForeignKey(Arch, limit_choices_to=release.archs.all())
+    arch = models.ForeignKey(Arch)
     # Type of stage; hardened, uclibc, etc
     #type = CharField(maxlength=25) # const.stage_types
     # Revision of jobtage tree to use (not to be entered by user)
@@ -124,7 +136,7 @@ class Job(models.Model):
         unique_together = ['name', 'maintainer']
 
     def __unicode__(self):
-        return '%s/%s' % (self.maintainer, self.name)
+        return '%s/%s (%s)' % (self.maintainer, self.name, self.state)
 
     def save(self):
         atoms = self._get_deplist(self.atoms.split(), self.jobtagerev)
diff --git a/master/master/process/__init__.py b/master/master/process/__init__.py
index b0610e2..f66021b 100644
--- a/master/master/process/__init__.py
+++ b/master/master/process/__init__.py
@@ -10,11 +10,11 @@ import random
 from ..models import Mirror
 
 def generate_stage_url(job):
-    mirror = random.choice(Mirror.objects.filter(owner=job.provider))
+    mirror = random.choice(Mirror.objects.filter(owner=job.release.provider))
     url = mirror.server+mirror.prefix+mirror.structure
     data = {}
     data['owner'] = mirror.owner.name
-    data['stage'] = job.stage.name
+    data['stage'] = job.stage
     data['arch'] = job.arch.specific
     data['gen_arch'] = job.arch.generic
     data['release'] = job.release.name
diff --git a/master/master/slave_api.py b/master/master/slave_api.py
index 7c65bed..e08f4d7 100644
--- a/master/master/slave_api.py
+++ b/master/master/slave_api.py
@@ -7,24 +7,22 @@
 #
 
 import cPickle as pickle
+import os
+
+from autotua import crypt
 
 from django.conf import settings
-from django.http import HttpResponse
+from django.http import HttpResponse, Http404
 from django.shortcuts import *
 
-from master.models import Job, User
-import process
+from master.models import Job, User, Slave
+import process, const
 
-def job_list(request, **kwargs):
-    filters = {}
-    if kwargs.has_key('username'):
-        filters['maintainer'] = get_object_or_404(User, username=kwargs['username'])
-        if kwargs.has_key('job_name'):
-            filters['name'] = kwargs['job_name']
-    jobs = []
-    for job in get_list_or_404(Job, **filters):
-        jobs.append(job_data(job))
-    return HttpResponse(pickle.dumps(jobs, 2), mimetype='application/octet-stream')
+def _pickled_http_response(response, crypto=None, recipient=None):
+    response = pickle.dumps(response, 0)
+    if crypto:
+        response = crypto.encrypt(response, recipient)
+    return HttpResponse(response, mimetype='application/octet-stream')
 
 def job_data(job):
     data = {'maintainer': {'username': job.maintainer.username,
@@ -35,3 +33,39 @@ def job_data(job):
             'stage': process.generate_stage_url(job),
             'atoms': job.atoms,}
     return data
+
+def job_list(request, **kwargs):
+    filters = {}
+    if kwargs.has_key('username'):
+        filters['maintainer'] = get_object_or_404(User, username=kwargs['username'])
+        if kwargs.has_key('job_name'):
+            filters['name'] = kwargs['job_name']
+    jobs = []
+    for job in get_list_or_404(Job, **filters):
+        jobs.append(job_data(job))
+    if kwargs.has_key('job_name') and len(jobs) == 1:
+        jobs = jobs[0]
+    return _pickled_http_response(jobs)
+
+def accept_job(request):
+    if request.method == 'GET' or not request.POST.has_key('data'):
+        raise Http404
+    crypto = crypt.Crypto(gpghome=const.GPGHOME)
+    (data, sender) = crypto.decrypt(request.POST['data'])
+    data = pickle.loads(data)
+    data['maintainer'] = get_object_or_404(User, username=data['maintainer'])
+    job = get_object_or_404(Job, **data)
+    slave = get_object_or_404(Slave, gpg_fp=sender)
+    job.slaves.add(slave)
+    response = 'Job Registered'
+    # Encrypting predictable responses is a security risk
+    # The private key becomes vulnerable to detection
+    # Hence, do not encrypt this response.
+    return _pickled_http_response(response)
+
+def get_pubkey(request):
+    pubkey_file = '%s/autotua_master.asc' % const.GPGHOME
+    if not os.path.exists(pubkey_file):
+        crypto = crypt.Crypto(gpghome=const.GPGHOME)
+        crypto.export_pubkey(pubkey_file, 'AutotuA Master')
+    return HttpResponse(open(pubkey_file).read())
diff --git a/master/master/urls.py b/master/master/urls.py
index c00ad82..2888411 100644
--- a/master/master/urls.py
+++ b/master/master/urls.py
@@ -23,8 +23,10 @@ urlpatterns += patterns('master.views',
 
 urlpatterns += patterns('master.slave_api',
     (r'^slave_api/jobs/$', 'job_list'),
-    (r'^slave_api/jobs/(?P<username>[a-zA-Z0-9_]+)/$', 'job_list'),
-    (r'^slave_api/jobs/(?P<username>[a-zA-Z0-9_]+)/(?P<job_name>[^/]+)/$', 'job_list'),
+    (r'^slave_api/jobs/~(?P<username>[a-zA-Z0-9_]+)/$', 'job_list'),
+    (r'^slave_api/jobs/~(?P<username>[a-zA-Z0-9_]+)/(?P<job_name>[^/]+)/$', 'job_list'),
+    ('^slave_api/autotua_master.asc', 'get_pubkey'),
+    (r'^slave_api/slaves/accept/$', 'accept_job'),
 )
 
 # Static media serving for development purposes
diff --git a/master/setup-master.py b/master/setup-master.py
index 5bfcfef..5be5803 100755
--- a/master/setup-master.py
+++ b/master/setup-master.py
@@ -109,6 +109,15 @@ def syncdb_master():
             serverobj.prefix = server[1]
             serverobj.save()
 
+def setup_gpg():
+    from autotua import crypt
+    from master import const
+    data = {'name': 'AutotuA Master',
+            'email': 'autotua@localhost',
+            'expire': '1m'}
+    print 'Creating a "sample" gpg key (expires in 1 month)'
+    crypt.Crypto(gpghome=const.GPGHOME).init_gpghome(**data)
+
 def setup_sample_job():
     from sample_data import sample_job
     job = Job()
@@ -156,6 +165,7 @@ elif sys.argv[1] == 'syncdb':
     # Start stuff
     syncdb_master()
     setup_sample_job()
+    setup_gpg()
     print "All done! Now you can start the master with `python manage.py runserver`"
 else:
     print_help()
diff --git a/slave/autotua/__init__.py b/slave/autotua/__init__.py
index 3e23fc3..5f02947 100644
--- a/slave/autotua/__init__.py
+++ b/slave/autotua/__init__.py
@@ -9,25 +9,94 @@
 import os, shutil, urllib2, atexit
 import os.path as osp
 import cPickle as pickle
-from autotua import fetch, config, sync, chroot, jobuild
+from urllib import urlencode
+from autotua import fetch, config, sync, chroot, jobuild, crypt
+
+def decrypt_if_required(data, crypto):
+    gpg_header = '-----BEGIN PGP MESSAGE-----'
+    if data.split('\n')[0] != gpg_header:
+        return data
+    if not crypto:
+        raise Exception('Encryption selected, but no "crypto"')
+    return crypto.decrypt(data)[0]
+
+def talk(url, data=None, crypto=None, encrypt=False):
+    """
+    Talk to the master server
+    @param url: relative URL to talk to on the master server
+    @type url: string
+
+    @param data: Data to POST to the server
+    @type data: Anything!
+
+    @param encrypt: Whether to encrypt the data to the POSTed
+    @type encrypt: bool
+    """
+    # We not wantz leading '/'
+    if url[0] == '/':
+        url = url[1:]
+    url = urllib2.quote(url)
+    url = '/'.join([config.AUTOTUA_MASTER, url])
+    if data:
+        # ASCII till I figure out str->bytes
+        data = pickle.dumps(data, 0)
+        if encrypt:
+            if not crypto:
+                raise Exception('Encryption selected, but no "crypto"')
+            data = crypto.encrypt(data)
+        data = urlencode({'data': data})
+    data = urllib2.urlopen(url, data).read()
+    data = decrypt_if_required(data, crypto)
+    data = pickle.loads(data)
+    return data
 
 class Jobs:
-    """Interface to jobs on the master server that we can do"""
+    """Interface to jobs on the master server"""
 
     def __init__(self):
-        self.pubkey = ''
+        self.crypto = crypt.Crypto()
+
+    def import_master_pubkey(self):
+        pubkey = 'autotua_master.asc'
+        url = '/'.join([config.AUTOTUA_MASTER, 'slave_api', pubkey])
+        pubkey = urllib2.urlopen(url).read()
+        self.crypto.import_pubkey(pubkey)
 
-    def getjobs(self):
+    def getlist(self, maintainer=None):
         """
         Get a list of jobs
-        (skeleton code atm)
         """
         jobs = []
-        job_list = pickle.load(urllib2.urlopen(config.AUTOTUA_MASTER+'/slave_api/jobs'))
+        url = 'slave_api/'
+        if maintainer:
+            url += '~%s/' % maintainer
+        url += 'jobs/' 
+        job_list = talk(url, crypto=self.crypto)
         for job_data in job_list:
             jobs.append(Job(job_data))
         return jobs
 
+    def getjob(self, maintainer, job_name):
+        """
+        Get a job's data
+        """
+        url = 'slave_api/jobs/~%s/%s' % (maintainer, job_name)
+        job_data = talk(url, crypto=self.crypto)
+        return Job(job_data)
+
+    def takejob(self, maintainer, job_name):
+        """
+        Take a specific job for running
+        """
+        job = self.getjob(maintainer, job_name)
+        url = 'slave_api/slaves/accept/'
+        data = {'maintainer': job.maint, 'name': job.name}
+        ret = talk(url, data, crypto=self.crypto, encrypt=True)
+        if not ret:
+            raise Exception('Unable to register job')
+        print ret
+        return job
+
 class Job:
     """A Job."""
 
@@ -42,6 +111,7 @@ class Job:
         self.atoms = job_data['atoms']
         self.jobuilds = []
         self.chroot = chroot.WorkChroot(self.jobdir, self.stage.filename)
+        self.next_phase = 'fetch'
         atexit.register(self.tidy)
 
     def __repr__(self):
@@ -92,6 +162,7 @@ class Job:
                     rev=self.jobtagerev, scheme="git-export").sync()
         ## Read config, get portage snapshot if required
         #self._fetch_portage_snapshot()
+        self.next_phase = 'prepare'
 
     def prepare(self):
         # Chroot setup needs to be done before parsing jobuilds
@@ -103,6 +174,7 @@ class Job:
             self.jobuilds.append(jobuild.Jobuild(self.jobtagedir, atom))
         print 'Fetch jobuild SRC_URI and hardlink/copy into chroot'
         self._setup_jobfiles()
+        self.next_phase = 'run'
 
     def run(self):
         processor = jobuild.Processor(None, self.chroot)
@@ -110,10 +182,12 @@ class Job:
             processor.jobuild = jbld
             print 'Running jobuild "%s"' % jbld.atom
             processor.run_phase('all')
+        self.next_phase = 'tidy'
 
     def tidy(self):
         print 'Tidying up..'
         self.chroot.tidy()
+        self.next_phase = ''
 
     def clean(self):
         # Tidy up before cleaning
@@ -121,12 +195,19 @@ class Job:
         shutil.rmtree(self.jobdir)
         os.removedirs(osp.join(config.WORKDIR, self.maint))
 
+    def everything(self):
+        while self.next_phase:
+            exec('self.%s()' % self.next_phase)
+        print 'Everything done.'
+
 if __name__ == "__main__":
-    job = Jobs().getjobs()[0]
-    job.fetch()
+    jobs = Jobs()
+    print 'Importing server public key'
+    jobs.import_master_pubkey()
+    print 'Registering sample job "Sample AutotuA job" for running'
+    job = jobs.takejob('test_user', 'Sample AutotuA job')
+    job.next_phase = 'prepare'
     if os.getuid() == 0:
-        job.prepare()
-        job.run()
-        job.tidy()
+        job.everything()
     else:
-        print 'You need to be root to run job.prepare(), job.run() and job.tidy()'
+        print 'You need to be root to run job.everything()'
diff --git a/slave/autotua/config.py b/slave/autotua/config.py
index 11fe625..7ab4267 100644
--- a/slave/autotua/config.py
+++ b/slave/autotua/config.py
@@ -18,8 +18,10 @@ IGNORE_PROXY        = False
 
 LOGFILE             = '/var/log/autotua/slave.log'
 TMPDIR              = '/var/tmp/autotua'
+GPGHOME             = '/var/tmp/autotua/.gnupg'
 
 AUTOTUA_MASTER      = ''
+MASTER_PUBKEY       = 'autotua_master.asc'
 JOBTAGE_URI         = 'git://git.overlays.gentoo.org/proj/jobtage.git'
 
 # Bind mounted inside the chroots for use if defined
diff --git a/slave/autotua/crypt/__init__.py b/slave/autotua/crypt/__init__.py
new file mode 100644
index 0000000..bb97d17
--- /dev/null
+++ b/slave/autotua/crypt/__init__.py
@@ -0,0 +1,111 @@
+# vim: set sw=4 sts=4 et :
+# Copyright: 2008 Gentoo Foundation
+# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com>
+# License: GPL-3
+#
+
+import subprocess, os
+from .. import config
+
+class Crypto(object):
+    """
+    Data Encrypter/Decrypter
+    """
+
+    def __init__(self, gpghome=config.GPGHOME):
+        """
+        @param gpghome: Home directory for GPG
+        @type gpghome: string
+        """
+        self.gpghome = gpghome
+        self.gpgcmd = 'gpg -a --keyid-format long --trust-model always '
+        self.gpgcmd += '--homedir="%s" ' % self.gpghome
+        if not os.path.exists(self.gpghome+'/secring.gpg'):
+            raise Exception('"%s": Invalid GPG homedir' % self.gpghome)
+
+    def _get_fp_from_keyid(self, keyid):
+        gpg_args = '--with-colons --fingerprint --list-keys "%s"' % keyid
+        process = subprocess.Popen(self.gpgcmd+gpg_args, shell=True,
+                                   stdout=subprocess.PIPE)
+        output = process.stdout.readlines()
+        process.wait()
+        for line in output:
+            # Fingerprint line
+            if line.startswith('fpr'):
+                # Fingerprint
+                return line.split(':')[-2]
+
+    def export_pubkey(self, file, which):
+        gpg_args = '--export "%s" > "%s"' % (which, file)
+        print self.gpgcmd+gpg_args
+        subprocess.check_call(self.gpgcmd+gpg_args, shell=True)
+
+    def import_pubkey(self, pubkey):
+        gpg_args = '--import <<<"%s"' % pubkey
+        subprocess.check_call(self.gpgcmd+gpg_args, shell=True)
+
+    def init_gpghome(self, name='Test AutotuA Slave', email='test_slave@test.org',
+                     expire='5y', length='4096'):
+        """
+        Initialize a GnuPG home by generating keys
+        """
+        params = (('Key-Type', 'DSA'),
+                  ('Key-Length', '1024'),
+                  ('Subkey-Type', 'ELG-E'),
+                  ('Subkey-Length', length),
+                  ('Name-Real', name),
+                  ('Name-Email', email),
+                  ('Expire-Date', expire),)
+        # Batch mode
+        gpg_args = '--batch --gen-key <<<"'
+        gpg_args += '%echo Generating keys.. [Worship the gods of randomness :p]'
+        for param in params:
+            gpg_args += '\n%s: %s' % param
+        gpg_args += '"'
+        subprocess.check_call(self.gpgcmd+gpg_args, shell=True)
+        print 'Done.'
+
+    def encrypt(self, data, recipient='Autotua Master'):
+        """
+        @param data: Data to be encrypted
+        @type data: string
+
+        @param recipient: Recipient for the data
+        @type recipient: string
+
+        returns: encrypted_data
+        """
+        gpg_args = '--encrypt --sign --recipient "%s" <<<"%s"' % (recipient, data)
+        process = subprocess.Popen(self.gpgcmd+gpg_args, shell=True,
+                                   stdout=subprocess.PIPE)
+        data = process.stdout.read()
+        process.wait()
+        if process.returncode < 0:
+            raise Exception('Unable to encrypt, something went wrong :(')
+        return data
+
+    def decrypt(self, data):
+        """
+        @param data: Data to be encrypted
+        @type data: string
+
+        returns: (decrypted_data, sender)
+        """
+        gpg_args = '--decrypt <<<"%s"' % data
+        process = subprocess.Popen(self.gpgcmd+gpg_args, shell=True,
+                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        ddata = process.stdout.read()[:-1] # Extra \n at the end :-/
+        # Get the output to stderr
+        gpg_out = process.stderr.readlines()
+        process.wait()
+        if process.returncode < 0 or not ddata:
+            raise Exception('Unable to decrypt, something went wrong')
+        # Get the line with the DSA long key ID
+        for line in gpg_out:
+            if line.find('using DSA key') != -1:
+                # Get the long key ID
+                gpg_out = line.split()[-1]
+                break
+        # Get the fingerprint
+        sender = self._get_fp_from_keyid(gpg_out)
+        return (ddata, sender)
diff --git a/slave/config/slave.cfg b/slave/config/slave.cfg
index dda3226..47216ca 100644
--- a/slave/config/slave.cfg
+++ b/slave/config/slave.cfg
@@ -8,6 +8,7 @@ verbose             = True
 ;ignore_proxy       = False
 ;logfile            = '/var/log/autotua/slave.log'
 ;tmpdir             = '/var/tmp/autotua'
+;gpg_home           = '/var/tmp/autotua/.gnupg'
 # You need to set this if you're running a local AutotuA master
 ;autotua_master     = ''
 ;jobtage_uri        = 'git://git.overlays.gentoo.org/proj/jobtage.git'