dicts.py => mappings.py

aside from that, removed a lot of older portage funcs from fs.py, and added pydoc documentation

7 files changed, 103 insertions, 323 deletions

diff --git a/portage/util/file.py b/portage/util/file.py
index 18e19cc..d4dd11f 100644
--- a/portage/util/file.py
+++ b/portage/util/file.py
@@ -1,11 +1,11 @@
 # Copyright: 2005 Gentoo Foundation
 # Author(s): Brian Harring (ferringb@gentoo.org)
 # License: GPL2
-# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/file.py,v 1.1 2005/07/20 14:33:12 ferringb Exp $
+# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/file.py,v 1.2 2005/08/09 07:43:43 ferringb Exp $
 
 import re
 from shlex import shlex
-from portage.util.dicts import ProtectedDict
+from mappings import ProtectedDict
 
 def iter_read_bash(file):
 	"""read file honoring bash commenting rules.  Note that it's considered good behaviour to close filehandles, as such, 
@@ -24,6 +24,9 @@ def read_bash(file):
 	return list(iter_read_bash(file))
 
 def read_dict(file, splitter="=", ignore_malformed=False, source_isiter=False):
+	"""
+	read key value pairs, splitting on specified splitter, using iter_read_bash for filtering comments
+	"""
 	d = {}
 	if not source_isiter:
 		i = iter_read_bash(file)
@@ -47,6 +50,11 @@ def read_dict(file, splitter="=", ignore_malformed=False, source_isiter=False):
 	return d
 
 def read_bash_dict(file, vars_dict={}, ignore_malformed=False, sourcing_command=None):
+	"""read bash source, yielding a dict of vars
+	vars_dict is the initial 'env' for the sourcing, and is protected from modification.
+	sourcing_command controls whether a source command exists, if one does and is encountered, then this func
+	recursively sources that file
+	"""
 	from shlex import shlex
 	from types import StringTypes
 	f = open(file, "r")
diff --git a/portage/util/fs.py b/portage/util/fs.py
index 848c73a..a3676fc 100644
--- a/portage/util/fs.py
+++ b/portage/util/fs.py
@@ -1,7 +1,7 @@
 # Copyright 2004 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
-# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/fs.py,v 1.2 2005/07/13 05:51:35 ferringb Exp $
-cvs_id_string="$Id: fs.py,v 1.2 2005/07/13 05:51:35 ferringb Exp $"[5:-2]
+# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/fs.py,v 1.3 2005/08/09 07:43:43 ferringb Exp $
+cvs_id_string="$Id: fs.py,v 1.3 2005/08/09 07:43:43 ferringb Exp $"[5:-2]
 
 import os
 
@@ -16,319 +16,6 @@ except:
 	bsd_chflags = None
 
 
-def grabfile(myfilename, compat_level=0):
-	"""This function grabs the lines in a file, normalizes whitespace and returns lines in a list; if a line
-	begins with a #, it is ignored, as are empty lines"""
-
-	try:
-		myfile=open(myfilename,"r")
-	except IOError:
-		return []
-	mylines=myfile.readlines()
-	myfile.close()
-	newlines=[]
-	for x in mylines:
-		#the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line
-		#into single spaces.
-		myline = ' '.join(x.split())
-		if not len(myline):
-			continue
-		if myline[0]=="#":
-			# Check if we have a compat-level string. BC-integration data.
-			# '##COMPAT==>N<==' 'some string attached to it'
-			mylinetest = myline.split("<==", 1)
-			if len(mylinetest) == 2:
-				myline_potential = mylinetest[1]
-				mylinetest = mylinetest[0].split("##COMPAT==>")
-				if len(mylinetest) == 2:
-					if compat_level >= int(mylinetest[1]):
-						# It's a compat line, and the key matches.
-						newlines.append(myline_potential)
-				continue
-			else:
-				continue
-		newlines.append(myline)
-	return newlines
-
-
-def grab_multiple(basename, locations, handler, all_must_exist=0):
-	mylist = []
-	for x in locations:
-		mylist.append(handler(x+"/"+basename))
-	return mylist
-
-
-def grabdict(myfilename,juststrings=0,empty=0):
-	"""This function grabs the lines in a file, normalizes whitespace and returns lines in a dictionary"""
-	newdict={}
-	try:
-		myfile=open(myfilename,"r")
-	except IOError,e:
-		return newdict 
-	mylines=myfile.readlines()
-	myfile.close()
-	for x in mylines:
-		#the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line
-		#into single spaces.
-		if x[0] == "#":
-			continue
-		myline = x.split()
-		if len(myline)<2 and empty==0:
-			continue
-		if len(myline)<1 and empty==1:
-			continue
-		if juststrings:
-			newdict[myline[0]]=" ".join( myline[1:] )
-		else:
-			newdict[myline[0]]=myline[1:]
-	return newdict
-
-
-def grabints(myfilename):
-	newdict={}
-	try:
-		myfile=open(myfilename,"r")
-	except IOError:
-		return newdict 
-	mylines=myfile.readlines()
-	myfile.close()
-	for x in mylines:
-		#the split/join thing removes leading and trailing whitespace, and converts any whitespace in the line
-		#into single spaces.
-		myline=x.split()
-		if len(myline)!=2:
-			continue
-		newdict[myline[0]]=int(myline[1])
-	return newdict
-
-
-def writeints(mydict,myfilename):
-	try:
-		myfile=open(myfilename,"w")
-	except IOError:
-		return 0
-	for x in mydict.keys():
-		myfile.write(x+" "+`mydict[x]`+"\n")
-	myfile.close()
-	return 1
-
-
-def writedict(mydict,myfilename,writekey=1):
-	"""Writes out a dict to a file; writekey=0 mode doesn't write out
-	the key and assumes all values are strings, not lists."""
-	try:
-		myfile=open(myfilename,"w")
-	except IOError:
-		writemsg("Failed to open file for writedict(): "+str(myfilename)+"\n")
-		return 0
-	if not writekey:
-		for x in mydict.values():
-			myfile.write(x+"\n")
-	else:
-		for x in mydict.keys():
-			myfile.write(x+" ")
-			for y in mydict[x]:
-				myfile.write(y+" ")
-			myfile.write("\n")
-	myfile.close()
-	return 1
-
-
-def getconfig(mycfg,tolerant=0,allow_sourcing=False):
-	import shlex, string
-	mykeys={}
-	try:
-		f=open(mycfg,'r')
-	except IOError:
-		return None
-	try:
-		lex=shlex.shlex(f)
-		lex.wordchars=string.digits+string.letters+"~!@#$%*_\:;?,./-+{}"     
-		lex.quotes="\"'"
-		if allow_sourcing:
-			lex.source="source"
-		while 1:
-			key=lex.get_token()
-			if (key==''):
-				#normal end of file
-				break;
-			equ=lex.get_token()
-			if (equ==''):
-				#unexpected end of file
-				#lex.error_leader(self.filename,lex.lineno)
-				if not tolerant:
-					writemsg("!!! Unexpected end of config file: variable "+str(key)+"\n")
-					raise Exception("ParseError: Unexpected EOF: "+str(mycfg)+": on/before line "+str(lex.lineno))
-				else:
-					return mykeys
-			elif (equ!='='):
-				#invalid token
-				#lex.error_leader(self.filename,lex.lineno)
-				if not tolerant:
-					writemsg("!!! Invalid token (not \"=\") "+str(equ)+"\n")
-					raise Exception("ParseError: Invalid token (not '='): "+str(mycfg)+": line "+str(lex.lineno))
-				else:
-					return mykeys
-			val=lex.get_token()
-			if (val==''):
-				#unexpected end of file
-				#lex.error_leader(self.filename,lex.lineno)
-				if not tolerant:
-					writemsg("!!! Unexpected end of config file: variable "+str(key)+"\n")
-					raise portage_exception.CorruptionError("ParseError: Unexpected EOF: "+str(mycfg)+": line "+str(lex.lineno))
-				else:
-					return mykeys
-			mykeys[key]=varexpand(val,mykeys)
-	except SystemExit, e:
-		raise
-	except Exception, e:
-		raise e.__class__, str(e)+" in "+mycfg
-	return mykeys
-	
-
-#cache expansions of constant strings
-cexpand={}
-def varexpand(mystring,mydict={}):
-	"""
-	new variable expansion code.  Removes quotes, handles \n, etc.
-	This code is used by the configfile code, as well as others (parser)
-	This would be a good bunch of code to port to C.
-	"""
-	mystring=" "+mystring
-	if mystring in cexpand:
-		return cexpand[mystring]
-	numvars=0
-	#in single, double quotes
-	insing=0
-	indoub=0
-	pos=1
-	newstring=" "
-	while (pos<len(mystring)):
-		if (mystring[pos]=="'") and (mystring[pos-1]!="\\"):
-			if (indoub):
-				newstring=newstring+"'"
-			else:
-				insing=not insing
-			pos=pos+1
-			continue
-		elif (mystring[pos]=='"') and (mystring[pos-1]!="\\"):
-			if (insing):
-				newstring=newstring+'"'
-			else:
-				indoub=not indoub
-			pos=pos+1
-			continue
-		if (not insing): 
-			#expansion time
-			if (mystring[pos]=="\n"):
-				#convert newlines to spaces
-				newstring=newstring+" "
-				pos=pos+1
-			elif (mystring[pos]=="\\"):
-				#backslash expansion time
-				if (pos+1>=len(mystring)):
-					newstring=newstring+mystring[pos]
-					break
-				else:
-					a=mystring[pos+1]
-					pos=pos+2
-					if a=='a':
-						newstring=newstring+chr(007)
-					elif a=='b':
-						newstring=newstring+chr(010)
-					elif a=='e':
-						newstring=newstring+chr(033)
-					elif (a=='f') or (a=='n'):
-						newstring=newstring+chr(012)
-					elif a=='r':
-						newstring=newstring+chr(015)
-					elif a=='t':
-						newstring=newstring+chr(011)
-					elif a=='v':
-						newstring=newstring+chr(013)
-					elif a!='\n':
-						#remove backslash only, as bash does: this takes care of \\ and \' and \" as well
-						newstring=newstring+mystring[pos-1:pos]
-						continue
-			elif (mystring[pos]=="$") and (mystring[pos-1]!="\\"):
-				pos=pos+1
-				if mystring[pos]=="{":
-					pos=pos+1
-					braced=True
-				else:
-					braced=False
-				myvstart=pos
-				validchars=string.ascii_letters+string.digits+"_"
-				while mystring[pos] in validchars:
-					if (pos+1)>=len(mystring):
-						if braced:
-							cexpand[mystring]=""
-							return ""
-						else:
-							pos=pos+1
-							break
-					pos=pos+1
-				myvarname=mystring[myvstart:pos]
-				if braced:
-					if mystring[pos]!="}":
-						cexpand[mystring]=""
-						return ""
-					else:
-						pos=pos+1
-				if len(myvarname)==0:
-					cexpand[mystring]=""
-					return ""
-				numvars=numvars+1
-				if mydict.has_key(myvarname):
-					newstring=newstring+mydict[myvarname] 
-			else:
-				newstring=newstring+mystring[pos]
-				pos=pos+1
-		else:
-			newstring=newstring+mystring[pos]
-			pos=pos+1
-	if numvars==0:
-		cexpand[mystring]=newstring[1:]
-	return newstring[1:]
-
-
-def pickle_write(data,filename,debug=0):
-	import cPickle,os
-	try:
-		myf=open(filename,"w")
-		cPickle.dump(data,myf,cPickle.HIGHEST_PROTOCOL)
-		myf.close()
-		writemsg("Wrote pickle: "+str(filename)+"\n",1)
-		os.chown(myefn,uid,portage_gid)
-		os.chmod(myefn,0664)
-	except SystemExit, e:
-		raise
-	except Exception, e:
-		return 0
-	return 1
-
-
-def pickle_read(filename,default=None,debug=0):
-	import cPickle,os
-	if not os.access(filename, os.R_OK):
-		writemsg("pickle_read(): File not readable. '"+filename+"'\n",1)
-		return default
-	data = None
-	try:
-		myf = open(filename)
-		mypickle = cPickle.Unpickler(myf)
-		mypickle.find_global = None
-		data = mypickle.load()
-		myf.close()
-		del mypickle,myf
-		writemsg("pickle_read(): Loaded pickle. '"+filename+"'\n",1)
-	except SystemExit, e:
-		raise
-	except Exception, e:
-		writemsg("!!! Failed to load pickle: "+str(e)+"\n",1)
-		data = default
-	return data
-
 # throw this out.
 def movefile(src,dest,newmtime=None,sstat=None,mysettings=None):
 	"""moves a file from src to dest, preserving all permissions and attributes; mtime will
diff --git a/portage/util/inheritance.py b/portage/util/inheritance.py
index 0105e9b..e563fde 100644
--- a/portage/util/inheritance.py
+++ b/portage/util/inheritance.py
@@ -1,7 +1,7 @@
 # Copyright: 2005 Gentoo Foundation
 # Author(s): Brian Harring (ferringb@gentoo.org)
 # License: GPL2
-# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/inheritance.py,v 1.1 2005/08/05 04:42:41 ferringb Exp $
+# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/inheritance.py,v 1.2 2005/08/09 07:43:43 ferringb Exp $
 
 def check_for_base(obj, allowed):
 	"""Look through __class__ to see if any of the allowed classes are found, returning the first allowed found"""
diff --git a/portage/util/lists.py b/portage/util/lists.py
index dfa03e0..4b13d22 100644
--- a/portage/util/lists.py
+++ b/portage/util/lists.py
@@ -1,7 +1,7 @@
 # Copyright: 2005 Gentoo Foundation
 # Author(s): 
 # License: GPL2
-# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/lists.py,v 1.1 2005/07/20 14:33:12 ferringb Exp $
+# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/lists.py,v 1.2 2005/08/09 07:43:43 ferringb Exp $
 
 def unique(s):
 	"""lifted from python cookbook, credit: Tim Peters
@@ -54,3 +54,7 @@ def iterflatten(l):
 				yield x
 		except StopIteration:
 			iters.pop(-1)
+
+def flatten(l):
+	"""flatten, returning a list rather then an iterable"""
+	return list(iterflatten(l))
diff --git a/portage/util/dicts.py b/portage/util/mappings.py
index 675ca06..036564c 100644
--- a/portage/util/dicts.py
+++ b/portage/util/mappings.py
@@ -1,7 +1,9 @@
 # Copyright: 2005 Gentoo Foundation
 # Author(s): Brian Harring (ferringb@gentoo.org)
 # License: GPL2
-# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/Attic/dicts.py,v 1.3 2005/07/20 14:33:12 ferringb Exp $
+# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/mappings.py,v 1.1 2005/08/09 07:43:43 ferringb Exp $
+
+from itertools import imap
 
 class IndexableSequence(object):
 	def __init__(self, get_keys, get_values, recursive=False, returnEmpty=False, 
@@ -105,7 +107,10 @@ class IndexableSequence(object):
 
 
 class LazyValDict(object):
-
+	"""
+	given a function to get keys, and to look up the val for those keys, it'll 
+	lazy load key definitions, and values as requested
+	"""
 	def __init__(self, get_keys_func, get_val_func):
 		self.__val_func = get_val_func
 		self.__keys_func = get_keys_func
@@ -168,6 +173,10 @@ class LazyValDict(object):
 
 
 class ProtectedDict(object):
+	"""
+	given an initial dict, this wraps that dict storing changes in a secondary dict, protecting
+	the underlying dict from changes
+	"""
 	__slots__=("orig","new","blacklist")
 
 	def __init__(self, orig):
@@ -227,3 +236,72 @@ class ProtectedDict(object):
 	def items(self):
 		return list(self.iteritems())
 
+class Unchangable(Exception):
+	def __init__(self, key):	self.key = key
+	def __str__(self):			return "key '%s' is unchangable" % self.key
+
+
+class LimitedChangeSet(object):
+	"""
+	set that supports limited changes, specifically deleting/adding a key only once per commit, 
+	optionally blocking changes to certain keys.
+	"""
+	_removed 	= 0
+	_added		= 1
+
+	def __init__(self, initial_keys, unchangable_keys=[]):
+		self.__new = set(initial_keys)
+		self.__blacklist = set(unchangable_keys)
+		self.__changed = set()
+		self.__change_order = []
+		self.__orig = frozenset(self.__new)
+
+	def add(self, key):
+		if key in self.__new:
+			return
+
+		if key in self.__changed or key in self.__blacklist:
+			# it's been del'd already once upon a time.
+			raise Unchangable(key)
+
+		self.__new.add(key)
+		self.__changed.add(key)
+		self.__change_order.append((self._added, key))
+
+	def remove(self, key):
+		if key in self.__changed or key in self.__blacklist:
+			raise Unchangable(key)
+		
+		self.__new.remove(key)
+		self.__changed.add(key)
+		self.__change_order.append((self._removed, key))
+
+	def __contains__(self, key):
+		return key in self.__new
+
+	def changes_count(self):
+		return len(self.__change_order)
+
+	def commit(self):
+		self.__orig = frozenset(self.__new)
+		self.__changed.clear()
+		self.__change_order = []
+
+	def rollback(self, point=0):
+		l = self.changes_count()
+		if point < 0 or point > l:
+			raise TypeError("%s point must be >=0 and <= changes_count()" % point)
+		while l > point:
+			change, key = self.__change_order.pop(-1)
+			self.__changed.remove(key)
+			if change == self._removed:
+				self.__new.add(key)
+			else:
+				self.__new.remove(key)					
+			l -= 1
+
+	def __str__(self):
+		return str(self.__new).replace("set(","LimitedChangeSet(", 1)
+
+	def __iter__(self):
+		return iter(self.__new)
diff --git a/portage/util/modules.py b/portage/util/modules.py
index 2ad7c35..81685a1 100644
--- a/portage/util/modules.py
+++ b/portage/util/modules.py
@@ -1,13 +1,14 @@
 # Copyright: 2005 Gentoo Foundation
 # Author(s): Brian Harring (ferringb@gentoo.org)
 # License: GPL2
-# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/modules.py,v 1.3 2005/08/03 00:22:52 ferringb Exp $
+# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/modules.py,v 1.4 2005/08/09 07:43:43 ferringb Exp $
 
 class FailedImport(ImportError):
 	def __init__(self, trg, e):	self.trg, self.e = trg, e
 	def __str__(self):	return "Failed importing target '%s': '%s'" % (self.trg, self.e)
 
 def load_module(name):
+	"""load a module, throwing a FailedImport if __import__ fails"""
 	try:
 		m = __import__(name)
 		nl = name.split('.')
@@ -22,6 +23,7 @@ def load_module(name):
 	
 
 def load_attribute(name):
+	"""load a specific attribute, rather then a module"""
 	try:
 		i = name.rfind(".")
 		if i == -1:
diff --git a/portage/util/strings.py b/portage/util/strings.py
index d0da87c..2bb7645 100644
--- a/portage/util/strings.py
+++ b/portage/util/strings.py
@@ -1,9 +1,10 @@
 # Copyright: 2005 Gentoo Foundation
 # Author(s): Brian Harring (ferringb@gentoo.org)
 # License: GPL2
-# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/strings.py,v 1.1 2005/07/20 14:33:12 ferringb Exp $
+# $Header: /local/data/ulm/cvs/history/var/cvsroot/gentoo-src/portage/portage/util/strings.py,v 1.2 2005/08/09 07:43:43 ferringb Exp $
 
 def iter_tokens(s, splitter=(" ")):
+	"""iterable yielding of splitting of a string"""
 	pos = 0
 	strlen = len(s)
 	while pos < strlen: