"""
Miscellaneous functions written by Martin Preishuber
"""
from string import strip, lower, zfill, find, atoi, join, replace
from os import getcwd, chdir, mkdir, lstat, unlink, rmdir, utime, environ, getpid
import sys
from urlparse import urlparse
from time import time, sleep
from popen2 import popen3
if (sys.platform != "win32"):
from popen2 import Popen3
from re import match
import signal
import os
import string
import copy
import log4py
import process
if (sys.platform != "win32"):
from os import wait, kill
# Boolean constants
TRUE = 1
FALSE = 0
# Color constants
BLACK = "30"
RED = "31"
GREEN = "32"
YELLOW = "33"
BLUE = "34"
PURPLE = "35"
AQUA = "36"
WHITE = "37"
"""
String operations
"""
def split(s, sep = None, special = None):
""" Extended split (doesn't split within special characters). """
if (special == None):
if (sep == None):
return string.split(s)
else:
return string.split(s, sep)
else:
list = []
elem = ""
mode = 0
for i in range(len(s)):
if (s[i] == sep):
if (mode == 0) and (elem != ""):
list.append(elem)
elem = ""
elif (mode == 1):
elem = "%s%s" % (elem, s[i])
elif (s[i] == special):
if (mode == 0):
mode = 1
elif (mode == 1):
mode = 0
else:
elem = "%s%s" % (elem, s[i])
list.append(elem)
return list
def cfill(source, char, width):
""" Fill a string with a character up to a given width. """
while len(source) < width:
source = "%s%s" % (source, char)
return source
def pfill(source, char, width):
""" Pad a string with a character up to a given width. """
while len(source) < width:
source = char + source
return source
def lstrip(string):
""" Strip blanks (NOT whitespaces) on the left side of a string. """
while (string[0] == " ") and (len(string) > 0):
string = string[1:]
return string
def merge(string1, string2):
""" Merge two strings byte-by-byte. """
pos1 = 0
pos2 = 0
result = ""
while (len(string1) >= (pos1 + 1)) and (len(string2) >= (pos2 + 1)):
result = "%s%s%s" % (result, string1[pos1], string2[pos2])
pos1 = pos1 + 1
pos2 = pos2 + 1
if (len(string1) >= (pos1 + 1)):
result = "%s%s" % (result, string1[pos1:])
if (len(string2) >= (pos2 + 1)):
result = "%s%s" % (result, string2[pos2:])
return result
"""
RE module functions
"""
def unfold(regexp):
splitted = split(regexp, "\n")
for i in range(len(splitted)):
splitted[i] = strip(splitted[i])
return join(splitted, "")
"""
Type conversion functions
"""
def bool2str(value):
""" Converts a Boolean value to the corresponding String. """
if (value == FALSE):
return "False"
elif (value == TRUE):
return "True"
else:
return "Invalid boolean value: %s" % str(value)
def str2bool(value):
""" Converts a String value to the corresponding Boolean. """
if (lower(value) == "true") or (lower(value) == "yes") or (value == "1"):
return TRUE
elif (lower(value) == "false") or (lower(value) == "no") or (value == "0"):
return FALSE
else:
return None
def lng2str(number, prettyprint = TRUE):
""" Returns a string representation of a number (with a pretty-print option). """
if (prettyprint == TRUE):
number = list(str(number))
if number[-1] == "L":
number = number[:-1]
number.reverse()
dots = divmod(len(number) - 1, 3)[0]
for i in range(dots):
number.insert(3 + (i * 3) + i, ".")
number.reverse()
return join(number, "")
else:
return str(number)
"""
List operations
"""
def striplist(list):
""" Strip each value of a list. """
for i in range(len(list)):
list[i] = strip(list[i])
return list
def cleanlist(list):
""" Removes empty entries from a list. """
newlist = []
for i in range(len(list)):
if (list[i] <> ""):
newlist.append(list[i])
return newlist
def listdiff(list1, list2):
""" Get the difference of two lists. """
difflist = []
for i in range(len(list1)):
if (not list1[i] in list2):
difflist.append(list1[i])
for i in range(len(list2)):
if (not list2[i] in list1):
difflist.append(list2[i])
return difflist
def listmatch(list, expression):
""" Returns the matching element number and match object. """
number = -1
element = None
for i in range(len(list)):
if (match(expression, list[i])):
number = i
element = list[i]
break
return (number, element)
"""
Time functions
"""
def nicetime(seconds):
""" Converts time in seconds to a nicely formatted string. """
minutes, seconds = divmod(seconds, 60)
return "%s:%s" % (zfill(minutes, 2), zfill(seconds, 2))
"""
File and directory operations
"""
def ls(directory, recursive = FALSE, includepath = FALSE, onlydirectories = FALSE):
""" Creates a directory listing, optional including full filenames and recursive. """
longlist = []
allfiles = os.listdir(directory)
if (directory == "."): directory = ""
if ((len(directory) > 0) and (directory[-1] != os.sep)): directory = "%s%s" % (directory, os.sep)
for i in range(len(allfiles)):
filename = "%s%s" % (directory, allfiles[i])
longfilename = "%s%s%s" % (os.path.abspath(directory), os.sep, filename)
if (os.path.isdir(filename)):
if (includepath == TRUE):
longlist.append(longfilename)
else:
longlist.append(filename)
if (recursive == TRUE):
subdirlist = ls("%s%s" % (filename, os.sep), recursive, includepath, onlydirectories)
if (len(subdirlist) > 0):
longlist = longlist + subdirlist
else:
if (onlydirectories == FALSE):
if (includepath == TRUE):
longlist.append(longfilename)
else:
longlist.append(filename)
return longlist
def rm(file, maximum = -1, deleted = 0):
""" Remove a file or directory (recursive) and return the number of directories, files, bytes & links deleted. """
directories = files = bytes = links = 0L
if (os.path.islink(file)):
if (maximum == -1) or ((maximum != -1) and (deleted < maximum)):
links = links + 1
unlink(file)
elif (os.path.isfile(file)):
filesize = lstat(file)[6]
if (maximum == -1) or ((maximum != -1) and ((deleted + filesize) < maximum)):
bytes = bytes + filesize
files = files + 1
unlink(file)
else:
filelist = os.listdir(file)
for i in range(len(filelist)):
result = rm("%s%s%s" % (file, os.sep, filelist[i]), maximum, bytes)
directories = directories + result[0]
files = files + result[1]
bytes = bytes + result[2]
links = links + result[3]
filelist = listdir(file)
if (maximum == -1) or ((len(filelist) == 0) and (maximum != -1) and (deleted < maximum)):
rmdir(file)
directories = directories + 1
return [directories, files, bytes, links]
def mkdirtree(directory):
""" Creates a whole direcotry tree. """
if (directory[-1:] == os.sep):
directory = directory[:-1]
if (not os.path.exists(directory)):
pwd = getcwd()
splitted = split(directory, os.sep)
for i in range(1, len(splitted)):
path = join(splitted[:i + 1], os.sep)
if (not os.path.exists(path)):
mkdir(path)
chdir(pwd)
def du(file):
""" Calculate the disk usage of a given directory. """
if (os.path.islink(file)) or (os.path.isfile(file)):
size = lstat(file)[6]
elif (os.path.isdir(file)):
filelist = listdir(file)
size = 0
for i in range(len(filelist)):
size = size + du("%s%s%s" % (file, os.sep, filelist[i]))
else:
# the file is a special device (socket)
size = 0
return size
def __internal_which(file):
if (file == ""):
return ""
for item in split(os.environ["PATH"], os.pathsep):
filename="%s%s%s" % (item, os.sep, file)
if (sys.platform == "win32"):
if (os.path.exists("%s.exe" % filename)):
return "%s.exe" % filename
elif (os.path.exists("%s.com" % filename)):
return "%s.com" % filename
elif (os.path.exists("%s.bat" % filename)):
return "%s.bat" % filename
elif (os.path.exists("%s.cmd" % filename)):
return "%s.cmd" % filename
else:
if (os.path.exists(filename)):
return filename
return ""
def which(file):
""" Check the path to find some specified program. """
filename = __internal_which(file)
if (find(filename, " ") != -1):
return "\"%s\"" % filename
else:
return filename
def filecopy(source, target):
""" Copy a source file to a target file. """
sourcefile = open(source, "r")
targetfile = open(target, "w")
targetfile.write(sourcefile.read())
targetfile.close()
sourcefile.close()
def includefile(source, target):
""" Include a file into a already open file. """
sourcefile = open(source, "r")
line = "line"
while (line != ""):
line = sourcefile.readline()
if (line != ""):
target.write(line)
sourcefile.close()
def listdir(path, incfiles = TRUE, incdirs = TRUE, inclinks = TRUE):
""" A modified listdir with some optional parameters. """
files = os.listdir(path)
if ((incfiles == TRUE) and (incdirs == TRUE) and (inclinks == TRUE)):
cleanfiles = files
else:
cleanfiles = []
if (path[-1] != os.sep):
path = "%s%s" % (path, os.sep)
for i in range(len(files)):
file = "%s%s" % (path, files[i])
addFile = TRUE
if (incfiles == FALSE) and (os.path.isfile(file)):
addFile = FALSE
if (incdirs == FALSE) and (os.path.isdir(file)):
addFile = FALSE
if (inclinks == FALSE) and (os.path.islink(file)):
addFile = FALSE
if (addFile == TRUE):
cleanfiles.append(files[i])
cleanfiles.sort()
return cleanfiles
def touch(filename):
""" Implements the functionality of the unix command "touch". """
if (os.path.exists(filename)):
now = time()
utime(filename, (now, now))
else:
file = open(filename, "w")
file.close()
def df(directory = ""):
""" Returns the values of "df" in list form. """
df_command = which("df")
df_command = strip("%s --block-size=1 %s" % (df_command, directory))
output = striplist(cmdoutput(df_command)[1:])
entry_list = []
for i in range(len(output)):
splitted = cleanlist(split(output[i], " "))
entry = {}
entry["FILESYSTEM"] = splitted[0]
entry["SIZE"] = atoi(splitted[1])
entry["USED"] = atoi(splitted[2])
entry["AVAILABLE"] = atoi(splitted[3])
entry["PERCENTAGE"] = atoi(splitted[4][:-1])
entry["MOUNT_POINT"] = splitted[5]
entry_list.append(entry)
return entry_list
def cmpdirs(dict1, dict2):
""" Compares two dictionaries of lstatfiles and returns a tuple ( {added}, {deleted}, {modified]) """
added = {}
deleted = {}
modified = {}
for i in range(len(dict2.keys())):
key = dict2.keys()[i]
value = dict2[key]
if (not key in dict1.keys()):
added[key] = value
else:
if (value[8] != dict1[key][8]):
modified[key] = value
for i in range(len(dict1.keys())):
key = dict1.keys()[i]
value = dict1[key]
if (not key in dict2.keys()):
deleted[key] = value
return (added, deleted, modified)
def lstatfiles(filelist):
""" Returns a dictionary {filename: lstat} for a given [filelist]. """
statdict = {}
for i in range(len(filelist)):
statinfo = lstat(filelist[i])
statdict[filelist[i]] = statinfo
return statdict
def escapedfilename(filename):
""" escapes special characters in filenames (currently "`") """
filename = replace(filename, "`", "\`")
return filename
"""
Pipe functions
"""
def cmdoutput(command, strip = FALSE, waitpid = TRUE, returnerror = FALSE):
""" Read the complete output of a command. """
pipe = popen3(command)
# os.wait is not available on the win32 platform so it is necessary to set this values
if (sys.platform == "win32"):
waitpid = FALSE
if (returnerror == TRUE):
output = pipe[2].readlines()
else:
output = pipe[0].readlines()
if (strip == TRUE):
output = striplist(output)
if (pipe[0] != None):
pipe[0].close()
if (pipe[1] != None):
pipe[1].close()
if (pipe[2] != None):
pipe[2].close()
if (waitpid == TRUE):
os.wait()
del pipe
pipe = None
return output
def cmdexec(cmd):
""" Executes a command in a subshell and returns (return_value, (stdout, stderr)). """
if (sys.platform == "win32"):
cmd = "(%s) 2>&1" % (cmd)
result = 0
stderr_output = []
stdout_output = cmdoutput(cmd)
else:
my_popen3 = Popen3(cmd, True)
# wait until the sub process has finished
while (my_popen3.poll() == -1):
sleep(0.01)
stderr_output = my_popen3.fromchild.readlines()
stdout_output = my_popen3.childerr.readlines()
# read the result value
result = my_popen3.poll()
if (my_popen3.fromchild != None):
my_popen3.fromchild.close()
if (my_popen3.childerr != None):
my_popen3.childerr.close()
if (my_popen3.tochild != None):
my_popen3.tochild.close()
return (result, (stdout_output, stderr_output))
"""
General functions
"""
def SigChldHandler(signal_number, stack_frame):
""" Handler, which waits for children to be closed. """
# Reinstall the sigchild handler, because it gets deleted on libc5/sysv machines
signal.signal(signal.SIGCHLD, SigChldHandler)
pid = -1
while (pid != 0):
try:
pid = os.waitpid(0, os.WNOHANG)[0]
except:
pid = 0
def ansi(text, fg = WHITE, bg = BLACK, bold = FALSE):
""" ANSI text (fore- & background, bold font). """
if (bold == TRUE):
fg = "%s;1" % fg
bg = str(atoi(bg) + 10)
text = "\033[%s;%sm%s\033[0m" % (bg, fg, text)
return text
"""
Extended version of urlparse (includes username & password)
"""
# extended urlparse function (including username, password)
# returns: <scheme>://<netloc>/<path>;<params>?<query>#<fragment>
# returns: <scheme>://<user>:<password>@<netloc>:<port>/<path>;<params>?<query>#<fragment>
urlparseorg = urlparse
_services_cache = {}
def readservices():
global _services_cache
services = {}
if (len(_services_cache) == 0):
if (os.path.exists("/etc/services")):
file = open("/etc/services", "r")
lines = file.readlines()
file.close()
for i in range(len(lines)):
line = strip(lines[i])
if (find(line, "#") != -1):
line = strip(line[:find(line, "#")])
if (len(line) > 0):
nameportpair = split(line)
if (len(nameportpair) > 1):
name, port = nameportpair[:2]
if find(port, "/") != -1:
port, protocol = split(port, "/")
services[name] = port
else:
services["ftp"] = 21
services["http"] = 80
services["rsync"] = 873
_services_cache = copy.copy(services)
else:
services = copy.copy(_services_cache)
return services
def urlparse(url):
rsync_found = FALSE
if (url[0:5] == "rsync"):
rsync_found = TRUE
url = "ftp%s" % url[5:]
scheme, netloc, path, params, query, fragment = urlparseorg(url)
if (rsync_found):
scheme = "rsync"
services = readservices()
user = password = ""
if services.has_key(scheme):
port = atoi(services[scheme])
else:
port = 0
if (find(netloc, "@") != -1):
user, netloc = split(netloc, "@")
if (find(user, ":") != -1):
user, password = split(user, ":")
if (find(netloc, ":") != -1):
netloc, port = split(netloc, ":")
port = atoi(port)
if (path == ""):
path = "/"
tuple = scheme, user, password, netloc, port, path, params, query, fragment
return tuple
"""
Database functions
"""
def db_null_string(s):
if (s == "") or (s == None):
return "NULL"
else:
return s
"""
Miscellaneous functions
"""
def get_username():
""" Returns the username of the current user. """
if (environ.has_key("USER")):
return environ["USER"]
elif (environ.has_key("USERNAME")):
return environ["USERNAME"]
else:
return "Unknown"
def get_tempdir():
# Default directory on POSIX machines
if ((environ.has_key("TMPDIR")) and (os.path.isdir(environ["TMPDIR"]))):
return environ["TMPDIR"]
# TEMP / TMP are for win32 systems
elif ((environ.has_key("TEMP")) and (os.path.isdir(environ["TEMP"]))):
return environ["TEMP"]
elif ((environ.has_key("TMP")) and (os.path.isdir(environ["TMP"]))):
return environ["TMP"]
else:
# Try to find another existing temporary directory
if (os.path.exists("/tmp")):
return "/tmp"
elif (os.path.exists("/var/tmp")):
return "/var/tmp"
else:
return "/usr/tmp"
"""
Wrapper class for gettext (works for all python versions)
"""
try:
import gettext
_gettext = gettext
gettext_available = TRUE
except ImportError:
gettext_available = FALSE
class gettext:
def __init__(self, domain, localedir = None):
""" Initialize the gettext wrapper. """
if (gettext_available == TRUE):
_gettext.bindtextdomain(domain, localedir)
def install(self, domain, localedir = None):
if (gettext_available == TRUE):
_gettext.install(domain, localedir)
def find(self, domain, localedir = None):
if (gettext_available == TRUE):
return _gettext.find(domain, localedir)
else:
return None
def textdomain(self, domain = None):
if (gettext_available == TRUE):
return _gettext.textdomain(domain)
else:
return None
def gettext(self, message):
if (gettext_available == TRUE):
return _gettext.gettext(message)
else:
return message
def dgettext(self, domain, message):
if (gettext_available == TRUE):
return _gettext.dgettext(domain, message)
else:
return message
"""
Adaptive file locking
"""
class FileLock:
# Status constants
UNLOCKED = 0
LOCKED_SELF = 1
LOCKED_OTHER = 2
ORPHAN = 3
STALE = 4
def __init__(self, filename, staletime = 43200):
self.log4py = log4py.Logger().get_instance(self)
self.filename = filename
self.ownerpid = None # used to carry PID between status calls and actions
self.staletime = staletime # Default is 12 hours
self.ps = process.ProcessTable() # Process table for checking if PID lives
def acquire(self):
if (self.status() != FileLock.UNLOCKED):
self.log4py.error("Already locked: %s" % self.filename)
return FALSE
self.log4py.debug("Acquiring lock: %s" % self.filename);
try:
file = open(self.filename, "w")
file.write(str(getpid()))
file.close()
except IOError, msg:
self.log4py.error("Failed to create lock file: %s (%s)" % (self.filename, msg))
return FALSE
return TRUE
def release(self):
if (self.status() != FileLock.LOCKED_SELF):
self.log4py.error("Not locked or not owner: %s" % self.filename)
return FALSE
self.log4py.debug("Releasing lock: %s" % self.filename);
try:
unlink(self.filename)
except IOError, msg:
self.log4py.error("Failed to unlink lock file: %s (%s)" % (self.filename, msg))
return FALSE
return TRUE
def refresh(self):
if (self.status() != FileLock.LOCKED_SELF):
self.log4py.error("Not owner: %s" % self.filename)
return FALSE
self.log4py.debug("Refreshing lock: %s" % self.filename);
now = time()
utime(self.filename, (now, now))
return TRUE
def killStaleLock(self):
status = self.status()
self.log4py.debug("Killing stale lock: %s" % self.filename);
if (status == FileLock.ORPHAN):
try:
unlink(self.filename)
except IOError, msg:
self.log4py.error("Failed to unlink stale lock file: %s (%s)"
% (self.filename, msg))
return FALSE
return TRUE
elif (status == FileLock.STALE):
try:
unlink(self.filename)
except IOError, msg:
self.log4py.error("Failed to unlink stale lock file: %s (%s)"
% (self.filename, msg))
return FALSE
try:
dead = FALSE
kill(self.ownerpid, signal.SIGTERM)
# Give it five minutes to die gracefully
for tries in range(5):
time.sleep(60)
self.ps.read()
if (self.ps.process(atoi(self.ownerpid)) == None):
dead = TRUE
break
if not dead:
# Use the *big* club
kill(self.ownerpid, signal.SIGKILL)
except:
self.log4py.error("Failed to kill stale lock owner process: %s" % self.ownerpid)
return FALSE
return TRUE
else:
self.log4py.error("No lock, or not stale: %s" % self.filename)
return FALSE
def status(self):
if (not os.path.exists(self.filename)):
return FileLock.UNLOCKED
self.ownerpid = None
try:
file = open(self.filename, "r")
self.ownerpid = file.readline()
file.close()
except IOError, msg:
self.log4py.error("Failed to read PID from lock file: %s (%s)"
% (self.filename, msg));
# Assume its locked - perhaps owner didn't finish writing yet?
return FileLock.LOCKED_OTHER
# Does the current process own the lock?
if (atoi(self.ownerpid) == getpid()):
return FileLock.LOCKED_SELF
# Determine if the lock file owner is alive
self.ps.read()
if (self.ps.process(atoi(self.ownerpid)) == None):
return FileLock.ORPHAN
# Determine if the lock is held by a hung process
timestamp = os.path.getmtime(self.filename)
now = time()
if (now - timestamp > self.staletime):
return FileLock.STALE
# There is a valid process that is refreshing the lock file
return FileLock.LOCKED_OTHER
syntax highlighted by Code2HTML, v. 0.9.1