My convenience stuff (for Python3) {{c """Convenience methods"""

import sys,os,os.path,re,glob,collections,shutil,functools,itertools,operator import math,random,json,urllib.parse,subprocess,datetime from importlib import reload as reloadmod from pyquery import PyQuery as pq

def product(xs,a=1): """Alias for reduce(operator.mul,xs,a), a defaults to 1""" return functools.reduce(operator.mul,xs,a) def prepend_to_path_if_exists(x): a = os.path.expanduser(x) if os.path.exists(a): sys.path.insert(0,a) def append_to_path_if_exists(x): a = os.path.expanduser(x) if os.path.exists(a): sys.path.append(a) def pqload(x): """load html into pyquery""" return pq(open(x).read()) def jload(x): """load and parse json from file x""" return json.loads(open(x).read()) def jsave(x,y): """save json dump of y to file x""" open(x,"wt").write(json.dumps(y)) def readf(x,mode="rt"): """read from file x""" with open(x,mode) as f: return f.read() def writef(x,y,mode="wt"): """write y to file x""" with open(x,mode) as f: return f.write(y) def lm(xs): """list map shorthand""" return list(map(xs)) def lf(xs): """list filter shorthand""" return list(filter(xs)) def lz(xs): """list zip shorthand""" return list(zip(xs)) def lso(*xs,*kw): """list sorted shorthand""" return list(sorted(xs,**kw))

def rls(x): """open, read, strip, split by newlines""" return open(x).read().strip().split("\n") def pwf(f,xs): """take f and xs, produce list of pairs of form (x,f(x))""" return list(map(lambda t: (t,f(t)),xs))

def glx(xs,expanduser=True): """glob shorthand""" if expanduser: xs = lm(os.path.expanduser,xs) return glob.glob(xs) def gstat(xs): """glob and produce pairs of form [x, os.stat(x)]""" return pwf(os.stat,glx(xs)) def gsp(x,*ps): """takes glob pattern x, obtains pairs of form (x,os.stat(x)), then takes a list of desired attributes, ps, and produces a list of lists of the form x, os.stat(x).st_size, os.stat(x).stetc ps is a list of attribute names, without the leading st.""" return list(map(lambda t: t[:1]+list(map(lambda s: t[1].getattribute("st_"+s),ps)),gstat(x))) def gsz(x): return pwf(os.path.getsize,glx(x)) def gsgb(x): return pwf(lambda t: os.path.getsize(t)/(10243),glx(x)) def gsgf(x): return pwf(lambda t: "{:0.3fGiB".format(os.path.getsize(t)/(10243)),glx(x))

def pin(x): return lambda t: x in t def phas(x): return lambda t: t in x def pre(x): return re.compile(x).search def prem(x): return re.compile(x).match def presub(x,y,*ys,*kw): r = re.compile(x).sub return lambda t: r(y,t,ys,**kw) def sprep(x): return lambda t: x+t def sapp(x): return lambda t: t+x def oplus(x,y): return x+y

def camel1(x): "Single word to CamelCase, first upper, rest lower" if len(x) < 2: return x.upper() else: return x[0].upper()+x[1:].lower() def camell(xs): "List to CamelCase" return "".join(map(camel1,xs)) def camel(x): "Space delimited string to CamelCase" return camell(re.split(r"\s+",x)) def lcamell(xs): "List to lowerCamelCase" a = "".join(map(camel1,xs)) return a[0].lower() + a[1:] def lcamel(x): "Space delimited string to lowerCamelCase" return lcamell(re.split(r"\s+",x)) def cname(x): return camel1(x.replace(" ",""))

def dictcopy(dest,src,*names): """Takes keys from 'comma,delimited,strings' as args, and (shallow) copies those from src dict to dest dict. Provided src admits getattr and dest admits setattr this will work ok.

Usual use is with locals() as src to easily copy local variables to a dict (usually so as to json dump or pickle). """ for x in names: for y in x.split(","): if y in src: dest[y] = src[y] class Cmd: "On creation runs command, saves stdout, stderr, return value"

Sort input out later

def __init__(self,xs):
    self.cmdline = xs
    s = subprocess
    p = s.Popen(xs,stdout=s.[PIPE](PIPE),stderr=s.[PIPE](PIPE),stdin=s.[DEVNULL](DEVNULL))
    o,e = p.communicate()
    self.r = p.returncode
    self.o = o.decode("utf-8")
    self.e = e.decode("utf-8")

def cmdx(x): "Run command x and return (retval,stdout,stderr) triple"

usual default dicts

def dictl(): return collections.defaultdict(list) def dicti(): return collections.defaultdict(int) def dicts(): return collections.defaultdict(set) def dictstr(): return collections.defaultdict(str)

hierarchically queriable dict

use hdict(x) and dict(x) to convert

class hdict(dict): """Hierarcically queriable dictionary. Keys are split at "." (so keys can't contain . anymore)

Example: d = 'a':1,'b':2,'c':{'d':42,{'e':23} } # normal dict h = hdict(d) h['c.d.e'] # returns 23 h['c.x.y'] = 41 # assigns to d['c']['x']['y'], creating as necessary

also: h['c.d']['e.f'] = 42 h['c.d.e.f'] # returns 42 h['c']['d']['e']['f'] is h['c.d.e.f'] # returns True """ def getitem(self,k): """Like usual getitem, but a.b.c results in recursive descent""" ks = k.split(".") if len(ks) == 1: y = dict.getitem(self,k) return y else: d = self for x in ks: if not x in d: raise KeyError("'' for ''".format(x,k)) d = d[x] return d def setitem(self,k,v): """Like usual setitem, but a.b.c = r results in recursive descent""" ks = k.split(".") if len(ks) == 1: return dict.setitem(self,k,v) kp = ks[:-1] kk = ks[-1] # last item d = self for x in kp: if not x in d: dict.setitem(d,x,hdict()) d = dict.getitem(d,x)

success guaranteed from here, so no issue of rollback

        else:
            try:
                d = dict.__getitem__(d,x)
            except [TypeError](TypeError):
                raise [KeyError](KeyError)("'' for '' not a dict",x,k) 
    try:
        dict.__setitem__(d,kk,v)
        d[kk] = v
    except [TypeError](TypeError):
        raise [KeyError](KeyError)("'' for '' not a dict",kk,k) 

class Counter: def init(self,a,l): self.a = a self.l = l self.i = 0 digs = 1+int(math.log10(l)) self.fmt = "(: :>Xd/:Xd)".replace("X",str(digs)) def call(self): self.i += 1 return self.fmt.format(self.a,self.i,self.l)

def exf(x): """Constructs function to extract named field when iterating over a list, [x] will be applied to each element """ return lambda t: t[x] def exfs(xs): """Constructs function to extract named fields as list""" return lambda t: lm(lambda q: t[q],x) def exfx(xs): """As exfs, except entries may be . delimited, and if so, recursion will happen."""

construct func list

fs = []
for x in xs:
    a = x.split(".")
    if len(a) == 1:
        fs.append(exf(x))
    else:
        def f(t):
            for y in a:
                t = t[y]
            return t
        fs.append(f)
def f(t):
    return lm(lambda q: q(t),fs)
return f

}}}%TIME=1483718310