grep -r in python

Question

i'd like to implement the unix command 'grep -r' in a python function. i know about commands.getstatusoutput(), but for now i don't want to use that. i came up with this:

def grep_r (str, dir):
    files = [ o[0]+"/"+f for o in os.walk(dir) for f in o[2] if os.path.isfile(o[0]+"/"+f) ]
    return [ l for f in files for l in open(f) if str in l ]

but that of course doesn't use a regex, it just checks if 'str' is a substring of 'l'. so i tried the following:

def grep_r (pattern, dir):
    r = re.compile(pattern)
    files = [ o[0]+"/"+f for o in os.walk(dir) for f in o[2] if os.path.isfile(o[0]+"/"+f) ]
    return [ l for f in files for l in open(f) if r.match(l) ]

but that doesn't work, it doesn't give me any matches even where the former function did. what changed? i could just split it up into a bunch of nested loops, but i'm more interested in being succinct than readable.

Answer 1

re.match only checks the beginning of the string.

Use re.search()

From the docs:

Python offers two different primitive operations based on regular expressions: match checks for a match only at the beginning of the string, while search checks for a match anywhere in the string (this is what Perl does by default).

Answer 2

You might want to search() instead of match() to catch matches in the middle of lines, as noted in http://docs.python.org/library/re.html#matching-vs-searching

Also, the structure and intent of your code is quite hidden. I've pythonized it.

def grep_r (pattern, dir):
    r = re.compile(pattern)
    for parent, dnames, fnames in os.walk(dir):
        for fname in fnames:
            filename = os.path.join(parent, fname)
            if os.path.isfile(filename):
                with open(filename) as f:
                    for line in f:
                        if r.search(line):
                            yield line

Answer 3

import os, re

def grep_r(regex, dir):
    for root, dirs, files in os.walk(dir):
        for f in files:
            for m in grep(regex, os.path.join(root, f)):
                yield m

def grep(regex, filename):
    for i, line in enumerate(open(filename)):
        if re.match(regex, line): # or re.search depending on your default
           yield "%s:%d: %s" % (os.path.basename(filename), i+1, line)

Answer 4

why do you need to use regex?

path=os.path.join("/dir1","dir2","dir3")
pattern="test"
for r,d,f in os.walk(path):
    for files in f:
        for n,line in enumerate(open( os.path.join(r,files) ) ):
            if pattern in line:
                print "%s found in line: %d of file: %s" %(pattern, n+1, files)

Answer 5

Put all this code into a file called pygrep and chmod +x pygrep:

#!/usr/bin/python

import os
import re
import sys

def file_match(fname, pat):
    try:
        f = open(fname, "rt")
    except IOError:
        return
    for i, line in enumerate(f):
        if pat.search(line):
            print "%s: %i: %s" % (fname, i+1, line)
    f.close()


def grep(dir_name, s_pat):
    pat = re.compile(s_pat)
    for dirpath, dirnames, filenames in os.walk(dir_name):
        for fname in filenames:
            fullname = os.path.join(dirpath, fname)
            file_match(fullname, pat)

if len(sys.argv) != 3:
    u = "Usage: pygrep <dir_name> <pattern>\n"
    sys.stderr.write(u)
    sys.exit(1)

grep(sys.argv[1], sys.argv[2])