6 files changed, 597 insertions, 0 deletions
diff --git a/apps/patchwork/bin/parsemail-batch.sh b/apps/patchwork/bin/parsemail-batch.sh
new file mode 100644
index 0000000..dbf81cc
--- /dev/null
+++ b/apps/patchwork/bin/parsemail-batch.sh
@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# Patchwork - automated patch tracking system
+# Copyright (C) 2008 Jeremy Kerr <jk@ozlabs.org>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+PATCHWORK_BASE="/srv/patchwork"
+
+if $# -ne 2
+then
+	echo "usage: $0 <dir>" >&2
+	exit 1
+fi
+
+mail_dir="$1"
+
+if ! -d "$mail_dir"
+then
+	echo "$mail_dir should be a directory"?&2
+	exit 1
+fi
+
+ls -1rt "$mail_dir" |
+while read line;
+do
+	echo $line
+	PYTHONPATH="$PATCHWORK_BASE/apps":"$PATCHWORK_BASE/lib/python" \
+		DJANGO_SETTINGS_MODULE=settings \
+		"$PATCHWORK_BASE/apps/patchworkbin/parsemail.py" <
+		"$mail_dir/$line"
+done
+
+
+
diff --git a/apps/patchwork/bin/parsemail.py b/apps/patchwork/bin/parsemail.py
new file mode 100755
index 0000000..d41bd92
--- /dev/null
+++ b/apps/patchwork/bin/parsemail.py
@@ -0,0 +1,263 @@
+#!/usr/bin/python
+#
+# Patchwork - automated patch tracking system
+# Copyright (C) 2008 Jeremy Kerr <jk@ozlabs.org>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import sys
+import re
+import datetime
+import time
+import operator
+from email import message_from_file
+from email.header import Header
+from email.utils import parsedate_tz, mktime_tz
+
+from patchparser import parse_patch
+from patchwork.models import Patch, Project, Person, Comment
+
+list_id_headers = ['List-ID', 'X-Mailing-List']
+
+def find_project(mail):
+    project = None
+    listid_re = re.compile('.*<([^>]+)>.*', re.S)
+
+    for header in list_id_headers:
+        if header in mail:
+            match = listid_re.match(mail.get(header))
+            if not match:
+                continue
+
+            listid = match.group(1)
+
+            try:
+                project = Project.objects.get(listid = listid)
+                break
+            except:
+                pass
+
+    return project
+
+def find_author(mail):
+
+    from_header = mail.get('From').strip()
+    (name, email) = (None, None)
+
+    # tuple of (regex, fn)
+    #  - where fn returns a (name, email) tuple from the match groups resulting
+    #    from re.match().groups()
+    from_res = [
+        # for "Firstname Lastname" <example@example.com> style addresses
+       (re.compile('"?(.*?)"?\s*<([^>]+)>'), (lambda g: (g[0], g[1]))),
+
+       # for example@example.com (Firstname Lastname) style addresses
+       (re.compile('"?(.*?)"?\s*\(([^\)]+)\)'), (lambda g: (g[1], g[0]))),
+
+       # everything else
+       (re.compile('(.*)'), (lambda g: (None, g[0]))),
+    ]
+
+    for regex, fn in from_res:
+        match = regex.match(from_header)
+        if match:
+            (name, email) = fn(match.groups())
+            break
+
+    if email is None:
+        raise Exception("Could not parse From: header")
+
+    email = email.strip()
+    if name is not None:
+        name = name.strip()
+
+    try:
+        person = Person.objects.get(email = email)
+    except Person.DoesNotExist:
+        person = Person(name = name, email = email)
+
+    return person
+
+def mail_date(mail):
+    t = parsedate_tz(mail.get('Date', ''))
+    if not t:
+        print "using now()"
+        return datetime.datetime.utcnow()
+    return datetime.datetime.utcfromtimestamp(mktime_tz(t))
+
+def mail_headers(mail):
+    return reduce(operator.__concat__,
+            ['%s: %s\n' % (k, Header(v, header_name = k, \
+                    continuation_ws = '\t').encode()) \
+                for (k, v) in mail.items()])
+
+def find_content(project, mail):
+    patchbuf = None
+    commentbuf = ''
+
+    for part in mail.walk():
+        if part.get_content_maintype() != 'text':
+            continue
+
+        #print "\t%s, %s" % \
+        #    (part.get_content_subtype(), part.get_content_charset())
+
+        charset = part.get_content_charset()
+        if not charset:
+            charset = mail.get_charset()
+        if not charset:
+            charset = 'utf-8'
+
+        payload = unicode(part.get_payload(decode=True), charset, "replace")
+
+        if part.get_content_subtype() == 'x-patch':
+            patchbuf = payload
+
+        if part.get_content_subtype() == 'plain':
+            if not patchbuf:
+                (patchbuf, c) = parse_patch(payload)
+            else:
+                c = payload
+
+            if c is not None:
+                commentbuf += c.strip() + '\n'
+
+    patch = None
+    comment = None
+
+    if patchbuf:
+        mail_headers(mail)
+        patch = Patch(name = clean_subject(mail.get('Subject')),
+                content = patchbuf, date = mail_date(mail),
+                headers = mail_headers(mail))
+
+    if commentbuf:
+        if patch:
+	    cpatch = patch
+	else:
+            cpatch = find_patch_for_comment(mail)
+            if not cpatch:
+                return (None, None)
+        comment = Comment(patch = cpatch, date = mail_date(mail),
+                content = clean_content(commentbuf),
+                headers = mail_headers(mail))
+
+    return (patch, comment)
+
+def find_patch_for_comment(mail):
+    # construct a list of possible reply message ids
+    refs = []
+    if 'In-Reply-To' in mail:
+        refs.append(mail.get('In-Reply-To'))
+
+    if 'References' in mail:
+        rs = mail.get('References').split()
+        rs.reverse()
+        for r in rs:
+            if r not in refs:
+                refs.append(r)
+
+    for ref in refs:
+        patch = None
+
+        # first, check for a direct reply
+        try:
+            patch = Patch.objects.get(msgid = ref)
+            return patch
+        except Patch.DoesNotExist:
+            pass
+
+        # see if we have comments that refer to a patch
+        try:
+            comment = Comment.objects.get(msgid = ref)
+            return comment.patch
+        except Comment.DoesNotExist:
+            pass
+
+
+    return None
+
+re_re = re.compile('^(re|fwd?)[:\s]\s*', re.I)
+prefix_re = re.compile('^\[.*\]\s*')
+whitespace_re = re.compile('\s+')
+
+def clean_subject(subject):
+    subject = re_re.sub(' ', subject)
+    subject = prefix_re.sub('', subject)
+    subject = whitespace_re.sub(' ', subject)
+    return subject.strip()
+
+sig_re = re.compile('^(-{2,3} ?|_+)\n.*', re.S | re.M)
+def clean_content(str):
+    str = sig_re.sub('', str)
+    return str.strip()
+
+def main(args):
+    mail = message_from_file(sys.stdin)
+
+    # some basic sanity checks
+    if 'From' not in mail:
+        return 0
+
+    if 'Subject' not in mail:
+        return 0
+
+    if 'Message-Id' not in mail:
+        return 0
+
+    hint = mail.get('X-Patchwork-Hint', '').lower()
+    if hint == 'ignore':
+        return 0;
+
+    project = find_project(mail)
+    if project is None:
+        print "no project found"
+        return 0
+
+    msgid = mail.get('Message-Id').strip()
+
+    author = find_author(mail)
+
+    (patch, comment) = find_content(project, mail)
+
+    if patch:
+        author.save()
+        patch.submitter = author
+        patch.msgid = msgid
+        patch.project = project
+        try:
+            patch.save()
+        except Exception, ex:
+            print ex.message
+
+    if comment:
+        author.save()
+        # looks like the original constructor for Comment takes the pk
+        # when the Comment is created. reset it here.
+        if patch:
+            comment.patch = patch
+        comment.submitter = author
+        comment.msgid = msgid
+        try:
+            comment.save()
+        except Exception, ex:
+            print ex.message
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))
diff --git a/apps/patchwork/bin/parsemail.sh b/apps/patchwork/bin/parsemail.sh
new file mode 100755
index 0000000..0178e18
--- /dev/null
+++ b/apps/patchwork/bin/parsemail.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+# Patchwork - automated patch tracking system
+# Copyright (C) 2008 Jeremy Kerr <jk@ozlabs.org>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+PATCHWORK_BASE="/srv/patchwork"
+
+PYTHONPATH="$PATCHWORK_BASE/apps":"$PATCHWORK_BASE/lib/python" \
+        DJANGO_SETTINGS_MODULE=settings \
+        "$PATCHWORK_BASE/apps/patchworkbin/parsemail.py"
+
+exit 0
diff --git a/apps/patchwork/bin/patchparser.py b/apps/patchwork/bin/patchparser.py
new file mode 100644
index 0000000..16d1de4
--- /dev/null
+++ b/apps/patchwork/bin/patchparser.py
@@ -0,0 +1,158 @@
+#!/usr/bin/python
+#
+# Patchwork - automated patch tracking system
+# Copyright (C) 2008 Jeremy Kerr <jk@ozlabs.org>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+
+import re
+
+def parse_patch(text):
+    patchbuf = ''
+    commentbuf = ''
+    buf = ''
+
+    # state specified the line we just saw, and what to expect next
+    state = 0
+    # 0: text
+    # 1: suspected patch header (diff, ====, Index:)
+    # 2: patch header line 1 (---)
+    # 3: patch header line 2 (+++)
+    # 4: patch hunk header line (@@ line)
+    # 5: patch hunk content
+    #
+    # valid transitions:
+    #  0 -> 1 (diff, ===, Index:)
+    #  0 -> 2 (---)
+    #  1 -> 2 (---)
+    #  2 -> 3 (+++)
+    #  3 -> 4 (@@ line)
+    #  4 -> 5 (patch content)
+    #  5 -> 1 (run out of lines from @@-specifed count)
+    #
+    # Suspected patch header is stored into buf, and appended to
+    # patchbuf if we find a following hunk. Otherwise, append to
+    # comment after parsing.
+
+    # line counts while parsing a patch hunk
+    lc = (0, 0)
+    hunk = 0
+
+    hunk_re = re.compile('^\@\@ -\d+(?:,(\d+))? \+\d+(?:,(\d+))? \@\@')
+
+    for line in text.split('\n'):
+        line += '\n'
+
+        if state == 0:
+            if line.startswith('diff') or line.startswith('===') \
+                    or line.startswith('Index: '):
+                state = 1
+                buf += line
+
+            elif line.startswith('--- '):
+                state = 2
+                buf += line
+
+            else:
+                commentbuf += line
+
+        elif state == 1:
+            buf += line
+            if line.startswith('--- '):
+                state = 2
+
+        elif state == 2:
+            if line.startswith('+++ '):
+                state = 3
+                buf += line
+
+            elif hunk:
+                state = 1
+                buf += line
+
+            else:
+                state = 0
+                commentbuf += buf + line
+                buf = ''
+
+        elif state == 3:
+            match = hunk_re.match(line)
+            if match:
+
+                def fn(x):
+                    if not x:
+                        return 1
+                    return int(x)
+
+                lc = map(fn, match.groups())
+
+                state = 4
+                patchbuf += buf + line
+                buf = ''
+
+            elif line.startswith('--- '):
+                patchbuf += buf + line
+                buf = ''
+                state = 2
+
+            elif hunk:
+                state = 1
+                buf += line
+
+            else:
+                state = 0
+                commentbuf += buf + line
+                buf = ''
+
+        elif state == 4 or state == 5:
+            if line.startswith('-'):
+                lc[0] -= 1
+            elif line.startswith('+'):
+                lc[1] -= 1
+            else:
+                lc[0] -= 1
+                lc[1] -= 1
+
+            patchbuf += line
+
+            if lc[0] <= 0 and lc[1] <= 0:
+                state = 3
+                hunk += 1
+            else:
+                state = 5
+
+        else:
+            raise Exception("Unknown state %d! (line '%s')" % (state, line))
+
+    commentbuf += buf
+
+    if patchbuf == '':
+        patchbuf = None
+
+    if commentbuf == '':
+        commentbuf = None
+
+    return (patchbuf, commentbuf)
+
+if __name__ == '__main__':
+    import sys
+    (patch, comment) = parse_patch(sys.stdin.read())
+    if patch:
+        print "Patch: ------\n" + patch
+    if comment:
+        print "Comment: ----\n" + comment
diff --git a/apps/patchwork/bin/setup.py b/apps/patchwork/bin/setup.py
new file mode 100755
index 0000000..7d55815
--- /dev/null
+++ b/apps/patchwork/bin/setup.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+#
+# Patchwork - automated patch tracking system
+# Copyright (C) 2008 Jeremy Kerr <jk@ozlabs.org>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+
+from patchwork.models import UserProfile
+from django.contrib.auth.models import User
+
+# give each existing user a userprofile
+for user in User.objects.all():
+    p = UserProfile(user = user)
+    p.save()
diff --git a/apps/patchwork/bin/update-patchwork-status.py b/apps/patchwork/bin/update-patchwork-status.py
new file mode 100755
index 0000000..c774d63
--- /dev/null
+++ b/apps/patchwork/bin/update-patchwork-status.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+#
+# Patchwork - automated patch tracking system
+# Copyright (C) 2008 Jeremy Kerr <jk@ozlabs.org>
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+
+import sys
+import subprocess
+from optparse import OptionParser
+
+def commits(options, revlist):
+    cmd = ['git-rev-list', revlist]
+    proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd = options.repodir)
+
+    revs = []
+
+    for line in proc.stdout.readlines():
+        revs.append(line.strip())
+
+    return revs
+
+def commit(options, rev):
+    cmd = ['git-diff', '%(rev)s^..%(rev)s' % {'rev': rev}]
+    proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, cwd = options.repodir)
+
+    buf = proc.communicate()[0]
+
+    return buf
+
+
+def main(args):
+    parser = OptionParser(usage = '%prog [options] revspec')
+    parser.add_option("-p", "--project", dest = "project", action = 'store',
+                  help="use project PROJECT", metavar="PROJECT")
+    parser.add_option("-d", "--dir", dest = "repodir", action = 'store',
+                  help="use git repo in DIR", metavar="DIR")
+
+    (options, args) = parser.parse_args(args[1:])
+
+    if len(args) != 1:
+        parser.error("incorrect number of arguments")
+
+    revspec = args[0]
+    revs = commits(options, revspec)
+
+    for rev in revs:
+        print rev
+        print commit(options, rev)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))
+
+