|
|
| Zeile 1: |
Zeile 1: |
| #!/usr/bin/env python
| | Script findet sich hier: |
| # -*- coding: utf-8 -*-
| | https://gist.github.com/3488254 |
| import urllib2
| |
| import json
| |
| import os
| |
| import csv
| |
| | |
| API_URL = "http://wiki.piratenbrandenburg.de/api.php"
| |
| CATEGORIES = ["Kategorie:Sonstiger_Antrag_AF_LPT_2012.1",
| |
| "Kategorie:Satzungsänderungsantrag_AF_LPT_2012.1",
| |
| "Kategorie:Programmantrag_AF_LPT_2012.1"]
| |
| MAX_PAGEIDS = 50
| |
| | |
| def get_json(endpoint):
| |
| url = ''.join([
| |
| API_URL,
| |
| '?',
| |
| endpoint,
| |
| '&format=json',
| |
| ])
| |
| return urllib2.urlopen(url).read()
| |
| | |
| def get_category(category, query_continue=""):
| |
| data = get_json("action=query&list=categorymembers&cmtitle=%s&cmcontinue=%s" % (category, query_continue))
| |
| json_data = json.loads(data)
| |
| pages = json_data["query"]["categorymembers"]
| |
| if "query-continue" in json_data:
| |
| pages += get_category(category,json_data["query-continue"]["categorymembers"]["cmcontinue"])
| |
| return pages
| |
| | |
| def list_applications(categories):
| |
| if os.path.isfile("application_list"):
| |
| f = open('application_list','r')
| |
| return json.load(f)
| |
| return download_applications(categories)
| |
| | |
| def download_applications(categories):
| |
| applications = _list_applications(categories)
| |
| f = open('application_list','w+')
| |
| json.dump(applications, f)
| |
| f.flush()
| |
| return applications
| |
| | |
| def _list_applications(categories):
| |
| applications = {}
| |
| for category in categories:
| |
| pages = get_category(category)
| |
| applications[category] = pages
| |
| return applications
| |
| | |
| def get_raw_pageid(pageid):
| |
| data = get_json("action=query&prop=revisions&rvprop=content&pageids=%s" % pageid)
| |
| json_data = json.loads(data)
| |
| pages = json_data["query"]["pages"]
| |
| content = []
| |
| for pageids in pages:
| |
| content += pages[pageids]["revisions"]
| |
| return content
| |
| | |
| def chunks(l, n):
| |
| for i in xrange(0, len(l), n):
| |
| yield l[i:i+n]
| |
| | |
| def get_pageid(pageids):
| |
| pages = []
| |
| for chunk in chunks(pageids, MAX_PAGEIDS):
| |
| pages += get_raw_pageid("|".join(str(i) for i in chunk))
| |
| return pages
| |
| | |
| def _list_content(applications):
| |
| pageids = {}
| |
| content = {}
| |
| for category in applications.iterkeys():
| |
| for application in applications[category]:
| |
| if category in pageids:
| |
| pageids[category] += [application["pageid"]]
| |
| else:
| |
| pageids[category] = [application["pageid"]]
| |
| content[category] = get_pageid(pageids[category])
| |
| return content
| |
| | |
| def download_content(applications):
| |
| content = _list_content(applications)
| |
| f = open('content','w+')
| |
| json.dump(content,f)
| |
| f.flush()
| |
| return content
| |
| | |
| def list_content(applications):
| |
| if os.path.isfile("content"):
| |
| f = open('content','r')
| |
| return json.load(f)
| |
| return download_content(applications)
| |
| | |
| def parse_content(content):
| |
| applications = {}
| |
| for category in content.iterkeys():
| |
| applications_for_category = []
| |
| for application_content in content[category]:
| |
| application = mediawiki_template(application_content["*"])
| |
| if application["Eingereicht"] != "":
| |
| applications_for_category.append(application)
| |
| applications_for_category.sort(key = lambda a: a["Titel"])
| |
| applications[category] = applications_for_category
| |
| return applications
| |
| | |
| def mediawiki_template(mw_string):
| |
| """ returns media wiki template element as a hash"""
| |
| #Split content inside Template
| |
| strings = mw_string.split("{{")[1].split("}}")[0].split("\n|")
| |
| #remove "Antragsfabrikat"
| |
| strings = strings[1:]
| |
| mw_hash = {}
| |
| for string in strings:
| |
| keyval = string.split("=",1)
| |
| if 2 != len(keyval):
| |
| raise SyntaxError("Mediawiki parsing Error %s" % keyval)
| |
| keyval = [s.strip() for s in keyval]
| |
| key, val = keyval
| |
| mw_hash[key] = val
| |
| return mw_hash
| |
| | |
| def filter_content(content):
| |
| """ simple filter for some html tags to plain text"""
| |
| content = content.replace("<sup>1</sup>","¹")
| |
| content = content.replace("<sup>2</sup>","²")
| |
| content = content.replace("<sup>3</sup>","³")
| |
| content = content.replace("<br>","\n")
| |
| content = content.replace("<br\>","\n")
| |
| content = content.replace("<br\\n>","\n")
| |
| content = content.replace("<br />","\n")
| |
| content = content.replace("<br/>","\n")
| |
| return content
| |
| | |
| | |
| def write_content(applications, applications_position=[]):
| |
| open_position = []
| |
| open_position.extend(applications_position)
| |
| for category in applications:
| |
| f = open(category,'w+')
| |
| writer = csv.writer(f,delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
| |
| writer.writerow( ("Number","Title","Text","Reason",
| |
| "Submitter (First Name)","Submitter (Last Name)"))
| |
| for a in applications[category]:
| |
| try:
| |
| number = applications_position.index(a["Titel"]) + 1
| |
| open_position.remove(a["Titel"])
| |
| except ValueError:
| |
| print '"' + a["Titel"] + '" im Antragsbuch nicht gefunden'
| |
| number = ""
| |
| writer.writerow( ( number, # number starts at 1
| |
| a["Titel"].encode('utf8'),
| |
| filter_content(a["Antragstext"].encode('utf8')),
| |
| filter_content(a[u'Begr\xfcndung'].encode('utf8')),
| |
| a["Antragsteller"].encode('utf8'),
| |
| "") ) #Last Name
| |
| f.flush()
| |
| f.close()
| |
| if open_position != []:
| |
| print "Anträge aus dem Antragsbuch, die nicht gefunden wurden: "
| |
| for a in open_position:
| |
| print a
| |
| | |
| def get_application_positions(filename):
| |
| f = open(filename,'r')
| |
| lines = [l.strip().decode('utf8') for l in f.readlines()]
| |
| return lines
| |
| | |
| if __name__ == '__main__':
| |
| #download_applications(CATEGORIES)
| |
| applications = list_applications(CATEGORIES)
| |
| #download_content(applications)
| |
| content = list_content(applications)
| |
| applications = parse_content(content)
| |
| #Ein Titel per Zeile, TO-Reihenfolge gegeben
| |
| #positions = get_application_positions("reihenfolge-to")
| |
| write_content(applications, positions)
| |
