')
return doc
def monobook_hack_skin_css(doc, url):
"""
Hacks Mediawiki 1.4beta6 Monobook main CSS file for better looks.
Removes flower background. Defines *Hacked CSS ids, so we can add
an orange bar at the top, and clear the orange bar right above the
footer.
"""
global config
if not url.endswith('monobook/main.css'):
return doc
doc = "/* Monobook skin automatically modified by mw2html. */" + doc
doc = doc.replace('url("headbg.jpg")', '')
doc += """
/* Begin hacks by mw2html */
#globalWrapperHacked {
font-size:127%;
width: 100%;
background-color: White;
border-top: 1px solid #fabd23;
border-bottom: 1px solid #fabd23;
margin: 0.6em 0em 1em 0em;
padding: 0em 0em 1.2em 0em;
}
#footerHacked {
background-color: White;
margin: 0.6em 0em 0em 0em;
padding: 0.4em 0em 0em 0em;
text-align: center;
font-size: 90%;
}
#footerHacked li {
display: inline;
margin: 0 1.3em;
}
"""
c1 = '#column-one { padding-top: 160px; }'
c2 = '#column-one { padding-top: 3.0em; }'
assert c1 in doc
doc = doc.replace(c1, '/* edit by mw2html */\n' + c2 +
'\n/* end edit by mw2html */\n')
# Remove external link icons.
if config.remove_png:
doc = re.sub(r'#bodyContent a\[href \^="http://"\][\s\S]+?\}', r'', doc)
return doc
def post_css_transform(doc, url):
"""
User-customizable CSS transform.
Given a CSS document (with URLs already rewritten), returns
modified CSS document.
"""
global config
if config.hack_skin and not config.special_mode:
if config.skin == MONOBOOK_SKIN:
doc = monobook_hack_skin_css(doc, url)
else:
raise ValueError('unknown skin')
return doc
def move_to_index_if_needed(ans):
global config
if ans.endswith(config.index):
ans = ans[:len(ans) - len(config.index)] + INDEX_HTML
return ans
def file_exists_in_written_set(filename):
return os.path.normcase(os.path.normpath(filename)) in wrote_file_set
def find_unused_filename(filename, exists=os.path.exists):
"""
Return 'file' if 'file' doesn't exist, otherwise 'file1', 'file2', etc.
Existance is determined by the callable exists(), which takes
a filename and returns a boolean.
"""
if not exists(filename):
return filename
(head, tail) = os.path.split(filename)
i = 1
while True:
numbered = (os.path.splitext(tail)[0] + str(i) +
os.path.splitext(tail)[1])
fullname = os.path.join(head, numbered)
if not exists(fullname):
return fullname
i += 1
def clean_filename(url, ans):
# Split outdir and our file/dir under outdir
# (Note: ans may not be a valid filename)
global config
(par, ans) = (ans[:len(config.outdir)], ans[len(config.outdir):])
if ans.startswith(os.sep):
ans = ans[1:]
# Replace % escape codes with underscores, dashes with underscores.
while '%%' in ans:
ans = ans[:ans.index('%%')] + '_' + ans[ans.index('%%') + 2:]
while '%25' in ans:
ans = ans[:ans.index('%25')] + '_' + ans[ans.index('%25') + 5:]
while '%' in ans:
ans = ans[:ans.index('%')] + '_' + ans[ans.index('%') + 3:]
ans = ans.replace('-', '_')
while '__' in ans:
ans = ans.replace('__', '_')
while '_.' in ans:
ans = ans.replace('_.', '.')
# Rename math thumbnails
if '/math/' in url:
tail = os.path.split(ans)[1]
if os.path.splitext(tail)[1] == '.png':
tail = os.path.splitext(tail)[0]
if set(tail) <= set('0123456789abcdef') and len(tail) == 32:
ans = 'math_' + hashlib.md5(tail).hexdigest()[:4] + '.png'
return os.path.join(par, ans)
def flatten_filename(url, filename):
global config
def get_fullname(relname):
return os.path.join(config.outdir, relname)
orig_ext = os.path.splitext(filename)[1]
(head, tail) = os.path.split(filename)
if tail == INDEX_HTML:
(head, tail) = os.path.split(head)
ans = tail
if os.path.splitext(ans)[1] != orig_ext:
ans = os.path.splitext(ans)[0] + orig_ext
return os.path.join(config.outdir, ans)
def split_section(url):
"""
Splits into (head, tail), where head contains no '#' and is max length.
"""
if '#' in url:
i = url.index('#')
return (url[:i], url[i:])
else:
return (url, '')
def url_open(url):
# download a file and retrieve its content and mimetype
global conn, domain, counter, redir_cache, errors, headers
l_redir = []
redirect = url
while redirect != '':
l_redir += [url]
L = urlparse.urlparse(url)
if L[1] != domain:
conn.close()
if L[1] == '': return(['',''])
print "connection to", domain, "closed."
conn = httplib.HTTPConnection(L[1])
domain = L[1]
print "connection to", domain, "opened."
rel_url = url
pos = url.find(domain)
if pos != -1:
rel_url = url[pos + len(domain):]
attempts = 0
#number of attempts
total_attempts = 3
recovered = False
success = False
while not success and attempts < total_attempts:
#increment httplib requests counter
counter += 1
try:
conn.request("GET", rel_url,headers=headers)
r = conn.getresponse()
print 'Status', r.status, r.reason, 'accessing', rel_url
if r.status == 404:
print " it's not possible to recover this error."
errors += 1
return ('', '')
if r.status == 500:
print " eventually this error might be recovered. let's try again."
print ' reconnecting...'
conn = httplib.HTTPConnection(domain)
attempts += 1
continue
if r.status == 403:
print " that shouldn't happen, but let's try again anyway."
print ' reconnecting...'
conn = httplib.HTTPConnection(domain)
attempts += 1
continue
if attempts != 0:
recovered = True
if r.status != 200:
print " Status other than 200, 404, 500, 403. It is: ", r.status
success = True
except httplib.HTTPException, e:
print 'ERROR', e.__class__.__name__, 'while retrieving', url
conn.close
if e.__class__.__name__ in ['BadStatusLine', 'ImproperConnectionState', 'NotConnected', 'IncompleteRead', 'ResponseNotReady']:
print "eventually this error might be recovered. let's try again."
print 'reconnecting...'
conn = httplib.HTTPConnection(domain)
attempts += 1
else:
print "it's not possible to recover this error."
errors += 1
return ('', '')
if recovered:
print "error recovered"
if not success:
print "it was not possible to recover this error."
errors += 1
return ('', '')
redirect = r.getheader('Location', '').split(';')[0]
if redirect != "":
url = redirect
else:
doc = r.read()
for item in l_redir:
redir_cache[normalize_url(item)] = normalize_url(url)
mimetype = r.getheader('Content-Type', '').split(';')[0].lower()
return (doc, mimetype)
def url_to_filename(url):
"""
Translate a full url to a full filename (in local OS format) under outdir.
Transforms web url into local url and caches it.
Downloads the file to disk and works with it there instead of download the same file two times (Performance Improvement).
"""
global config
nurl = normalize_url(url)
if nurl in url_filename_cache:
return url_filename_cache[nurl]
#ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='')
turl = re.sub(r'm/index.php\?title=', r'man/', nurl)
turl = re.sub(r'.css&[\S\s]+', r'.css', turl)
L = list(urlparse.urlparse(turl))
#this way the url will not create a folder outside of the maindomain
droot = get_domain(config.rooturl)
if (L[1] != droot):
L[1] = droot
L[2] = L[2].strip('/')
lpath = L[2].split('/')
if not '.' in lpath[-1]:
# url ends with a directory name. Store it under index.html.
# L[2] += '/' + INDEX_HTML
L[2]=L[2]
else:
# 'title=' parsing
if L[4].startswith('title=') and L[2].endswith('index.php'):
L[4] = L[4][len('title='):]
L[2] = L[2][:-len('index.php')]
if lpath[-1]=='man':
L[2] = INDEX_HTML
if lpath[-1].lower().startswith( 'quick_help'):
L[2] = QHELP_HTML
L[3] = ''
L[2] = L[2].strip('/')
#don't sanitize / for path
L[0] = ''
L[2] = urllib.quote_plus(L[2], '/')
L[3] = urllib.quote_plus(L[3])
L[4] = urllib.quote_plus(L[4])
L[5] = urllib.quote_plus(L[5])
# Local filename relative to outdir
# os.sep - O.S. directory separator
# (More transformations are made to this below...).
FL = []
for i in L:
if i != '':
FL += [i]
subfile = os.sep.join(FL)
(doc, mimetype) = url_open(url)
if doc == '' or mimetype == '':
url_filename_cache[nurl] = ''
return ''
# Fix up extension based on mime type.
# Maps mimetype to file extension
MIME_MAP = {
'image/jpeg': 'jpg', 'image/png': 'png', 'image/gif': 'gif',
'image/tiff': 'tiff', 'text/plain': 'txt', 'text/html': 'html',
'text/rtf': 'rtf', 'text/css': 'css', 'text/sgml': 'sgml',
'text/xml': 'xml', 'application/zip': 'zip'
}
if mimetype in MIME_MAP:
(root, ext) = os.path.splitext(subfile)
ext = '.' + MIME_MAP[mimetype]
subfile = root + ext
subfile = subfile.lower()
ans = os.path.join(config.outdir, subfile)
if config.flatten:
ans = flatten_filename(nurl, ans)
if config.clean:
ans = clean_filename(nurl, ans)
if config.index != None:
ans = move_to_index_if_needed(ans)
ans = find_unused_filename(ans, file_exists_in_written_set)
# Cache and return answer.
wrote_file_set.add(os.path.normcase(os.path.normpath(ans)))
url_filename_cache[nurl] = ans
mode = ['wb', 'w'][mimetype.startswith('text')]
# Make parent directory if it doesn't exist.
try:
os.makedirs(os.path.split(ans)[0])
except OSError, e:
if e.errno != errno.EEXIST:
raise
# Not really needed since we checked that the directory
# outdir didn't exist at the top of run(), but let's double check.
if os.path.exists(ans) and not config.overwrite:
out.write('File already exists: ' + str(ans)) #@UndefinedVariable
sys.exit(1)
f = open(ans, mode)
f.write(doc)
f.close()
return ans
def url_to_relative(url, cururl):
"""
Translate a full url to a filename (in URL format) relative to cururl.
Relative url from curul to url.
"""
cururl = split_section(cururl)[0]
(url, section) = split_section(url)
L1 = url_to_filename(url).replace(os.sep, '/').strip('/').split('/')
if L1 == '':
return ''
L2 = url_to_filename(cururl).replace(os.sep, '/').strip('/').split('/')
while L1 != [] and L2 != [] and L1[0] == L2[0]:
L1 = L1[1:]
L2 = L2[1:]
rel_url = urllib.quote('../' * (len(L2) - 1) + '/'.join(L1)) + section
if rel_url == '':
return '#'
else:
return rel_url
def parse_css(doc, url):
"""
Returns (modified_doc, new_urls), where new_urls are absolute URLs for
all links found in the CSS.
"""
global config
new_urls = []
L = htmldata.urlextract(doc, url, 'text/css')
for item in L:
# Store url locally.
u = item.url
if config.no_images and any(u.strip().lower().endswith(suffix) for suffix in ('.jpg', '.gif', '.png', '.ico')):
item.url = ''
continue
new_urls += [u]
item.url = url_to_relative(u, url)
newdoc = htmldata.urljoin(doc, L)
newdoc = post_css_transform(newdoc, url)
return (newdoc, new_urls)
def should_follow(url):
"""
Returns a boolean for whether url should be spidered
Given that 'url' was linked to from site, return whether
'url' should be spidered as well.
"""
global config
# we don't have search on the local version
if (url.endswith('#searchInput')):
return False
# False if different domains.
nurl = normalize_url(url)
droot = get_domain(config.rooturl)
dn = get_domain(nurl)
if droot != dn and not (dn.endswith(droot) or droot.endswith(dn)):
if config.debug:
print url, 'not in the same domain'
return False
# False if multiple query fields or parameters found
if (url.count('&') >= 1 or url.count(';') > 0) and not any(x in url for x in ('.css', 'gen=css')):
if config.debug:
print url, 'with multiple query fields'
return False
if any(x in url for x in ('Special:', 'Image:', 'Talk:', 'User:', 'Help:', 'User_talk:', 'MediaWiki_talk:', 'File:', 'action=edit', 'title=-')):
if config.debug:
print url, 'is a forbidden wiki page'
return False
if config.no_images and any(url.strip().lower().endswith(suffix) for suffix in ('.jpg', '.gif', '.png', '.ico')):
if config.debug:
print url, 'is a image and you are in no-images mode'
return False
if any(url.strip().lower().endswith(suffix) for suffix in ('.zip', '.7z')):
if config.debug:
print url, 'is a compressed file'
return False
# limit_parent support
ncurl = normalize_url(config.rooturl)
if config.limit_parent and not nurl.startswith(ncurl):
L = nurl.split('/')
if ('.' not in L[-1]):
if config.debug:
print url, 'is a file outside of scope with unknown extension'
return False
# JKC: we do allow css from 'strange' places.
if '.css' in L[-1]:
return True
forbidden_parents = ['.php', '.html', '.htm']
for fp in forbidden_parents:
if fp in L[-1]:
if config.debug:
print url, 'is a page outside of scope'
return False
return True
def parse_html(doc, url):
"""
Returns (modified_doc, new_urls), where new_urls are absolute URLs for
all links we want to spider in the HTML.
"""
global config
BEGIN_COMMENT_REPLACE = '