# import commands
import logging
import re

class HTMLDocument(PluginInterface.File):

	title = None
	contents = None
	encoding = None

	HTMLDocument_catalogable = [ "title","contents","encoding" ]

	def __init__(self,filename,mimetype,volume):
		PluginInterface.File.__init__(self,filename,mimetype,volume)
		
		f = file(filename,"rb")
		output = f.read(-1)
		f.close()
			
		filerepr = HTMLFile(output)

		try:
			encoding = filerepr.get_encoding()
			utftitle = filerepr.get_title_utf8()
			utfbody = filerepr.get_plaintext_utf8()
		except (TypeError,LookupError):
			PluginInterface.log("text-html").info("%s's declared encoding %s is invalid",filename,repr(encoding))
			return
			

		try:
			self.encoding = encoding
			if utftitle: self.title = PluginInterface.utf8_to_platform(utftitle)
			if utfbody: self.contents = PluginInterface.utf8_to_platform(utfbody)
		except (UnicodeDecodeError, UnicodeEncodeError):
			PluginInterface.log("text-html").info("%s is in an encoding %s that is unsupported by the platform",filename,repr(encoding),exc_info=True)


	def __str__(self):
		return "HTML document %s (%s)"%(self.get_filename(),self.title)


import htmlentitydefs
class HTMLFile:

	title = None
	sent_markup = None
	utf8_markup = None
	encoding = None
	utf8_body = None
	plaintext = None
	
	def __init__(self,markup):
		if type(markup) is not str:
			raise TypeError, "HTMLFiles can only be instanced with regular strings"
		self.sent_markup = markup
		# save the markup
		
	def __process(self):
		# detect encoding  DONE
# 		encoding = self.get_encoding()
		# transcode everything to UTF-8
# 		utf8_markup = self.get_utf8_markup()
		# get the body
		body,title = self.get_body_and_title()
		# remove all tags, putting spaces wherever was one
		# replace all entities for their Unicode codepoints
		def strip_scripts(text):
			return re.compile("<script.*?>.*?</script.*?>",re.DOTALL|re.IGNORECASE).sub("",text)
		def strip_comments(text):
			return re.compile("<!.*?-->",re.DOTALL|re.IGNORECASE).sub("",text)
		def strip_tags(text):
			return re.compile("<.*?>",re.DOTALL).sub("",text)
		def replace_entities(text):
			def convert_codepoint_to_utf8_char(codepoint):
				cp = str(codepoint)
				return ("\u" + "0" * (4 - len(cp)) + cp).decode("unicode_escape").encode("UTF-8")
			def map_entity_to_codepoint(entity):
				import htmlentitydefs
				entity = entity.group(1)
				if entity[0] == "#": return convert_codepoint_to_utf8_char(entity[1:])
				else:
					d = htmlentitydefs.name2codepoint
					if d.has_key(entity[1:]): return convert_codepoint_to_utf8_char(d[entity[1:]])
				return ""
			return re.compile("&(.*?);").sub(map_entity_to_codepoint,text)
		if body:
			bodytext = body
			bodytext = strip_comments(bodytext)
			bodytext = strip_scripts(bodytext)
			bodytext = strip_tags(bodytext)
			bodytext = replace_entities(bodytext)
			bodytext = PluginInterface.remove_whitespace(bodytext)
			body = bodytext
			self.plaintext = body
		if title:
			titletext = title
			titletext = strip_tags(titletext)
			titletext = replace_entities(titletext)
			titletext = PluginInterface.remove_whitespace(titletext)
			title = titletext
			self.title = title
		# save the final text in a member variable

	def get_utf8_markup(self):
		if self.utf8_markup is None:
			original_encoding = self.get_encoding()
			self.utf8_markup = PluginInterface.transcode(self.sent_markup,original_encoding,"UTF-8")
		return self.utf8_markup
	
	def get_declared_encoding(self):
		# return the file's declared encoding
		def get_charset_regexp():
			charset="charset"
			igual="="
			definit = ".*?"
			comilla = "[" + "'" + '"' + "]"
			expresion = charset + igual + "(%s)"%definit + comilla
			return expresion
		def get_encoding_regexp():
			encoding="encoding"
			igual="="
			definit = ".*?"
			comilla = "[" + "'" + '"' + "]"
			expresion = encoding + igual + comilla + "(%s)"%definit + comilla
			return expresion
		
		objeto = re.compile(get_encoding_regexp(),re.IGNORECASE)
		cuadre = objeto.search(self.sent_markup)
		if cuadre:
			return cuadre.group(1).strip()
		objeto = re.compile(get_charset_regexp(),re.IGNORECASE)
		cuadre = objeto.search(self.sent_markup)
		if cuadre:
			return cuadre.group(1).strip()

	def get_encoding(self):
		if self.encoding is None:
		
			# auto-detect encoding
			self.encoding = self.get_declared_encoding()
			if self.encoding is None:
				encoding = PluginInterface.detect_encoding(self.sent_markup)
				if encoding: self.encoding = encoding
				else: self.encoding = "ISO-8859-1" #fallback!
		
		return self.encoding
		
	def get_body_and_title(self):
		utf8_markup = self.get_utf8_markup()
		bodyobject = re.compile("<body.*?>(.*)</body.*?>",re.DOTALL | re.IGNORECASE )
		bodymatch = bodyobject.search(utf8_markup)
		titleobject = re.compile("<title.*?>(.*)</title.*?>",re.DOTALL | re.IGNORECASE )
		titlematch = titleobject.search(utf8_markup)
		if bodymatch: body = bodymatch.group(1).strip()
		else: body = ""
		if titlematch: title = titlematch.group(1).strip()
		else: title = ""
		
		return (body,title)
		
	def get_title_utf8(self):
		if self.title is None:
			self.__process()
		return self.title
		
	def get_plaintext_utf8(self):
		if self.plaintext is None:
			self.__process()
		return self.plaintext



def get_factory():
	return HTMLDocument

def get_mimetypes():
	return ["text/html"]
