from VolumeCrawlerManager import VolumeCrawlerManager

from MetadataService.InfospaceAgent import InfospaceAgent
from MetadataService.ProducerConsumer import Consumer, Empty
from FilePluginManager import FilePluginManager
from VolumeExpert import VolumeExpert
from VolumeCrawlerManager import VolumeCrawlerManager
from InfospaceObjects import FilesystemInfospace, Volume
from platform import strip_mountpoint, is_or_is_below
import time
import os
import ConfigParser
# FIXME: wrap this database error and other database errors produced by postgresql in nice Catalog errors, and replace this stuff with the replaced Catalog errors
# in other words, this is just a band-aid
from pgdb import DatabaseError

class FilesystemAgent(InfospaceAgent,Consumer):

	file_plugin_manager = None
	log = None
	crawler_manager = None
	volume_expert = None
	catalog = None

	def __init__(self,index,config):
		InfospaceAgent.__init__(self,public_interface=None,index=index,infospace_name="filesystem",config=config)
		self.set_component_name("FilesystemAgent")
		Consumer.__init__(self)
		self.file_plugin_manager = FilePluginManager()

		try:
			stoplist = self.config.get("filesystem infospace","exclude")
			if stoplist: stoplist = stoplist.split(":")
			else: stoplist = []
		except (ConfigParser.NoOptionError,ConfigParser.NoSectionError):
			stoplist = []
		dbdir = self.config.get("index","path")
		stoplist = stoplist + [ dbdir ]

		self.log.debug("Using stoplist %s",stoplist)
		self.volume_expert = VolumeExpert(stoplist)
		self.crawler_manager = VolumeCrawlerManager(self.volume_expert)
		self.consume_from(self.crawler_manager)
		
		# FIXME make the db name configurable
		# FIXME make the metadata service start its own postgres daemon
		
	def check_index_schema(self):
		root = self.context.root()
		self.infospaces = root["infospaces"]
		if not self.infospaces.has_key(self.infospace_name):
			self.context.begin_transaction()
			self.log.debug("Registering %s infospace",self.infospace_name)
			self.infospaces[self.infospace_name] = FilesystemInfospace()
			self.context.commit_transaction()
		self.infospace = self.infospaces[self.infospace_name]

	def register_currently_known_volumes(self):
		self.volumes = self.infospace.volumes
		known_volumes = self.volume_expert.get_currently_known_volumes()
		for guid,vr in known_volumes.iteritems():
			# we will not register network volumes, ever
			# we will not register loop-mounted volumes for the time being FIXME?
			# we will also not register removable volumes FIXME
			if vr.loop_mounted:
				self.log.debug("Not registering volume %s (loop-mounted)",guid)
				continue
			if vr.network_volume:
				self.log.debug("Not registering volume %s (network volume)",guid)
				continue
			if vr.removable:
				self.log.debug("Not registering volume %s (removable)",guid)
				continue
			if not self.volumes.has_volume(guid):
				self.context.begin_transaction()
				mp = self.volume_expert.get_mountpoint(guid)
				volume = Volume(guid=guid,mountpoint=mp,filesystem_type=vr.filesystem_type,
							label=vr.label,removable=vr.removable)
				self.log.debug("Registering volume %s",volume)
				self.volumes.store_volume(volume)
				self.context.commit_transaction()
			else:
				self.context.begin_transaction()
				volume = self.volumes.get_volume(guid)
				mp = self.volume_expert.get_mountpoint(guid)
				volume.update_mountpoint(mp)
				self.context.commit_transaction()

	def start_crawlers(self):
		for volume in self.volumes.itervalues():
			if volume.is_mounted():
				if self.volume_expert.is_appropriate(volume.get_mountpoint()):
					if volume.fully_indexed is False:
						self.log.debug("Starting crawler on %s",volume)
						self.crawler_manager.get_crawler(volume)
					else: self.log.debug("Skipping crawler on %s (fully indexed)",volume)
				else: self.log.debug("Skipping crawler on %s (inappropriate)",volume)
			else: self.log.debug("Skipping crawler on %s (unmounted)",volume)

	def kill_crawlers(self):
		self.crawler_manager.kill_all_crawlers()
		
	def run(self):
		if 0:
			import hotshot
			prof = hotshot.Profile(self.get_component_name()+'.prof')
			retval = prof.runctx('self._run()', globals(),locals())
			prof.close()
		else: self._run()

	def _run(self):
		self.context = self.index.acquire_context()
		self.context.set_catalog_infospace("filesystem")
		
		self.check_index_schema()
		self.register_currently_known_volumes()
		
		# store volumes FIXME should also update them
		
		self.start_crawlers()
		
		while not self.killed():
			import time
			times = time.time()
			try:
				# should we follow the "proper" or the dirty way?
				# we could make it so the crawlers send an (0,"volumename") tuple when they are done indexing files, and here, if type(filename[0]) is int, the volume is done, which addiontially means cuz of the semantics of this shit, that the volume can be considered fully crawled, and the next time there's no need to start a crawler for that volume.   tal vez haya mejor IPC methods que esto!  Perhaps we should send a CrawlFinishedEvent object (se me ocurre ahorita) con un puntero al crawler en cuestion.
				# we could also make FileDiscovered, FileDeleted, FileUpdated, VolumeMounted, VolumeUnmounted events
				# The crawler manager sends FileDiscovered and FilesystemFullyCrawled events
				event = self.consume(1)  # this one sends FileDiscovered and FilesystemFullyCrawled events
# 				self.log.debug("Lingered %s sec. before consuming",time.time() - times)
				# I think we should try to act upon volume notifications first, then file notifications and then crawler notifications, in that order, to preemptively act upon volume unmounts, and to keep the index fresh with recent files.
				# we should also extend this mechanism to the file_update_monitor and volume_monitor in the future.
				if event[0] == "FileFound":
					self.index_file(event[1],event[2])
				elif event[0] == "FullyIndexed":
					self.mark_volume_as_fully_indexed(event[1])
				else: raise Exception, "Unknown event %s" % event
# 				self.log.debug("Cycle completed in %s sec.",time.time()-times)
				self.sleep()
			except Empty:
				self.log.debug("No files to crawl, lingering")
			except:
				self.log.error("Anomalous exception while indexing, aborting filesystem infospace agent",exc_info=True)
				break
			
		
		self.kill_crawlers()
		self.context.release()
		del (self.context)
		
	def mark_volume_as_fully_indexed(self,volume):
		self.log.debug("Volume %s is now fully indexed",volume)
		self.context.begin_transaction()
		volume.fully_indexed = True
		self.context.commit_transaction()
		self.log.debug("Volume %s indexed? %s",volume,repr(volume.fully_indexed))

	def index_file(self,filename,volume):
		self.log.debug("Starting with %s", filename)
		thetime = time.time()
		key = strip_mountpoint(filename,volume.get_mountpoint())
		cat_key = "|".join([volume.guid,key])
		file = None
		try:
			if volume.has_file(key) and not volume.has_file_changed_since(key,os.stat(filename).st_mtime):
				self.log.debug("%s up-to-date, skipped in %s sec.", filename,time.time() - thetime)
				return
			file = self.file_plugin_manager.get_information_object(filename=filename,volume=volume)

		except IOError,e:
			if e.errno == 2:
				self.log.debug("Removing %s from database (IOError)",filename,exc_info=True)
				if volume.has_file(key):
					self.context.begin_transaction()
					self.context.uncatalog_object(cat_key)
					volume.remove_file(key)
					self.context.commit_transaction()
					self.log.debug("Removed %s from database (no longer exists) in %s sec.",filename,time.time() - thetime)
			elif e.errno == 13: self.log.debug("Permission to open %s denied ",filename)
			else: self.log.warning("Metadata extraction failed for %s",filename,exc_info=True)
 		except:
 			self.log.warning("Metadata extraction failed for %s",filename,exc_info=True)

		if not file:
			 self.log.debug("Skipping %s (decided in %s sec.)", filename,time.time() - thetime)
			 return # because all cases have been handled before this
		
		# if the file was actually gotten
		self.context.begin_transaction()
		try:
			volume.store_file(file)
			self.context.catalog_object(cat_key,file)
		except DatabaseError:
			self.log.warning("Cannot store %s in the catalog",file,exc_info=True)
			self.context.abort_transaction()
		except:
			self.context.abort_transaction()
			raise
		
		self.context.commit_transaction()
		self.log.debug("Indexed %s in %s sec." , file , time.time() - thetime)
	
	def search(self,expression,context):
		from pgtools.query import Expression, optimize
		from MetadataService.Index import IndexContext
		from sets import Set
		assert isinstance(expression,Expression)
		assert isinstance(context,IndexContext)
		
		thetime = time.time()
		self.log.debug("Asked to search for %s",expression)
		
		volumes = context.root()["infospaces"]["filesystem"].volumes
		
		expression = decompose_into_volumes(expression,volumes.values())
		
		self.log.debug("Final expression is %s (computed in %s sec.)",expression,time.time()-thetime)
		
		
		thetime = time.time()
		
		context.set_catalog_infospace("filesystem")
		results = context.search(expression)
		assert isinstance(results,Set)
		
		self.log.debug("Catalog returned %s results in %s sec.",len(results),time.time() -thetime)
		
		thetime = time.time()
		resultset = []
		
		from urllib import pathname2url
		mpcache = {}
		for guid,volume in volumes.iteritems():
			mountpoint = volume.get_mountpoint()
			if mountpoint: mpcache[guid] = pathname2url(mountpoint)
		
		
		for result in results:
		# convert each result to an URL, which is a string encoded in platform encoding
			vguid,fpath = result.split("|",1)
# 			fpath = pathname2url(fpath)
			if mpcache[vguid]: path = "file://" + os.path.join(mpcache[vguid],fpath)
			else: path = "volume://"+vguid+"/"+fpath
			if path.endswith("/") and path != "file:///": path = path[:-1]
			resultset.append(path)
		self.log.debug("Total number of results: %s (paths generated in %s sec.)",len(resultset),time.time() -thetime)
		
# 		if len(resultset) < 10:
# 			for a in resultset: print "Result:", a # just for debugging!
		
		return resultset


from pgtools.query import Aggregation,Criterion,Equals,IsBelow,IsOrIsBelow,Or
def decompose_into_volumes(expression,volumes):

	if isinstance(expression,Aggregation):
		newexpressions = []
		for e in expression.expressions:
			exp = decompose_into_volumes(e,volumes)
			if exp is not None: newexpressions.append(exp)
		if len(newexpressions): expression.expressions = newexpressions # replace with the new expressions
		else: raise ValueError, "Expression %s validated to an empty expression" # invalid query by the user
		return expression
	
	elif isinstance(expression,Criterion):
		if expression.attribute != "filename": return expression
		filename = os.path.normpath(expression.value)
		
		if isinstance(expression.operator,Equals):
			volume = deepest_volume_for(filename,volumes)

			if volume is None: raise ValueError,"%s is not an absolute path"%filename
			mountpoint = volume.get_mountpoint()
			if mountpoint is None: raise RuntimeError, "%s got unmounted in the middle of an operation"%volume
			
			relative_path = strip_mountpoint(filename,mountpoint)
			
			
			
			exp1 = Criterion()
			exp1.attribute = "absolute_path"
			exp1.operator = Equals()
			exp1.value = os.path.normpath("/%s/%s"%(volume.guid,relative_path))
			
			return exp1
			
		elif isinstance(expression.operator,IsBelow) or isinstance(expression.operator,IsOrIsBelow):
		
			involved_volumes = volumes_below(filename,volumes)
			if involved_volumes is None: raise ValueError,"%s is not an absolute path"%filename
			
			
			volume_exps = []
			
			shortest_mountpoint = None
			for iv in involved_volumes: 
				if shortest_mountpoint is None:
					shortest_mountpoint = iv.get_mountpoint()
					if shortest_mountpoint is None: raise RuntimeError, "%s got unmounted in the middle of an operation"%iv
				else:
					if len(iv.get_mountpoint()) < len(shortest_mountpoint): shortest_mountpoint = iv.get_mountpoint()
					
			for iv in involved_volumes:
				mountpoint = iv.get_mountpoint()
				if mountpoint is None: raise RuntimeError, "%s got unmounted in the middle of an operation"%iv
				relative_path = strip_mountpoint(filename,mountpoint)
				
				exp2 = Criterion()
				exp2.attribute = "absolute_path"
				if isinstance(expression.operator,IsBelow) and mountpoint != shortest_mountpoint:
					exp2.operator = IsOrIsBelow()
				else: exp2.operator = expression.operator
				exp2.value = os.path.normpath("/%s/%s"%(iv.guid,relative_path))
					
				volume_exps.append(exp2)
			
			return Or(volume_exps)
		
		else: raise NotImplementedError, "operator %s not implemented yet for filename queries"%expression.operator
		
	else: raise RuntimeError, "What the fuck?"


# consider moviing this piece of shit to the platform modules
def deepest_volume_for(filename,volumes):
	# this may return None at some point!!!!
	filename = os.path.normpath(filename)
	mounted_volumes = [ v for v in volumes if v.get_mountpoint() is not None ]
	matching_volumes = [ v for v in mounted_volumes if is_or_is_below(filename,v.get_mountpoint()) ]
	if len(matching_volumes) == 0: return None
	deepest_volume = matching_volumes[0]
	for v in matching_volumes:
		if len(v.get_mountpoint()) > len(deepest_volume.get_mountpoint()): deepest_volume = v
	return deepest_volume
	
def volumes_below(path,volumes):
	# this may return None at some point!!!!
	path = os.path.normpath(path)
	mounted_volumes = [ v for v in volumes if v.get_mountpoint() is not None ]
	matching_volumes = [ v for v in mounted_volumes if is_or_is_below(v.get_mountpoint(),path)  ]
	if len(matching_volumes) == 0:
		deepest_volume = deepest_volume_for(path,volumes)
		if deepest_volume is None: matching_volumes = None
		else: matching_volumes = [ deepest_volume_for(path,volumes) ]
	return matching_volumes
