P4Connect.AssetStatusCache.cs #1

using UnityEditor;
using UnityEngine;
using System;
using System.Threading;
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Text;
using Perforce.P4;
using log4net;

namespace P4Connect
{
	/// <summary>
	/// Stores and periodically updates the Perforce status of all the assets
	/// </summary>
	public class AssetStatusCache
	{
        private static readonly ILog log = LogManager.GetLogger(typeof(AssetStatusCache));

		/// <summary>
		/// Simple asset status struct
		/// Stores the file state (checked out, modified, etc...) and revision state
		/// </summary>
		public struct AssetStatus
		{
			public DateTime TimeStamp;
			public FileState LocalState;
			public FileState OtherState;
			public DepotState DepotState;
			public RevisionState RevisionState;
			public ResolvedState ResolvedState;
			public StorageType StorageType;
			public LockState LockState;

            // Use this to create an AssetStatus for a local file (perforce would return null from the fstat call)
            public AssetStatus(string assetPath)
            {
                TimeStamp = default(DateTime);
                LocalState = FileState.None;
                OtherState = FileState.None;
                DepotState = DepotState.None;
                RevisionState = P4Connect.RevisionState.None;
                ResolvedState = P4Connect.ResolvedState.None;
                StorageType = StorageType.Text;
                LockState = LockState.None;
            }

            public override string ToString()
            {
                StringBuilder sb = new StringBuilder();
                if (TimeStamp == default(DateTime))
                {
                    sb.AppendFormat("Dummy AssetStatus");
                }
                else
                {
                    sb.AppendFormat("TimeStamp: {0} ", TimeStamp);
                    sb.AppendFormat("LocalState: {0} ", LocalState);
                    sb.AppendFormat("OtherState: {0} ", OtherState);
                    sb.AppendFormat("DepotState: {0} ", DepotState);
                    sb.AppendFormat("Revision: {0} ", RevisionState);
                    sb.AppendFormat("Resolved: {0} ", ResolvedState);
                    sb.AppendFormat("Storage: {0} ", StorageType);
                    sb.AppendFormat("Lockstate: {0} ", LockState);
                }
                return (sb.ToString());
            }

            public bool IsOnServer()
            {
                return (LocalState != FileState.None && LocalState != FileState.MarkedForAdd);
            }

            public bool IsLocked()
            {
                return (LockState == LockState.OurLock || LockState == LockState.TheirLock);
            }

            public bool IsOpened()
            {
                return (! (LocalState == FileState.None || LocalState == FileState.InDepot));
            }

            public bool IsInitialized()
            {
                return (this.TimeStamp != default(DateTime)); 
            }
		}

        /// <summary>
        /// This class is used to store the cached results for each "node" we keep track of in this project
        /// </summary>
        public class AssetEntry
        {
            public AssetStatus status;
            public FileMetaData meta;
            public DateTime update;

            public AssetEntry()
            {
                status = new AssetStatus();
                meta = new FileMetaData();
                status.TimeStamp = update = default(DateTime);
            }

            public bool Dirty { 
                get {return (update == default(DateTime)); }
                set { 
                    if (value == true){
                        status.TimeStamp = update = default(DateTime); 
                    } 
                }
            }
        }

		/// <summary>
		/// This delegate is triggered when the status of an asset changes
		/// </summary>
		public delegate void OnAssetStatusChangedDelegate(PerforceConnection aConnection);
		public static event OnAssetStatusChangedDelegate OnAssetStatusChanged
		{
			add
			{
				if (_OnAssetStatusChanged == null)
				{
					// We had no reason to get updated before, so do it now
					Engine.OnOperationPerformed += OnEngineOperationPerformed;
				}
				_OnAssetStatusChanged += value;
			}
			remove
			{
				_OnAssetStatusChanged -= value;
				if (_OnAssetStatusChanged == null)
				{
					// We no longer have any reason to be updated
					Engine.OnOperationPerformed -= OnEngineOperationPerformed;
				}
			}
		}

		// The internal event that is actually registered with
		static event OnAssetStatusChangedDelegate _OnAssetStatusChanged;

		// The caches
        static Dictionary<string, AssetEntry> _Entries;

	    private static MetaDataQueue _dataQueue;

        public static void Initialize()
		{
            _Entries = new Dictionary<string, AssetEntry>();
            PendingDirty = true;

            _dataQueue = new MetaDataQueue();
		}

		/// <summary>
		/// Generic method that fetches the cached AssetEntry which includes 
        /// AssetStatus (checked out, out of date, etc...)
        /// FileMetaData (server "fstat" result)
		/// </summary>
		public static AssetEntry GetAssetEntry(string aAssetPath)
		{
            AssetEntry entry;
            string path = GetEffectivePath(aAssetPath);
            if (! _Entries.TryGetValue(path, out entry))
            {
                //log.Debug("Creating DEFAULT: " + path);
                entry = new AssetEntry();
                _Entries[path] = entry;
            }
            
            return entry;
		}

        public static AssetStatus GetAssetStatus(string aAssetPath)
        {
            AssetEntry entry = GetAssetEntry(aAssetPath);
            return entry.status;
        }

        public static FileMetaData GetMetaData(string aAssetPath)
        {
            AssetEntry entry = GetAssetEntry(aAssetPath);
            return entry.meta;
        }

        public static FileMetaData GetMetaData(FileSpec spec)
        {
            string path = Utils.LocalPathToAssetPath(spec.LocalPath.Path);
            AssetEntry entry = GetAssetEntry(path);
            return entry.meta;
        }

        public static DateTime GetUpdated(string aAssetPath)
        {
            AssetEntry entry = GetAssetEntry(aAssetPath);
            return entry.update;
        }
 
        // A flag for pending changes to check if something has changedIgnore file name
        public static bool PendingDirty { get; set; }

		/// <summary>
        /// Called from the Icons code.
		/// Checks each Asset path for a valid status
		/// If not valid ask the server for an update.
		/// </summary>
        /// <param name="aAssetPaths">List of Asset Paths</param>
		public static void UpdateAssetData(List<string> aAssetPaths)
		{
            if (! Config.ValidConfiguration)
                return;

            // Remember the list
            HashSet<string> assets = new HashSet<string>(aAssetPaths);

            // Check each asset for a valid status
            foreach(string asset in aAssetPaths)
            {
                AssetStatus status = GetAssetStatus(asset);
                if (status.IsInitialized())
                {
                    assets.Remove(asset);
                }
            }
            if (assets.Any())
            {
                // Get any left-overs from the server  
                Refresh(assets);

               // Engine.PerformConnectionOperation(con =>
                //{
               //     UpdateAssetMetaData(con, GetFileSpecs(assets).ToList());
               // });
            }
		}

		/// <summary>
		/// Given the meta data, figures out the P4Connect icon representation for the file
		/// There is only a subset of all the metadata that P4Connect actually displays
		/// </summary>
		static AssetStatus CreateAssetStatus(FileMetaData aMeta)
		{
           // Debug.Log("CreateAssetStatus: " + Logger.FileMetaDataToString(aMeta));
			AssetStatus retData = new AssetStatus();

            if (aMeta == null)
                return retData;

			// Fill out the struct
			switch (aMeta.Type.BaseType)
			{
				case BaseFileType.Text:
					retData.StorageType = StorageType.Text;
					break;
				case BaseFileType.Binary:
					retData.StorageType = StorageType.Binary;
					break;
				default:
					retData.StorageType = StorageType.Other;
					break;
			}

            
			retData.LocalState = Engine.ParseFileAction(aMeta.Action);
            if (retData.LocalState == FileState.None)
            {
                // Check to see if Perforce knows this file so we can override the localState to InDepot.
                if (aMeta.IsMapped || aMeta.HeadAction != FileAction.None)
                    retData.LocalState = FileState.InDepot;
            }

			if (aMeta.OtherActions != null)
			{
				retData.OtherState = Engine.ParseFileAction(aMeta.OtherActions[0]);
			}
			if (aMeta.HaveRev >= aMeta.HeadRev)
				retData.RevisionState = RevisionState.HasLatest;
			else
				retData.RevisionState = RevisionState.OutOfDate;
			if (aMeta.Unresolved)
				retData.ResolvedState = ResolvedState.NeedsResolve;
			else
				retData.ResolvedState = ResolvedState.None;
			if (aMeta.HeadAction == FileAction.MoveDelete || aMeta.HeadAction == FileAction.Delete)
				retData.DepotState = DepotState.Deleted;
			else
				retData.DepotState = DepotState.None;
			retData.TimeStamp = DateTime.Now;

			retData.LockState = Engine.GetLockState(aMeta);

         //  Debug.Log("AssetStatus: " + retData.ToString());
			return retData;
		}

		/// <summary>
		/// This functions gets called when P4Connect does *something* to files
		/// In this case we update the files status
		/// </summary>
		static void OnEngineOperationPerformed(PerforceConnection aConnection, List<FileAndMeta> aFilesAndMetas)
		{
			if (aFilesAndMetas.Count > 0)
			{
				List<FileSpec> allSpecs = new List<FileSpec>();
				foreach (var fileAndMeta in aFilesAndMetas)
				{
					if (fileAndMeta.File != null)
					{
						allSpecs.Add(fileAndMeta.File);
					}
					if (fileAndMeta.Meta != null)
					{
						allSpecs.Add(fileAndMeta.Meta);
					}
				}
                //log.Debug("files changed: " + Logger.FileAndMetaListToString(aFilesAndMetas));

                // Only update entries with version numbers
                allSpecs = allSpecs.ValidSpecs().ToList();

                // But we can't pass the versions to fstat
                UpdateAssetMetaData(aConnection, allSpecs.UnversionedSpecs().ToList());    // Refresh the cache

				if (_OnAssetStatusChanged != null)
				{
					_OnAssetStatusChanged(aConnection);
				}
			}
		}

        /// <summary>
        /// Given a collection of folders and paths, refresh cache information
        /// </summary>
        /// <param name="paths"></param>
        public static void Refresh(IEnumerable<string> paths)
        {
            string[] paths1 = paths.AddDirectoryWildcards().ToArray();

            //log.Debug("Running async fstat: " + Logger.StringArrayToString(paths1));
            _dataQueue.QueueQuery(paths1);
        }

	 

        /// <summary>
        /// Given a folder, invalidate all cache data for members
        /// </summary>
        /// <param name="path">Path to dirty folder</param>
        public static void MarkFolderAsDirty(string path)
        {
            path = Utils.RemoveDirectoryWildcards(path);
            log.Debug("dirtypath: " + path);

             IEnumerable<string> fullKeys = _Entries.Keys.Where(currentKey => currentKey.StartsWith(path));

            foreach(string key in fullKeys)
            {
                _Entries[key].Dirty = true;
                log.Debug("dirt: " + key);
            }
        }

        /// <summary>
        /// Given an asset path, mark this asset as Dirty
        /// So it will be re-fstated later.
        /// </summary>
        /// <param name="assetPath"></param>
        public static void MarkAsDirty(string assetPath)
        {
            //log.Debug("DIRTY: " + assetPath);
            var entry = GetAssetEntry(assetPath);
            entry.update = default(DateTime);
            entry.status.TimeStamp = entry.update;
        }

        /// <summary>
        /// Mark the contents of a List of FileAndMetas as dirty
        /// </summary>
        /// <param name="fam">FileAndMetas Collection</param>
        static public void MarkAsDirty(IEnumerable<FileAndMeta> fam)
        {
            if (fam == null)
                return;

            foreach(var f in fam)
            {
                if (f.File != null)
                {
                    MarkAsDirty(f.File.ToAssetPath());
                }
                if (f.Meta != null)
                {
                    MarkAsDirty(f.Meta.ToAssetPath());
                }
            }
        }

        /// <summary>
        /// Using the Perforce connection, get a matching AssetStatus for each Assetpath
        /// </summary>
        /// <param name="aConnection">Perforce Connection</param>
        /// <param name="aFiles">A List of asset paths to retrieve</param>
        /// <param name="aOutStatuses">A list of statuses, one per aFile</param>
        public static void GetAssetStatusesFromPaths(PerforceConnection aConnection, List<string> aFiles, List<AssetStatus> aOutStatuses)
        {
            GetAssetStatuses(aConnection, GetEffectiveFileSpecs(aFiles).ToList(), aOutStatuses);
        }

        /// <summary>
        /// Using Perforce connection, get a matching AssetStatus for each Assetpath
        /// </summary>
        /// <param name="aConnection">Perforce Connection</param>
        /// <param name="aFiles">A List of asset paths to retrieve</param>
        /// <returns>A List of AssetStatus, one for each <paramref name="aFiles"/> entry</returns>
        static public List<AssetStatus> GetAssetStatusesFromPaths(PerforceConnection aConnection, List<string> aFiles)
        {
            List<AssetStatus> aOutStatuses = new List<AssetStatus>();
            GetAssetStatusesFromPaths(aConnection, aFiles, aOutStatuses);
            return aOutStatuses;
        }

        /// <summary>
        /// Using the Perforce connection, get a matching AssetStatus for each FileSpec
        /// If the metadata for a file is not in Perforce, we provide a "local" AssetStatus instead.
        /// </summary>
        /// <param name="aConnection">a valid Perforce Connection</param>
        /// <param name="aFiles"> A list of FileSpecs to be looked up </param>
        /// <param name="aOutStatuses">an AssetStatus is added to this list for each <paramref name="aFiles"/> entry</param>
		static void GetAssetStatuses(PerforceConnection aConnection, List<FileSpec> aFiles, List<AssetStatus> aOutStatuses)
		{
            IEnumerable<string> assetNames = GetAssetPathsFromFileSpecs(aFiles);
            string[] files = assetNames.ToArray();

            //Debug.Log("retrieving: " + Logger.StringArrayToString(files));

            List<string> notInitialized = new List<string>();

            // Collect results here for return later
            AssetStatus[] stats = new AssetStatus[files.Count()];

            // Get The Asset Status for each file,  if not initialized add to "not_initialized" list
            for (int i = 0; i < files.Count(); i++ )
            {
                stats[i] = GetAssetStatus(files[i]);
                if (! stats[i].IsInitialized())
                {
                    notInitialized.Add(files[i]);
                }  
            }
            
            // Now Ask for any uninitialized AssetStatus's from the server      
            if (notInitialized.Count > 0)
            {
                //log.Debug("Going to server for: " + Logger.StringArrayToString(notInitialized.ToArray()));
                Refresh(notInitialized);
                //Engine.PerformConnectionOperation(con =>
                //{
                //    UpdateAssetMetaData(aConnection, GetFileSpecs(notInitialized).ToList());
                //});
            }

            // Finally update the "holes" in the statuses originally returned
            for (int i = 0; i < files.Count(); i++)
            {
                if (!stats[i].IsInitialized())
                {
                    stats[i] = GetAssetStatus(files[i]);
                }
            }

            aOutStatuses.AddRange(stats);

            //log.Debug("assetstatuses: " + Logger.AssetStatusListToString(aOutStatuses));
		}

        /// <summary>
        /// Goes to the server, updates cache with latest about the files specified
        /// Frequently runs in a background thread.
        /// </summary>
        /// <param name="aConnection">A valid Perforce connection</param>
        /// <param name="aFiles">FileSpec List to retrieve info about</param>
        public static void UpdateAssetMetaData(PerforceConnection aConnection, List<FileSpec> aFiles)
        {
            if (aFiles != null && aFiles.Count > 0)
            {
                aFiles = aFiles.UnversionedSpecs().ToList();
                //log.Debug("update metadata: " + Logger.FileSpecListToString(aFiles));

                HashSet<string> original = new HashSet<string>(GetAssetPathsFromFileSpecs(aFiles));
                HashSet<string> checkedList = new HashSet<string>();

                DateTime startTimestamp = DateTime.Now;

                Options fsopts = new Options();
                fsopts["-Op"] = null;           // Return "real" clientpaths

                var mdlist = Engine.GetFileMetaData(aConnection, aFiles, fsopts);

                if (mdlist != null)
                {
                    // with a good response, we store the retrieved meta data in the AssetCache
                    StorePerforceMetaData(mdlist);

                    foreach (var md in mdlist)
                    {
                        checkedList.Add(GetEffectivePath(Utils.LocalPathToAssetPath(md.LocalPath.Path)));
                    }
                }

                // For "leftover" files not recognized by the server, we provide an updated AssetEntry 
                IEnumerable<string> leftOvers = original.Except(checkedList);
                StoreLocalMetaData(leftOvers);

                if (Config.DisplayP4Timings)
                {
                    double deltaInnerTime = (DateTime.Now - startTimestamp).TotalMilliseconds;
                    aConnection.AppendTimingInfo("UpdateAssetMetaData " + deltaInnerTime + " ms");
                }
                
            }
        }

        /// <summary>
        /// We have fresh metadata from the server
        /// Lets store it in the Asset Cache
        /// </summary>
        /// <param name="mdlist">List of Metadata to store</param>
        public static void StorePerforceMetaData(IList<FileMetaData> mdlist)
        {
            if (mdlist == null)
                return;

            foreach (var md in mdlist)
            {
                if (md == null || md.LocalPath == null || md.LocalPath.Path == null || md.LocalPath.Path.Length == 0)
                {
                    log.Debug("Missing LocalPath - MetaData: " + Logger.FileMetaDataToString(md));
                    continue;
                }
                
                // Create the AssetStatus and FileMetaData entries for each file known by the server
                string fname = GetEffectivePath(Utils.LocalPathToAssetPath(md.LocalPath.Path));
                AssetEntry entry = new AssetEntry
                {
                    meta = md,
                    status = CreateAssetStatus(md),
                    update = DateTime.Now
                };
                _Entries[fname] = entry;
                //log.Debug("AssetEntry: " + fname + " CREATED " + entry.status.ToStringNullSafe());
            }
        }

        /// <summary>
        /// The metadata from the server came back, and did not include these files
        /// So we need to make valid Cache entries as if they are Local files only
        /// </summary>
        /// <param name="files">Asset Paths of files to cache</param>
        public static void StoreLocalMetaData(IEnumerable<string> files)
        {
            if (files == null)
                return;

            // For "leftover" files not known by the server, we provide a default 
            // AssetEntry with the update field set.
            foreach(string f in files)
            {
                AssetEntry entry = new AssetEntry();
                entry.meta = new FileMetaData();
                entry.status = new AssetStatus();
                entry.status.TimeStamp = entry.update = DateTime.Now;
                _Entries[f] = entry;
                //log.Debug("AssetEntry: " + f + " CREATED Local " + entry.status.ToStringNullSafe());
            }
        }

		/// <summary>
		/// Returns a proper list of file spec given a list of paths, skipping folders, generating folder meta files instead
		/// </summary>
        /// <param name="aAssetPaths">Asset Path strings</param>
        /// <returns>FileSpecs</returns>
		static IEnumerable<FileSpec> GetEffectiveFileSpecs(IEnumerable<string> aAssetPaths)
		{
			foreach (var path in aAssetPaths)
			{
                yield return FileSpec.LocalSpec(Utils.AssetPathToLocalPath(GetEffectivePath(path)));
			}
            yield break;
		}

        /// <summary>
        /// Translates a collection of asset paths (including folders) into a collection of FileSpecs
        /// </summary>
        /// <param name="aAssetPaths">Asset paths to become FileSpec </param>
        /// <returns>FileSpecs</returns>
        public static IEnumerable<FileSpec> GetFileSpecs(IEnumerable<string> aAssetPaths)
		{
			foreach (var path in aAssetPaths)
			{
                yield return FileSpec.LocalSpec(Utils.AssetPathToLocalPath(path));
			}
            yield break;
		}

        /// <summary>
        /// Return a collection of Asset paths given a list of FileSpecs 
        /// (using the LocalPath of the FileSpec)
        /// </summary>
        /// <param name="fs">FileSpecs</param>
        /// <returns>strings</returns>
        static IEnumerable<string> GetAssetPathsFromFileSpecs(IEnumerable<FileSpec> fs)
        {
            foreach(var f in fs)
            {
                yield return Utils.LocalPathToAssetPath(f.LocalPath.Path);
            }
            yield break;
        }

        /// <summary>
        /// Return a collection of Asset paths given a list of FileMetaData
        /// </summary>
        /// <param name="fmd_list">FileMetaData collection</param>
        /// <returns>string collection </returns>
        static IEnumerable<string> GetAssetPathsFromFileMetaData(IEnumerable<FileMetaData>fmd_list)
        {
            foreach(var fm in fmd_list)
            {
                yield return Utils.LocalPathToAssetPath(fm.LocalPath.Path);
            }
            yield break;
        }

		/// <summary>
		/// Returns the path to use when printing the name, or looking up status info
		/// Directories get EffectivePaths because they are used to associate .meta files.
		/// </summary>
		/// <param name="aAssetPath">Asset Path</param>
		/// <returns>The effective path</returns>
		static string GetEffectivePath(string aAssetPath)
		{
			if (Utils.IsDirectory(aAssetPath))
			{
				return Utils.MetaFromAsset(Utils.RemoveDirectoryWildcards(aAssetPath));
			}
			else
			{
				return aAssetPath;
			}
		}
	}

    /// <summary>
    /// Encapsulates a thread with an input queue of string arrays.  
    /// Issues "p4 fstat" command for each entry
    /// </summary>
    class MetaDataQueue
    {
        private Thread _worker;
        readonly Queue<string[]> _queue = new Queue<string[]>();
        readonly object _locker = new object();
        private readonly EventWaitHandle _wh = new AutoResetEvent(false);

        public MetaDataQueue()
        {
           _worker = new Thread(Work);
           _worker.Start();
        }

        ~MetaDataQueue()
        {
            QueueQuery(null);
            _worker.Join();
            _wh.Close();
        }

        public void QueueQuery(string[] paths)
        {
            lock (_locker) _queue.Enqueue(paths);
            _wh.Set();
        }

        /// <summary>
        /// Work - This is run by the background thread.
        /// It issues a command on the queue
        /// Then waits for an Event to proceed.
        /// </summary>
        private void Work()
        {
            while (true)
            {
                string[] argument = null;
                lock (_locker)
                {
                    if (_queue.Count > 0)
                    {
                        argument = _queue.Dequeue();
                        if (argument == null)
                            return;
                    }
                }
                   
                if (argument != null)
                {
                    Engine.PerformConnectionOperation(con =>
                    {
                        AssetStatusCache.UpdateAssetMetaData(con, AssetStatusCache.GetFileSpecs(argument).ToList());
                    }, true);
                }
                else
                {
                    _wh.WaitOne();
                }
            }

        }

       
    }
}