Sha16512.cs

Variable 16Bit to 512Bit Hashing Algorithm

using System;
using System.Security.Cryptography;
[Serializable]
public class Sha16512 : HashAlgorithm
{
    private int           _bitWidth;
    private SHA512Managed _hash = new SHA512Managed();
    public Sha16512(int bitWidth)
    {
        if (bitWidth < 16 || bitWidth > 512)
            throw new ArgumentException($"Bit Width {bitWidth} must be between 16 and 512.");
        _bitWidth = bitWidth;
    }
    public override int HashSize => _bitWidth;
    public override void Initialize()
    {
    }
    protected override void HashCore(byte[] bytes, int ibStart, int cbSize)
    {
        var buf = bytes.SubByte(ibStart, cbSize);
        HashValue = _hash.ComputeHash(buf).SubByte(0, _bitWidth >> 3);
    }
    protected override byte[] HashFinal()
    {
        return (byte[]) HashValue.Clone();
    }
}

MonitorActionFuncWrapper.cs

Monitor Lock Wrapper

using System;
using System.Threading;
public class MonitorActionFuncWrapper : ConcurrencyCheck
{
    public void Lock(object localObject, Action action)
    {
        if (action == null)
            throw new Exception("Action argument cannot be null");
        if (CheckState())
        {
            var lockTaken = false;
            try
            {
                Monitor.Enter(localObject, ref lockTaken);
                action();
            }
            finally
            {
                if (lockTaken)
                    Monitor.Exit(localObject);
            }
        }
        else
        {
            action();
        }
    }
    public TResult Lock<TResult>(object localObject, Func<TResult> func)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
        {
            var lockTaken = false;
            try
            {
                Monitor.Enter(localObject, ref lockTaken);
                return func();
            }
            finally
            {
                if (lockTaken)
                    Monitor.Exit(localObject);
            }
        }
        return func();
    }
    public (TResult1, TResult2) Lock<TResult1, TResult2>(object localObject, Func<(TResult1, TResult2)> func)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
        {
            var lockTaken = false;
            try
            {
                Monitor.Enter(localObject, ref lockTaken);
                return func();
            }
            finally
            {
                if (lockTaken)
                    Monitor.Exit(localObject);
            }
        }
        return func();
    }
    public (TResult1, TResult2) Lock<TArg, TResult1, TResult2>(object localObject, Func<TArg, (TResult1, TResult2)> func, TArg arg)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
        {
            var lockTaken = false;
            try
            {
                Monitor.Enter(localObject, ref lockTaken);
                return func(arg);
            }
            finally
            {
                if (lockTaken)
                    Monitor.Exit(localObject);
            }
        }
        return func(arg);
    }
    public TResult Lock<TArg, TResult>(object localObject, Func<TArg, TResult> func, TArg arg)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
        {
            var lockTaken = false;
            try
            {
                Monitor.Enter(localObject, ref lockTaken);
                return func(arg);
            }
            finally
            {
                if (lockTaken)
                    Monitor.Exit(localObject);
            }
        }
        return func(arg);
    }
    public void Lock<TArg>(object localObject, Action<TArg> action, TArg arg)
    {
        if (action == null)
            throw new Exception("Action argument cannot be null");
        if (CheckState())
        {
            var lockTaken = false;
            try
            {
                Monitor.Enter(localObject, ref lockTaken);
                action(arg);
            }
            finally
            {
                if (lockTaken)
                    Monitor.Exit(localObject);
            }
        }
        else
        {
            action(arg);
        }
    }
}

ReaderWriterLockSlimActionFuncWrapper.cs

ReaderWriterLockSlim Wrapper See: MonitorActionFuncWrapper.cs

using System;
using System.Threading;
public class ReaderWriterLockSlimActionFuncWrapper : ConcurrencyCheck, IDisposable
{
    private bool                 _deadLock;
    private bool                 _disposed;
    private ReaderWriterLockSlim _rwl;
    public ReaderWriterLockSlimActionFuncWrapper(LockRecursionPolicy recursionPolicy = LockRecursionPolicy.NoRecursion)
    {
        _rwl = new ReaderWriterLockSlim(recursionPolicy);
        GC.SuppressFinalize(this);
    }
    public void Dispose()
    {
        Dispose(true);
    }
    public void Read(Action action)
    {
        if (action == null)
            throw new Exception("Action argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterReadLock();
                action();
            }
            finally
            {
                _rwl.ExitReadLock();
            }
        else
            action();
    }
    public TResult Read<TResult>(Func<TResult> func)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterReadLock();
                return func();
            }
            finally
            {
                _rwl.ExitReadLock();
            }
        return func();
    }
    public (TResult1, TResult2) Read<TResult1, TResult2>(Func<(TResult1, TResult2)> func)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterReadLock();
                return func();
            }
            finally
            {
                _rwl.ExitReadLock();
            }
        return func();
    }
    public (TResult1, TResult2) Read<TArg, TResult1, TResult2>(Func<TArg, (TResult1, TResult2)> func, TArg arg)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterReadLock();
                return func(arg);
            }
            finally
            {
                _rwl.ExitReadLock();
            }
        return func(arg);
    }
    public TResult Read<TArg, TResult>(Func<TArg, TResult> func, TArg arg)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterReadLock();
                return func(arg);
            }
            finally
            {
                _rwl.ExitReadLock();
            }
        return func(arg);
    }
    public void Read<TArg>(Action<TArg> action, TArg arg)
    {
        if (action == null)
            throw new Exception("Action argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterReadLock();
                action(arg);
            }
            finally
            {
                _rwl.ExitReadLock();
            }
        else
            action(arg);
    }
    public void Write(Action action)
    {
        if (action == null)
            throw new Exception("Action argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterWriteLock();
                action();
            }
            finally
            {
                _rwl.ExitWriteLock();
            }
        else
            action();
    }
    public TResult Write<TResult>(Func<TResult> func)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterWriteLock();
                return func();
            }
            finally
            {
                _rwl.ExitWriteLock();
            }
        return func();
    }
    public (TResult1, TResult2) Write<TResult1, TResult2>(Func<(TResult1, TResult2)> func)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterWriteLock();
                return func();
            }
            finally
            {
                _rwl.ExitWriteLock();
            }
        return func();
    }
    public (TResult1, TResult2) Write<TArg, TResult1, TResult2>(Func<TArg, (TResult1, TResult2)> func, TArg arg)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterWriteLock();
                return func(arg);
            }
            finally
            {
                _rwl.ExitWriteLock();
            }
        return func(arg);
    }
    public TResult Write<TArg, TResult>(Func<TArg, TResult> func, TArg arg)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterWriteLock();
                return func(arg);
            }
            finally
            {
                _rwl.ExitWriteLock();
            }
        return func(arg);
    }
    public void Write<TArg>(Action<TArg> action, TArg arg)
    {
        if (action == null)
            throw new Exception("Action argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterWriteLock();
                action(arg);
            }
            finally
            {
                _rwl.ExitWriteLock();
            }
        else
            action(arg);
    }
    public void ReadUpdate(Action action)
    {
        if (action == null)
            throw new Exception("Action argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterUpgradeableReadLock();
                try
                {
                    _rwl.EnterWriteLock();
                    action();
                }
                finally
                {
                    _rwl.ExitWriteLock();
                }
            }
            finally
            {
                _rwl.ExitUpgradeableReadLock();
            }
        else
            action();
    }
    public TResult ReadUpdate<TResult>(Func<TResult> func)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterUpgradeableReadLock();
                try
                {
                    _rwl.EnterWriteLock();
                    return func();
                }
                finally
                {
                    _rwl.ExitWriteLock();
                }
            }
            finally
            {
                _rwl.ExitUpgradeableReadLock();
            }
        return func();
    }
    public (TResult1, TResult2) ReadUpdate<TResult1, TResult2>(Func<(TResult1, TResult2)> func)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterUpgradeableReadLock();
                try
                {
                    _rwl.EnterWriteLock();
                    return func();
                }
                finally
                {
                    _rwl.ExitWriteLock();
                }
            }
            finally
            {
                _rwl.ExitUpgradeableReadLock();
            }
        return func();
    }
    public (TResult1, TResult2) ReadUpdate<TArg, TResult1, TResult2>(Func<TArg, (TResult1, TResult2)> func, TArg arg)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterUpgradeableReadLock();
                try
                {
                    _rwl.EnterWriteLock();
                    return func(arg);
                }
                finally
                {
                    _rwl.ExitWriteLock();
                }
            }
            finally
            {
                _rwl.ExitUpgradeableReadLock();
            }
        return func(arg);
    }
    public TResult ReadUpdate<TArg, TResult>(Func<TArg, TResult> func, TArg arg)
    {
        if (func == null)
            throw new Exception("Func argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterUpgradeableReadLock();
                try
                {
                    _rwl.EnterWriteLock();
                    return func(arg);
                }
                finally
                {
                    _rwl.ExitWriteLock();
                }
            }
            finally
            {
                _rwl.ExitUpgradeableReadLock();
            }
        return func(arg);
    }
    public void ReadUpdate<TArg>(Action<TArg> action, TArg arg)
    {
        if (action == null)
            throw new Exception("Action argument cannot be null");
        if (CheckState())
            try
            {
                _rwl.EnterUpgradeableReadLock();
                try
                {
                    _rwl.EnterWriteLock();
                    action(arg);
                }
                finally
                {
                    _rwl.ExitWriteLock();
                }
            }
            finally
            {
                _rwl.ExitUpgradeableReadLock();
            }
        else
            action(arg);
    }
    ~ReaderWriterLockSlimActionFuncWrapper()
    {
        if (_rwl != null)
            _rwl.Dispose();
    }
    protected virtual void Dispose(bool disposing)
    {
        if (_disposed)
            throw new Exception("Already Disposed");
        try
        {
            _rwl.Dispose();
            _disposed = true;
            _rwl      = null;
        }
        catch (SynchronizationLockException) when (disposing)
        {
            _deadLock = true;
        }
        finally
        {
            if (!_disposed && disposing)
                GC.ReRegisterForFinalize(this);
        }
    }
}

MapComparer.cs

Uses 64bit to 32bit hash Mapping for primitive, Value, and Reference types

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.Serialization.Formatters.Binary;
using System.Security;
[Serializable]
public class MapComparer<T> : IEqualityComparer<T>
{
    private static          Mapping64BitToHash32Bit _hash;
    private static          Type                    _type = typeof(T);
    private static readonly BinaryFormatter         _bf   = new BinaryFormatter();
    private static readonly MemoryStream            _ms   = new MemoryStream();
    private readonly        bool                    _isPrimitive;
    public MapComparer()
    {
        _isPrimitive = IsPrimitive();
        _hash        = new Mapping64BitToHash32Bit();
        _type        = typeof(T);
    }
    public bool Equals(T x, T y)
    {
        if (x == null || y == null)
            return false;
        if (_isPrimitive)
            return x.Equals(y);
        var xb = GetBytesObject(x, _type);
        var yb = GetBytesObject(y, _type);
        if (xb.Length != yb.Length)
            return false;
        return xb.Compare(yb);
    }
    public int GetHashCode(T obj)
    {
        var buf = GetBytesObject(obj, _type);
        return _hash.ComputeHash(buf).ToInt();
    }
    private static byte[] GetBytesObjectSerial(object value)
    {
        if (!_type.IsSerializable)
            return GetBytesObjectR(value);
        _ms.SetLength(0);
        _bf.Serialize(_ms, value);
        return _ms.ToArray().SubArray(0, (int) _ms.Length);
    }
    [SecurityCritical]
    private static byte[] GetBytesObjectR(object data)
    {
        var                    result  = new List<byte>();
        var                    type    = data.GetType();
        IEnumerable<FieldInfo> tFields = type.GetFields();
        foreach (var fieldInfo in tFields)
        {
            var value = fieldInfo.GetValue(data);
            var lt    = value.GetType();
            var buf   = GetBytesObject(value, lt);
            result.AddRange(buf);
        }
        var p = type.GetNestedTypes();
        if (p.Length > 0)
            foreach (var t in p)
            {
                var nFields = t.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static).ToArray();
                if (nFields.Length > 0)
                    if (!nFields[0].IsStatic)
                        return result.ToArray();
                foreach (var fieldInfo in nFields)
                {
                    var value = fieldInfo.GetValue(null);
                    var lt    = value.GetType();
                    var buf   = GetBytesObject(value, lt);
                    result.AddRange(buf);
                }
            }
        return result.ToArray();
    }
    [SecurityCritical]
    private static byte[] GetBytesObject(object obj, Type ltype)
    {
        switch (ltype.Name.Trim('[', ']'))
        {
            case "Byte":
                return !ltype.IsArray ? new[] {(byte) obj} : (byte[]) obj;
            case "Boolean":
                return !ltype.IsArray ? ((bool) obj).GetBytes() : ((bool[]) obj).GetBytes();
            case "SByte":
                return !ltype.IsArray ? ((sbyte) obj).GetBytes() : ((sbyte[]) obj).GetBytes();
            case "Char":
                return !ltype.IsArray ? ((char) obj).GetBytes() : ((char[]) obj).GetBytes();
            case "Int16":
                return !ltype.IsArray ? ((short) obj).GetBytes() : ((short[]) obj).GetBytes();
            case "UInt16":
                return !ltype.IsArray ? ((ushort) obj).GetBytes() : ((ushort[]) obj).GetBytes();
            case "Int32":
                return !ltype.IsArray ? ((int) obj).GetBytes() : ((int[]) obj).GetBytes();
            case "UInt32":
                return !ltype.IsArray ? ((uint) obj).GetBytes() : ((uint[]) obj).GetBytes();
            case "Int64":
                return !ltype.IsArray ? ((long) obj).GetBytes() : ((long[]) obj).GetBytes();
            case "UInt64":
                return !ltype.IsArray ? ((ulong) obj).GetBytes() : ((ulong[]) obj).GetBytes();
            case "Single":
                return !ltype.IsArray ? ((float) obj).GetBytes() : ((float[]) obj).GetBytes();
            case "Double":
                return !ltype.IsArray ? ((double) obj).GetBytes() : ((double[]) obj).GetBytes();
            case "String":
                return !ltype.IsArray ? ((string) obj).GetBytes() : ((string[]) obj).GetBytes();
            case "Decimal":
                return !ltype.IsArray ? ((decimal) obj).GetBytes() : ((decimal[]) obj).GetBytes();
            case "DateTime":
                return !ltype.IsArray ? ((DateTime) obj).GetBytes() : ((DateTime[]) obj).GetBytes();
        }
        return GetBytesObjectSerial(obj);
    }
    private bool IsPrimitive()
    {
        switch (Type.GetTypeCode(_type))
        {
            case TypeCode.Boolean:
            case TypeCode.Char:
            case TypeCode.SByte:
            case TypeCode.Byte:
            case TypeCode.Int16:
            case TypeCode.UInt16:
            case TypeCode.Int32:
            case TypeCode.UInt32:
            case TypeCode.Single:
            case TypeCode.String:
            case TypeCode.Decimal:
            case TypeCode.DateTime:
            case TypeCode.Int64:
            case TypeCode.UInt64:
            case TypeCode.Double:
                return true;
            default:
                return false;
        }
    }
}

Mapping256BitTo32BitHash.cs

Mapping 256Bit Hash to 32Bit Indexed Hash

using System.Security.Cryptography;
public class Mapping256BitTo32BitHash : HashAlgorithm
{
    private readonly SHA256Managed                  hasher = new SHA256Managed();
    public readonly  TinyDictionary<byte[], byte[]> map    = new TinyDictionary<byte[], byte[]>(101, new ArrayComparer());
    private          byte[]                         h160;
    public override  int                            HashSize => 32;
    public override void Initialize()
    {
    }
    protected override void HashCore(byte[] bytes, int ibStart, int cbSize)
    {
        h160 = hasher.ComputeHash(bytes, ibStart, cbSize);
        map.Add(h160, bytes);
    }
    protected override byte[] HashFinal()
    {
        HashValue = (byte[])map.FindKeyIndex(h160).GetBytes().Clone();
        return HashValue;
    }
}

BigHashsetSa.cs

Big Hash Set based onĀ BigArray.cs

Uses a single array.

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
[DebuggerDisplay("Count = {" + nameof(Count) + "}")]
[Serializable]
public class BigHashsetSa<T> : MonitorActionFunc, IEnumerable
{
    public enum Method
    {
        Grow,
        Compress
    }
    private volatile BigArray<Bucket>        _buckets;
    private          long                    _count;
    private          Method                  _method;
    internal         IBigEqualityComparer<T> Comparer;
    public BigHashsetSa(long size, Method method = Method.Grow) : this(size, new BigComparer<T>(), method)
    {
    }
    public BigHashsetSa(long size, IBigEqualityComparer<T> comparer, Method method = Method.Grow)
    {
        if (comparer == null)
            comparer = new BigComparer<T>();
        Comparer = comparer;
        _buckets = new BigArray<Bucket>(size);
        Count    = 0;
        _method  = method;
    }
    public long Count
    {
        get
        {
            return Lock(this, () =>
            {
                return _count;
            });
        }
        private set
        {
            Lock(this, () =>
            {
                _count = value;
            });
        }
    }
    public long                       ElementCount         => GetElementCount();
    public long                       NumberOfEmptyBuckets => GetNumberOfEmptyBuckets();
    public (long mDepth, long index)  MaximumBucketDepth   => GetMaximumBucketDepth();
    public float                      LoadRatio            => GetLoadRatio();
    public KeyValuePair<long, long>[] BucketDepthList      => GetBucketDepthList();
    IEnumerator IEnumerable.GetEnumerator()
    {
        return GetEnumerator();
    }
    public void Clear()
    {
        _buckets.Clear();
    }
    public bool Add(T item)
    {
        return Lock(this, () =>
        {
            if (_method == Method.Grow)
                EnsureSize();
            var hashCode = Comparer.GetHashCode(item) & long.MaxValue;
            if (FindEntry(item, hashCode).APos != -1)
                return false;
            var pos = hashCode % _buckets.Length;
            if (_buckets[pos] == null)
                _buckets[pos] = new Bucket();
            _buckets[pos].Add(item);
            Count++;
            return true;
        });
    }
    public T[] ToArray()
    {
        var newArray = new T[Count];
        using (var en = GetEnumerator())
        {
            var ptr = 0;
            while (en.MoveNext())
            {
                var value = en.Current;
                if (value == null)
                    break;
                newArray[ptr++] = value;
            }
            return newArray;
        }
    }
    private (long APos, long BPos) FindEntry(T item, long hashCode)
    {
        if (Count == 0)
            return (-1, -1);
        if (hashCode == 0)
        {
            var a = 0;
        }
        var aPos = hashCode % _buckets.Length;
        var bPos = 0;
        if (_buckets[aPos] == null)
        {
            _buckets[aPos] = new Bucket();
            return (-1, -1);
        }
        foreach (var i in _buckets[aPos].Values)
        {
            if (Comparer.Equals(i, item))
                return (aPos, bPos);
            bPos++;
        }
        return (-1, -1);
    }
    private void EnsureSize()
    {
        if (Count >= _buckets.Length)
        {
            var cArray = ToArray();
            _buckets = new BigArray<Bucket>(_buckets.Length + BigArray<T>.Granularity);
            foreach (var i in cArray)
            {
                var hashCode = Comparer.GetHashCode(i) & long.MaxValue;
                var pos      = hashCode % _buckets.Length;
                if (_buckets[pos] == null)
                    _buckets[pos] = new Bucket();
                _buckets[pos].Add(i);
            }
        }
    }
    public bool Contains(T item)
    {
        return Lock(this, () =>
        {
            var hashCode = Comparer.GetHashCode(item) & long.MaxValue;
            return FindEntry(item, hashCode).APos != -1;
        });
    }
    public IEnumerator<T> GetEnumerator()
    {
        return Lock(this, () =>
        {
            return GetEnum();
        });
    }
    public IEnumerator<T> GetEnum()
    {
        for (var i = 0; i < _buckets.Length; i++)
            if (_buckets[i] != null)
                for (var j = 0; j < _buckets[i].Count; ++j)
                    yield return _buckets[i].Values[j];
    }
    public long GetElementCount()
    {
        var count = 0;
        for (var i = 0; i < _buckets.Length; i++)
            if (_buckets[i] != null)
            {
                var c = _buckets[i].Count;
                count += c;
            }
        return count;
    }
    public long GetNumberOfEmptyBuckets()
    {
        var count = 0;
        for (var i = 0; i < _buckets.Length; i++)
            if (_buckets[i] == null)
                count++;
        return count;
    }
    public long GetNumberOfFilledBuckets()
    {
        var count = 0;
        for (var i = 0; i < _buckets.Length; i++)
            if (_buckets[i] != null)
                count++;
        return count;
    }
    public (long mDepth, long index) GetMaximumBucketDepth()
    {
        var max = 0;
        var j   = 0;
        for (var i = 0; i < _buckets.Length; i++)
            if (_buckets[i] != null)
            {
                var count = _buckets[i].Count;
                if (count > max)
                {
                    max = count;
                    j   = i;
                }
            }
        return (max, j);
    }
    public KeyValuePair<long, long>[] GetBucketDepthList()
    {
        var bdic = new Dictionary<long, long>();
        for (var i = 0; i < _buckets.Length; i++)
            if (_buckets[i] != null)
            {
                var count = _buckets[i].Count;
                if (!bdic.ContainsKey(count))
                {
                    bdic.Add(count, 0);
                    bdic[count]++;
                }
                else
                {
                    bdic[count]++;
                }
            }
        return bdic.OrderByDescending(x => x.Value).ToArray();
    }
    public float GetLoadRatio()
    {
        var x = Count;
        var y = _buckets.Length;
        var r = x / (float) y;
        return r;
    }
    internal class Bucket
    {
        public int Count;
        public T[] Values;
        public Bucket()
        {
            Values = new T[2];
            Count  = 0;
        }
        public void Add(T item)
        {
            if (Count >= Values.Length)
            {
                var ta = new T[Values.Length + 1];
                Array.Copy(Values, 0, ta, 0, Count);
                Values = ta;
            }
            Values[Count++] = item;
        }
    }
}

BigHashSet.cs

Big Hash Set based on BigArray.cs

Uses two arrays, for a single array use BigHashsetSa.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.Serialization;
using System.Threading;
[Serializable]
[DebuggerDisplay("Count = {Count}")]
[Obsolete]
public class BigHashSet<T> : MonitorActionFunc, IEnumerable<T>, ISerializable, IDeserializationCallback
{
    private readonly SerializationInfo sInfo;
    public volatile  Table             _table = new Table();
    public BigHashSet() : this(BigArray<T>.Granularity, new BigComparer<T>())
    {
    }
    public BigHashSet(long size) : this(size, new BigComparer<T>())
    {
    }
    public BigHashSet(IBigEqualityComparer<T> comparer) : this(BigArray<T>.Granularity, comparer)
    {
    }
    public BigHashSet(long size, IBigEqualityComparer<T> comparer)
    {
        Lock(this, () =>
        {
            if (size < BigArray<T>.Granularity)
                size = BigArray<T>.Granularity;
            if (comparer == null)
                comparer = new DynComparer64<T>();
            _table.Comparer                            = comparer;
            _table.HashBuckets                         = new BigArray<long>(size);
            _table.Slots                               = new BigArray<Slot>(size);
            _table._count                              = 0;
            _table.Position                            = -1;
            _table.HashBuckets.OverrideAutoConcurrency = true;
            _table.Slots.OverrideAutoConcurrency       = true;
        });
    }
    public BigHashSet(IEnumerable<T> collection)
    {
        Lock(this, () =>
        {
            var size = BigArray<T>.Granularity;
            _table.Comparer                            = new DynComparer64<T>();
            _table.HashBuckets                         = new BigArray<long>(size);
            _table.Slots                               = new BigArray<Slot>(size);
            _table._count                              = 0;
            _table.Position                            = -1;
            _table.HashBuckets.OverrideAutoConcurrency = true;
            _table.Slots.OverrideAutoConcurrency       = true;
            foreach (var item in collection)
                Insert(item, true);
        });
    }
    public BigHashSet(IEnumerable<T> collection, IBigEqualityComparer<T> comparer)
    {
        Lock(this, () =>
        {
            if (comparer == null)
                comparer = new DynComparer64<T>();
            var size = BigArray<T>.Granularity;
            _table.Comparer                            = comparer;
            _table.Comparer                            = new DynComparer64<T>();
            _table.HashBuckets                         = new BigArray<long>(size);
            _table.Slots                               = new BigArray<Slot>(size);
            _table._count                              = 0;
            _table.Position                            = -1;
            _table.HashBuckets.OverrideAutoConcurrency = true;
            _table.Slots.OverrideAutoConcurrency       = true;
            foreach (var item in collection)
                Insert(item, true);
        });
    }
    protected BigHashSet(SerializationInfo info, StreamingContext context)
    {
        sInfo = info;
    }
    public long Count
    {
        get
        {
            return Lock(this, () =>
            {
                return _table._count;
            });
        }
        private set
        {
            Lock(this, () =>
            {
                _table._count = value;
            });
        }
    }
    public T this[T item]
    {
        get
        {
            return Lock(this, () =>
            {
                var pos = FindEntry(item);
                if (pos == -1)
                    throw new Exception($"Getter: Index out of bounds {pos} must be contained within set.");
                return _table.Slots[pos]._value;
            });
        }
        set => Insert(item, true);
    }
    public T this[long index]
    {
        get
        {
            return Lock(this, () =>
            {
                if (index > _table._count || _table._count == 0)
                    SpinWait.SpinUntil(() => index < _table._count && _table._count > 0, 100);
                if (index > _table._count)
                    throw new Exception($"Getter: Index out of bounds {index} must be less than {_table._count}");
                return _table.Slots[index]._value;
            });
        }
    }
    public void OnDeserialization(object sender)
    {
        Lock(this, () =>
        {
            if (sInfo == null)
                return;
            var size = sInfo.GetInt64("Capacity");
            if (size != 0)
            {
                Clear();
                _table.HashBuckets = new BigArray<long>(size);
                _table.Slots       = new BigArray<Slot>(size);
                _table.Comparer    = (IBigEqualityComparer<T>) sInfo.GetValue("Comparer", typeof(IBigEqualityComparer<T>));
                _table._count      = sInfo.GetInt64("Count");
                _table.Position    = -1;
                var array = (Slot[][]) sInfo.GetValue("Elements", typeof(Slot[][]));
                if (array == null)
                    throw new SerializationException("Missing Elements.");
                var buckets = (long[][]) sInfo.GetValue("Buckets", typeof(long[][]));
                if (buckets == null)
                    throw new SerializationException("Missing Buckets.");
                _table.Slots.FromArray(array);
                _table.HashBuckets.FromArray(buckets);
            }
        });
    }
    IEnumerator<T> IEnumerable<T>.GetEnumerator()
    {
        return GetEnumerator();
    }
    IEnumerator IEnumerable.GetEnumerator()
    {
        return GetEnumerator();
    }
    public void GetObjectData(SerializationInfo info, StreamingContext context)
    {
        Lock(this, () =>
        {
            info.AddValue("Comparer", _table.Comparer, typeof(IBigEqualityComparer<T>));
            info.AddValue("Capacity", _table.HashBuckets.Length);
            info.AddValue("Count",    _table._count);
            var array = _table.Slots.ToArray();
            info.AddValue("Elements", array, typeof(BigArray<Slot>));
            var buck = _table.HashBuckets.ToArray();
            info.AddValue("Buckets", buck, typeof(BigArray<long>));
        });
    }
    public void Clear()
    {
        Lock(this, () =>
        {
            var size = BigArray<T>.Granularity;
            _table.HashBuckets = new BigArray<long>(size);
            _table.Slots       = new BigArray<Slot>(size);
            _table._count      = 0;
            _table.Position    = -1;
        });
    }
    public bool Add(T item)
    {
        return Insert(item, true);
    }
    public void AddRange(IEnumerable<T> collection)
    {
        Lock(this, () =>
        {
            foreach (var item in collection)
                Insert(item, true);
        });
    }
    public bool Contains(T item)
    {
        return Insert(item, false);
    }
    private long InternalGetHashCode(T item)
    {
        if (item == null)
            return 0;
        return _table.Comparer.GetHashCode(item) & long.MaxValue;
    }
    internal bool Insert(T item, bool add)
    {
        return Lock(this, () =>
        {
            var hashCode = InternalGetHashCode(item);
            if (FindEntry(item, hashCode) != -1)
                return true;
            _table.Position = -1;
            if (add)
            {
                if (_table._count >= _table.Slots.Length)
                {
                    var newSize        = _table.HashBuckets.Length << 1;
                    var newHashBuckets = new BigArray<long>(newSize);
                    var newSlots       = new BigArray<Slot>(newSize);
                    for (var i = 0L; i < _table._count; i++)
                    {
                        var pos = _table.Slots[i]._hashCode % newSize;
                        newSlots[i]         = new Slot(_table.Slots[i]._hashCode, newHashBuckets[pos] - 1, _table.Slots[i]._value);
                        newHashBuckets[pos] = i + 1;
                    }
                    _table.HashBuckets = newHashBuckets;
                    _table.Slots       = newSlots;
                }
                var hashPos = hashCode % _table.HashBuckets.Length;
                var news    = new Slot(hashCode, _table.HashBuckets[hashPos] - 1, item);
                _table.Slots[_table._count] = news;
                _table.HashBuckets[hashPos] = _table._count + 1;
                _table.Position             = _table._count;
                _table._count++;
            }
            return false;
        });
    }
    private long FindEntry(T item, long hashCode)
    {
        return Lock(this, () =>
        {
            for (var position = _table.HashBuckets[hashCode % _table.HashBuckets.Length] - 1; position >= 0; position = _table.Slots[position]._next)
                if (_table.Slots[position]._hashCode == hashCode && Equals(_table.Slots[position]._value, item))
                {
                    _table.Position = position;
                    return _table.Position;
                }
            return -1;
        });
    }
    public T[] ToArray()
    {
        return Lock(this, () =>
        {
            var array = new T[Count];
            for (var i = 0L; i < Count; i++)
                if (_table.Slots[i]._hashCode >= 0)
                    array[i] = _table.Slots[i]._value;
            return array;
        });
    }
    public long FindEntry(T item)
    {
        return FindEntry(item, InternalGetHashCode(item));
    }
    public bool Remove(T item)
    {
        return Lock(this, () =>
        {
            var hashCode = _table.Comparer.GetHashCode(item) & long.MaxValue;
            for (var position = _table.HashBuckets[hashCode % _table.HashBuckets.Length] - 1; position >= 0; position = _table.Slots[position]._next)
                if (_table.Slots[position]._hashCode == hashCode && Equals(_table.Slots[position]._value, item))
                {
                    var hashPos = hashCode % _table.HashBuckets.Length;
                    var news    = new Slot(0, -1, default);
                    _table.Slots[position]      = news;
                    _table.HashBuckets[hashPos] = -1;
                    return true;
                }
            return false;
        });
    }
    public IEnumerator<T> GetEnumerator()
    {
        return Lock(this, () =>
        {
            return GetEnum();
        });
    }
    public IEnumerator<T> GetEnum()
    {
        for (var i = 0; i < Count; i++)
            if (_table.Slots[i]._hashCode >= 0)
                yield return _table.Slots[i]._value;
    }
    public class Table
    {
        public            long                    _count;
        internal          IBigEqualityComparer<T> Comparer;
        internal volatile BigArray<long>          HashBuckets;
        public            long                    Position;
        internal volatile BigArray<Slot>          Slots;
    }
    [Serializable]
    internal struct Slot
    {
        public readonly long _hashCode;
        public readonly long _next;
        public readonly T    _value;
        public Slot(long hashcode, long next, T value)
        {
            _hashCode = hashcode;
            _next     = next;
            _value    = value;
        }
    }
}

BigComparer.cs

Big Comparer Class

Includes: IBigEqualityComparer.cs

using System;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
[Serializable]
public class BigComparer<T> : IBigEqualityComparer<T>
{
    private static   Type            _type = typeof(T);
    private readonly BinaryFormatter _bf   = new BinaryFormatter();
    private readonly bool            _isPrimitive;
    private readonly MemoryStream    _ms = new MemoryStream();
    private          bool            _disposed;
    public BigComparer()
    {
        _isPrimitive = IsPrimitive();
    }
    public bool Equals(T x, T y)
    {
        if (x == null || y == null)
            return false;
        if (_isPrimitive)
            return x.Equals(y);
        if (_type == null)
            _type = x.GetType();
        if (_type.IsArray)
        {
            if (!_isPrimitive)
            {
                var xb = GetBytesObjectSerial(x);
                var yb = GetBytesObjectSerial(y);
                return xb.Compare(yb);
            }
            else
            {
                var xb = GetBytesObject(x);
                var yb = GetBytesObject(y);
                return xb.Compare(yb);
            }
        }
        return x.Equals(y);
    }
    public long GetHashCode(T obj)
    {
        if (_isPrimitive)
            return obj.GetHashCode();
        if (_type == null)
            _type = obj.GetType();
        var    result = 0xCBF29CE484222325;
        byte[] buf;
        buf = !_isPrimitive ? GetBytesObjectSerial(obj) : GetBytesObject(obj);
        foreach (var b in buf)
            result = (result ^ b) * 0x100000001B3;
        return (long) result;
    }
    private void Dispose(bool disposing)
    {
        if (!_disposed)
            if (disposing)
                _ms?.Close();
        _disposed = true;
    }
    public byte[] GetBytesObjectSerial(object value)
    {
        if (_type == null)
            _type = value.GetType();
        if (!_type.IsSerializable)
            throw new Exception("Error: Object is not Serializable.");
        _ms.SetLength(0);
        _bf.Serialize(_ms, value);
        return _ms.ToArray().SubArray(0, (int) _ms.Length);
    }
    public bool IsPrimitive()
    {
        if (_type == null)
            _type = typeof(T);
        switch (Type.GetTypeCode(_type))
        {
            case TypeCode.Boolean:
            case TypeCode.Char:
            case TypeCode.SByte:
            case TypeCode.Byte:
            case TypeCode.Int16:
            case TypeCode.UInt16:
            case TypeCode.Int32:
            case TypeCode.UInt32:
            case TypeCode.Single:
            case TypeCode.String:
            case TypeCode.Decimal:
            case TypeCode.DateTime:
            case TypeCode.Int64:
            case TypeCode.UInt64:
            case TypeCode.Double:
                return true;
            default:
                return false;
        }
    }
    public byte[] GetBytesObject(object obj)
    {
        if (_type == null)
            _type = obj.GetType();
        switch (_type.Name.Trim('[', ']'))
        {
            case "Byte":
                return !_type.IsArray ? new[] {(byte) obj} : (byte[]) obj;
            case "Boolean":
                return !_type.IsArray ? ((bool) obj).GetBytes() : ((bool[]) obj).GetBytes();
            case "SByte":
                return !_type.IsArray ? ((sbyte) obj).GetBytes() : ((sbyte[]) obj).GetBytes();
            case "Char":
                return !_type.IsArray ? ((char) obj).GetBytes() : ((char[]) obj).GetBytes();
            case "Int16":
                return !_type.IsArray ? ((short) obj).GetBytes() : ((short[]) obj).GetBytes();
            case "UInt16":
                return !_type.IsArray ? ((ushort) obj).GetBytes() : ((ushort[]) obj).GetBytes();
            case "Int32":
                return !_type.IsArray ? ((int) obj).GetBytes() : ((int[]) obj).GetBytes();
            case "UInt32":
                return !_type.IsArray ? ((uint) obj).GetBytes() : ((uint[]) obj).GetBytes();
            case "Int64":
                return !_type.IsArray ? ((long) obj).GetBytes() : ((long[]) obj).GetBytes();
            case "UInt64":
                return !_type.IsArray ? ((ulong) obj).GetBytes() : ((ulong[]) obj).GetBytes();
            case "Single":
                return !_type.IsArray ? ((float) obj).GetBytes() : ((float[]) obj).GetBytes();
            case "Double":
                return !_type.IsArray ? ((double) obj).GetBytes() : ((double[]) obj).GetBytes();
            case "String":
                return !_type.IsArray ? ((string) obj).GetBytes() : ((string[]) obj).GetBytes();
            case "Decimal":
                return !_type.IsArray ? ((decimal) obj).GetBytes() : ((decimal[]) obj).GetBytes();
            case "DateTime":
                return !_type.IsArray ? ((DateTime) obj).GetBytes() : ((DateTime[]) obj).GetBytes();
        }
        return GetBytesObjectSerial(obj);
    }
}

public interface IBigEqualityComparer<in T>
{
    bool Equals(T      x, T y);
    long GetHashCode(T obj);
}

public interface IBigEqualityComparer32<in T>
{
    bool Equals(T      x, T y);
    int GetHashCode(T obj);
}

BigArray.cs

Big Array Class Arrays Over 2GB Limit

Updated: Dec-14,2020

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading;
using Microsoft.VisualBasic.Devices;
[DebuggerDisplay("Count = {Count}")]
[Serializable]
public class BigArray<T> : MonitorActionFunc, IEnumerable<T>, IDisposable
{
    public const     int   ShiftCount  = 19;
    public const     int   Granularity = 1 << ShiftCount;
    private          bool  _disposed;
    private volatile T[][] _mdArray;
    private volatile Table _table = new Table();
    private volatile bool  _writing;
    public BigArray() : this(0)
    {
    }
    public BigArray(long size)
    {
        if (size < Granularity)
            size = Granularity;
        var i = 0;
        try
        {
            _table.MaximumNumberOfArrays = (int) (new ComputerInfo().AvailablePhysicalMemory / Granularity);
            _table.NumberOfActiveArrays  = (int) ((size + (Granularity - 1))                 / Granularity);
            var val = (long) _table.NumberOfActiveArrays                                     * Granularity;
            _table.Length = Interlocked.Read(ref val);
            var ms  = new MeasureSize<T>();
            var oas = ms.GetByteSize(Granularity);
            var am  = new ComputerInfo().AvailablePhysicalMemory;
            var rm  = (ulong) (oas * _table.NumberOfActiveArrays * Granularity);
            var maa = am / (ulong) (oas * Granularity);
            if (rm > am || (ulong) _table.NumberOfActiveArrays > maa)
                throw new Exception($"Requested memory {rm} exceeds available memory {am}");
            _mdArray = new T[maa][];
            for (i = 0; i < _table.NumberOfActiveArrays; ++i)
                _mdArray[i] = new T[Granularity];
            _writing = false;
        }
        catch (Exception ex)
        {
            throw new Exception($"Exception: {ex.Message}");
        }
    }
    public long Count
    {
        get
        {
            return Lock(_table.Count, () =>
            {
                return _table.Count;
            });
        }
    }
    public long Length
    {
        get
        {
            return Lock(_table.Length, () =>
            {
                return _table.Length;
            });
        }
    }
    public T this[long index]
    {
        get
        {
            while (_writing)
                new SpinWait().SpinOnce();
            if (index >= _table.Length)
                throw new Exception($"Getter: Index out of bounds, Index: '{index}' must be less than the Length: '{Length}'.");
            return _mdArray[index >> ShiftCount][index & (Granularity - 1)];
        }
        set
        {
            Lock(this, () =>
            {
                if (index + 1 > _table.Length)
                    ResizeArray();
                _writing                                                 = true;
                _mdArray[index >> ShiftCount][index & (Granularity - 1)] = value;
                _table.Count++;
                _writing = false;
            });
        }
    }
    public void Dispose()
    {
        Dispose(true);
    }
    IEnumerator<T> IEnumerable<T>.GetEnumerator()
    {
        return GetEnumerator();
    }
    IEnumerator IEnumerable.GetEnumerator()
    {
        return GetEnumerator();
    }
    public void Add(T Item)
    {
        Lock(this, () =>
        {
            if (_table.Count + 1 > _table.Length)
                ResizeArray();
            _writing = true;
            var x = _table.Count >> ShiftCount;
            var y = _table.Count & (Granularity - 1);
            _mdArray[x][y] = Item;
            _table.Count++;
            _writing = false;
        });
    }
    private void ResizeArray()
    {
        try
        {
            Interlocked.Increment(ref _table.NumberOfActiveArrays);
            var val = (long) _table.NumberOfActiveArrays * Granularity;
            _table.Length = Interlocked.Read(ref val);
            var ms  = new MeasureSize<T>();
            var oas = ms.GetByteSize(Granularity);
            var am  = new ComputerInfo().AvailablePhysicalMemory;
            var rm  = (ulong) (oas * _table.NumberOfActiveArrays * Granularity);
            var maa = am / (ulong) (oas * Granularity);
            if (rm > am || (ulong) _table.NumberOfActiveArrays > maa)
                throw new Exception($"Requested memory {rm} exceeds available memory {am}");
            _mdArray                                  = new T[maa][];
            _mdArray[_table.NumberOfActiveArrays - 1] = new T[Granularity];
        }
        catch (Exception ex)
        {
            throw new Exception($"Exception: {ex.Message}");
        }
    }
    public void Clear()
    {
        Lock(this, () =>
        {
            _writing = true;
            for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
                Array.Clear(_mdArray[a], 0, Granularity);
            _table.Count = 0;
            _writing     = false;
        });
    }
    public long IndexOf(T item)
    {
        return Lock(this, () =>
        {
            var i = 0L;
            for (; i < _table.NumberOfActiveArrays; i++)
            {
                while (_writing)
                    new SpinWait().SpinOnce();
                var pos = Array.IndexOf(_mdArray[i], item, 0);
                if (pos != -1)
                    return i * Granularity + pos;
            }
            return -1;
        });
    }
    public BigArray<T> Copy(long newsize)
    {
        return Lock(this, () =>
        {
            var temp = new BigArray<T>(newsize);
            for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
            {
                while (_writing)
                    new SpinWait().SpinOnce();
                Array.Copy(_mdArray[a], temp._mdArray[a], Granularity);
            }
            temp._table.Count = Count;
            return temp;
        });
    }
    public void FromArray(T[][] array)
    {
        Lock(this, () =>
        {
            _table.NumberOfActiveArrays = array.GetUpperBound(0) + 1;
            var val = (long) _table.NumberOfActiveArrays * Granularity;
            _table.Length = Interlocked.Read(ref val);
            _mdArray      = new T[_table.NumberOfActiveArrays][];
            for (var i = 0; i < _table.NumberOfActiveArrays; ++i)
                _mdArray[i] = new T[Granularity];
            for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
                Array.Copy(array[a], _mdArray[a], Granularity);
        });
    }
    public T[][] ToArray()
    {
        return Lock(this, () =>
        {
            var ta = new T[_table.NumberOfActiveArrays][];
            for (var i = 0; i < _table.NumberOfActiveArrays; ++i)
                ta[i] = new T[Granularity];
            for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
                Array.Copy(_mdArray[a], ta[a], Granularity);
            return ta;
        });
    }
    private void Dispose(bool disposing)
    {
        if (!_disposed)
            if (disposing)
                _mdArray = null;
        _disposed = true;
    }
    public IEnumerator<T> GetEnumerator()
    {
        return Lock(this, () =>
        {
            return GetEnum();
        });
    }
    public IEnumerator<T> GetEnum()
    {
        for (var i = 0; i < Count; i++)
            yield return this[i];
    }
    private class Table
    {
        public          long Count;
        public          long Length;
        public volatile int  MaximumNumberOfArrays;
        public volatile int  NumberOfActiveArrays;
    }
}

SecureStringHelper.cs

Secure String Helper Class

public class SecureStringHelper : IDisposable
    {
        private readonly Encoding     _encoding;
        private readonly SecureString _secureString;
        private          byte[]       _bytes;
        private          bool         _disposed;
        public SecureStringHelper(SecureString secureString) : this(secureString, Encoding.Default)
        {
        }
        public SecureStringHelper(SecureString secureString, Encoding encoding)
        {
            if(secureString == null)
                throw new Exception(nameof(secureString));
            _encoding     = encoding ?? Encoding.Default;
            _secureString = secureString;
        }
        public void Dispose()
        {
            if(!_disposed)
            {
                Destroy();
                _disposed = true;
            }
            GC.SuppressFinalize(this);
        }
        public static string ToString(SecureString secureStr)
        {
            var Str = new NetworkCredential(string.Empty, secureStr).Password;
            return Str;
        }
        public static SecureString ToSecureString(string password)
        {
            if(password == null)
                throw new Exception("Error: Password cannot be null.");
            var securePassword = new SecureString();
            foreach(var c in password)
                securePassword.AppendChar(c);
            securePassword.MakeReadOnly();
            return securePassword;
        }
        internal unsafe byte[] ToByteArray()
        {
            if(_bytes != null)
            {
                _bytes.Fill(0);
                _bytes = null;
            }
            var maxLength = _encoding.GetMaxByteCount(_secureString.Length);
            var bytes     = IntPtr.Zero;
            var str       = IntPtr.Zero;
            try
            {
                bytes = Marshal.AllocHGlobal(maxLength);
                str   = Marshal.SecureStringToBSTR(_secureString);
                var chars = (char*) str.ToPointer();
                var bptr  = (byte*) bytes.ToPointer();
                var len   = _encoding.GetBytes(chars, _secureString.Length, bptr, maxLength);
                _bytes = new byte[len];
                for(var i = 0; i < len; ++i)
                {
                    _bytes[i] = *bptr;
                    bptr++;
                }
                return _bytes;
            }
            finally
            {
                if(bytes != IntPtr.Zero)
                    Marshal.FreeHGlobal(bytes);
                if(str != IntPtr.Zero)
                    Marshal.ZeroFreeBSTR(str);
            }
        }
        private void Destroy()
        {
            if(_bytes != null)
            {
                _bytes.Fill(0);
                _bytes = null;
            }
        }
    }

public static class SecureStringEx
    {
        /// <summary>
        ///     Converts a standard string to a secure string
        /// </summary>
        public static SecureString ToSecureString(this string password)
        {
            if(password == null)
                throw new Exception("Error: Password cannot be null.");
            var securePassword = new SecureString();
            foreach(var c in password)
                securePassword.AppendChar(c);
            securePassword.MakeReadOnly();
            return securePassword;
        }
    }