BigArray.cs

Big Array Class Arrays Over 2GB Limit

Updated: Dec-14,2020

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading;
using Microsoft.VisualBasic.Devices;
[DebuggerDisplay("Count = {Count}")]
[Serializable]
public class BigArray<T> : MonitorActionFunc, IEnumerable<T>, IDisposable
{
    public const     int   ShiftCount  = 19;
    public const     int   Granularity = 1 << ShiftCount;
    private          bool  _disposed;
    private volatile T[][] _mdArray;
    private volatile Table _table = new Table();
    private volatile bool  _writing;
    public BigArray() : this(0)
    {
    }
    public BigArray(long size)
    {
        if (size < Granularity)
            size = Granularity;
        var i = 0;
        try
        {
            _table.MaximumNumberOfArrays = (int) (new ComputerInfo().AvailablePhysicalMemory / Granularity);
            _table.NumberOfActiveArrays  = (int) ((size + (Granularity - 1))                 / Granularity);
            var val = (long) _table.NumberOfActiveArrays                                     * Granularity;
            _table.Length = Interlocked.Read(ref val);
            var ms  = new MeasureSize<T>();
            var oas = ms.GetByteSize(Granularity);
            var am  = new ComputerInfo().AvailablePhysicalMemory;
            var rm  = (ulong) (oas * _table.NumberOfActiveArrays * Granularity);
            var maa = am / (ulong) (oas * Granularity);
            if (rm > am || (ulong) _table.NumberOfActiveArrays > maa)
                throw new Exception($"Requested memory {rm} exceeds available memory {am}");
            _mdArray = new T[maa][];
            for (i = 0; i < _table.NumberOfActiveArrays; ++i)
                _mdArray[i] = new T[Granularity];
            _writing = false;
        }
        catch (Exception ex)
        {
            throw new Exception($"Exception: {ex.Message}");
        }
    }
    public long Count
    {
        get
        {
            return Lock(_table.Count, () =>
            {
                return _table.Count;
            });
        }
    }
    public long Length
    {
        get
        {
            return Lock(_table.Length, () =>
            {
                return _table.Length;
            });
        }
    }
    public T this[long index]
    {
        get
        {
            while (_writing)
                new SpinWait().SpinOnce();
            if (index >= _table.Length)
                throw new Exception($"Getter: Index out of bounds, Index: '{index}' must be less than the Length: '{Length}'.");
            return _mdArray[index >> ShiftCount][index & (Granularity - 1)];
        }
        set
        {
            Lock(this, () =>
            {
                if (index + 1 > _table.Length)
                    ResizeArray();
                _writing                                                 = true;
                _mdArray[index >> ShiftCount][index & (Granularity - 1)] = value;
                _table.Count++;
                _writing = false;
            });
        }
    }
    public void Dispose()
    {
        Dispose(true);
    }
    IEnumerator<T> IEnumerable<T>.GetEnumerator()
    {
        return GetEnumerator();
    }
    IEnumerator IEnumerable.GetEnumerator()
    {
        return GetEnumerator();
    }
    public void Add(T Item)
    {
        Lock(this, () =>
        {
            if (_table.Count + 1 > _table.Length)
                ResizeArray();
            _writing = true;
            var x = _table.Count >> ShiftCount;
            var y = _table.Count & (Granularity - 1);
            _mdArray[x][y] = Item;
            _table.Count++;
            _writing = false;
        });
    }
    private void ResizeArray()
    {
        try
        {
            Interlocked.Increment(ref _table.NumberOfActiveArrays);
            var val = (long) _table.NumberOfActiveArrays * Granularity;
            _table.Length = Interlocked.Read(ref val);
            var ms  = new MeasureSize<T>();
            var oas = ms.GetByteSize(Granularity);
            var am  = new ComputerInfo().AvailablePhysicalMemory;
            var rm  = (ulong) (oas * _table.NumberOfActiveArrays * Granularity);
            var maa = am / (ulong) (oas * Granularity);
            if (rm > am || (ulong) _table.NumberOfActiveArrays > maa)
                throw new Exception($"Requested memory {rm} exceeds available memory {am}");
            _mdArray                                  = new T[maa][];
            _mdArray[_table.NumberOfActiveArrays - 1] = new T[Granularity];
        }
        catch (Exception ex)
        {
            throw new Exception($"Exception: {ex.Message}");
        }
    }
    public void Clear()
    {
        Lock(this, () =>
        {
            _writing = true;
            for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
                Array.Clear(_mdArray[a], 0, Granularity);
            _table.Count = 0;
            _writing     = false;
        });
    }
    public long IndexOf(T item)
    {
        return Lock(this, () =>
        {
            var i = 0L;
            for (; i < _table.NumberOfActiveArrays; i++)
            {
                while (_writing)
                    new SpinWait().SpinOnce();
                var pos = Array.IndexOf(_mdArray[i], item, 0);
                if (pos != -1)
                    return i * Granularity + pos;
            }
            return -1;
        });
    }
    public BigArray<T> Copy(long newsize)
    {
        return Lock(this, () =>
        {
            var temp = new BigArray<T>(newsize);
            for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
            {
                while (_writing)
                    new SpinWait().SpinOnce();
                Array.Copy(_mdArray[a], temp._mdArray[a], Granularity);
            }
            temp._table.Count = Count;
            return temp;
        });
    }
    public void FromArray(T[][] array)
    {
        Lock(this, () =>
        {
            _table.NumberOfActiveArrays = array.GetUpperBound(0) + 1;
            var val = (long) _table.NumberOfActiveArrays * Granularity;
            _table.Length = Interlocked.Read(ref val);
            _mdArray      = new T[_table.NumberOfActiveArrays][];
            for (var i = 0; i < _table.NumberOfActiveArrays; ++i)
                _mdArray[i] = new T[Granularity];
            for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
                Array.Copy(array[a], _mdArray[a], Granularity);
        });
    }
    public T[][] ToArray()
    {
        return Lock(this, () =>
        {
            var ta = new T[_table.NumberOfActiveArrays][];
            for (var i = 0; i < _table.NumberOfActiveArrays; ++i)
                ta[i] = new T[Granularity];
            for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
                Array.Copy(_mdArray[a], ta[a], Granularity);
            return ta;
        });
    }
    private void Dispose(bool disposing)
    {
        if (!_disposed)
            if (disposing)
                _mdArray = null;
        _disposed = true;
    }
    public IEnumerator<T> GetEnumerator()
    {
        return Lock(this, () =>
        {
            return GetEnum();
        });
    }
    public IEnumerator<T> GetEnum()
    {
        for (var i = 0; i < Count; i++)
            yield return this[i];
    }
    private class Table
    {
        public          long Count;
        public          long Length;
        public volatile int  MaximumNumberOfArrays;
        public volatile int  NumberOfActiveArrays;
    }
}

Leave a Reply

Your email address will not be published. Required fields are marked *