Big Array Class Arrays Over 2GB Limit
Updated: Dec-14,2020
using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Threading; using Microsoft.VisualBasic.Devices; [DebuggerDisplay("Count = {Count}")] [Serializable] public class BigArray<T> : MonitorActionFunc, IEnumerable<T>, IDisposable { public const int ShiftCount = 19; public const int Granularity = 1 << ShiftCount; private bool _disposed; private volatile T[][] _mdArray; private volatile Table _table = new Table(); private volatile bool _writing; public BigArray() : this(0) { } public BigArray(long size) { if (size < Granularity) size = Granularity; var i = 0; try { _table.MaximumNumberOfArrays = (int) (new ComputerInfo().AvailablePhysicalMemory / Granularity); _table.NumberOfActiveArrays = (int) ((size + (Granularity - 1)) / Granularity); var val = (long) _table.NumberOfActiveArrays * Granularity; _table.Length = Interlocked.Read(ref val); var ms = new MeasureSize<T>(); var oas = ms.GetByteSize(Granularity); var am = new ComputerInfo().AvailablePhysicalMemory; var rm = (ulong) (oas * _table.NumberOfActiveArrays * Granularity); var maa = am / (ulong) (oas * Granularity); if (rm > am || (ulong) _table.NumberOfActiveArrays > maa) throw new Exception($"Requested memory {rm} exceeds available memory {am}"); _mdArray = new T[maa][]; for (i = 0; i < _table.NumberOfActiveArrays; ++i) _mdArray[i] = new T[Granularity]; _writing = false; } catch (Exception ex) { throw new Exception($"Exception: {ex.Message}"); } } public long Count { get { return Lock(_table.Count, () => { return _table.Count; }); } } public long Length { get { return Lock(_table.Length, () => { return _table.Length; }); } } public T this[long index] { get { while (_writing) new SpinWait().SpinOnce(); if (index >= _table.Length) throw new Exception($"Getter: Index out of bounds, Index: '{index}' must be less than the Length: '{Length}'."); return _mdArray[index >> ShiftCount][index & (Granularity - 1)]; } set { Lock(this, () => { if (index + 1 > _table.Length) ResizeArray(); _writing = true; _mdArray[index >> ShiftCount][index & (Granularity - 1)] = value; _table.Count++; _writing = false; }); } } public void Dispose() { Dispose(true); } IEnumerator<T> IEnumerable<T>.GetEnumerator() { return GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } public void Add(T Item) { Lock(this, () => { if (_table.Count + 1 > _table.Length) ResizeArray(); _writing = true; var x = _table.Count >> ShiftCount; var y = _table.Count & (Granularity - 1); _mdArray[x][y] = Item; _table.Count++; _writing = false; }); } private void ResizeArray() { try { Interlocked.Increment(ref _table.NumberOfActiveArrays); var val = (long) _table.NumberOfActiveArrays * Granularity; _table.Length = Interlocked.Read(ref val); var ms = new MeasureSize<T>(); var oas = ms.GetByteSize(Granularity); var am = new ComputerInfo().AvailablePhysicalMemory; var rm = (ulong) (oas * _table.NumberOfActiveArrays * Granularity); var maa = am / (ulong) (oas * Granularity); if (rm > am || (ulong) _table.NumberOfActiveArrays > maa) throw new Exception($"Requested memory {rm} exceeds available memory {am}"); _mdArray = new T[maa][]; _mdArray[_table.NumberOfActiveArrays - 1] = new T[Granularity]; } catch (Exception ex) { throw new Exception($"Exception: {ex.Message}"); } } public void Clear() { Lock(this, () => { _writing = true; for (var a = 0L; a < _table.NumberOfActiveArrays; a++) Array.Clear(_mdArray[a], 0, Granularity); _table.Count = 0; _writing = false; }); } public long IndexOf(T item) { return Lock(this, () => { var i = 0L; for (; i < _table.NumberOfActiveArrays; i++) { while (_writing) new SpinWait().SpinOnce(); var pos = Array.IndexOf(_mdArray[i], item, 0); if (pos != -1) return i * Granularity + pos; } return -1; }); } public BigArray<T> Copy(long newsize) { return Lock(this, () => { var temp = new BigArray<T>(newsize); for (var a = 0L; a < _table.NumberOfActiveArrays; a++) { while (_writing) new SpinWait().SpinOnce(); Array.Copy(_mdArray[a], temp._mdArray[a], Granularity); } temp._table.Count = Count; return temp; }); } public void FromArray(T[][] array) { Lock(this, () => { _table.NumberOfActiveArrays = array.GetUpperBound(0) + 1; var val = (long) _table.NumberOfActiveArrays * Granularity; _table.Length = Interlocked.Read(ref val); _mdArray = new T[_table.NumberOfActiveArrays][]; for (var i = 0; i < _table.NumberOfActiveArrays; ++i) _mdArray[i] = new T[Granularity]; for (var a = 0L; a < _table.NumberOfActiveArrays; a++) Array.Copy(array[a], _mdArray[a], Granularity); }); } public T[][] ToArray() { return Lock(this, () => { var ta = new T[_table.NumberOfActiveArrays][]; for (var i = 0; i < _table.NumberOfActiveArrays; ++i) ta[i] = new T[Granularity]; for (var a = 0L; a < _table.NumberOfActiveArrays; a++) Array.Copy(_mdArray[a], ta[a], Granularity); return ta; }); } private void Dispose(bool disposing) { if (!_disposed) if (disposing) _mdArray = null; _disposed = true; } public IEnumerator<T> GetEnumerator() { return Lock(this, () => { return GetEnum(); }); } public IEnumerator<T> GetEnum() { for (var i = 0; i < Count; i++) yield return this[i]; } private class Table { public long Count; public long Length; public volatile int MaximumNumberOfArrays; public volatile int NumberOfActiveArrays; } }
2 thoughts on “BigArray.cs”