Big Array Class Arrays Over 2GB Limit
Updated: Dec-14,2020
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading;
using Microsoft.VisualBasic.Devices;
[DebuggerDisplay("Count = {Count}")]
[Serializable]
public class BigArray<T> : MonitorActionFunc, IEnumerable<T>, IDisposable
{
public const int ShiftCount = 19;
public const int Granularity = 1 << ShiftCount;
private bool _disposed;
private volatile T[][] _mdArray;
private volatile Table _table = new Table();
private volatile bool _writing;
public BigArray() : this(0)
{
}
public BigArray(long size)
{
if (size < Granularity)
size = Granularity;
var i = 0;
try
{
_table.MaximumNumberOfArrays = (int) (new ComputerInfo().AvailablePhysicalMemory / Granularity);
_table.NumberOfActiveArrays = (int) ((size + (Granularity - 1)) / Granularity);
var val = (long) _table.NumberOfActiveArrays * Granularity;
_table.Length = Interlocked.Read(ref val);
var ms = new MeasureSize<T>();
var oas = ms.GetByteSize(Granularity);
var am = new ComputerInfo().AvailablePhysicalMemory;
var rm = (ulong) (oas * _table.NumberOfActiveArrays * Granularity);
var maa = am / (ulong) (oas * Granularity);
if (rm > am || (ulong) _table.NumberOfActiveArrays > maa)
throw new Exception($"Requested memory {rm} exceeds available memory {am}");
_mdArray = new T[maa][];
for (i = 0; i < _table.NumberOfActiveArrays; ++i)
_mdArray[i] = new T[Granularity];
_writing = false;
}
catch (Exception ex)
{
throw new Exception($"Exception: {ex.Message}");
}
}
public long Count
{
get
{
return Lock(_table.Count, () =>
{
return _table.Count;
});
}
}
public long Length
{
get
{
return Lock(_table.Length, () =>
{
return _table.Length;
});
}
}
public T this[long index]
{
get
{
while (_writing)
new SpinWait().SpinOnce();
if (index >= _table.Length)
throw new Exception($"Getter: Index out of bounds, Index: '{index}' must be less than the Length: '{Length}'.");
return _mdArray[index >> ShiftCount][index & (Granularity - 1)];
}
set
{
Lock(this, () =>
{
if (index + 1 > _table.Length)
ResizeArray();
_writing = true;
_mdArray[index >> ShiftCount][index & (Granularity - 1)] = value;
_table.Count++;
_writing = false;
});
}
}
public void Dispose()
{
Dispose(true);
}
IEnumerator<T> IEnumerable<T>.GetEnumerator()
{
return GetEnumerator();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
public void Add(T Item)
{
Lock(this, () =>
{
if (_table.Count + 1 > _table.Length)
ResizeArray();
_writing = true;
var x = _table.Count >> ShiftCount;
var y = _table.Count & (Granularity - 1);
_mdArray[x][y] = Item;
_table.Count++;
_writing = false;
});
}
private void ResizeArray()
{
try
{
Interlocked.Increment(ref _table.NumberOfActiveArrays);
var val = (long) _table.NumberOfActiveArrays * Granularity;
_table.Length = Interlocked.Read(ref val);
var ms = new MeasureSize<T>();
var oas = ms.GetByteSize(Granularity);
var am = new ComputerInfo().AvailablePhysicalMemory;
var rm = (ulong) (oas * _table.NumberOfActiveArrays * Granularity);
var maa = am / (ulong) (oas * Granularity);
if (rm > am || (ulong) _table.NumberOfActiveArrays > maa)
throw new Exception($"Requested memory {rm} exceeds available memory {am}");
_mdArray = new T[maa][];
_mdArray[_table.NumberOfActiveArrays - 1] = new T[Granularity];
}
catch (Exception ex)
{
throw new Exception($"Exception: {ex.Message}");
}
}
public void Clear()
{
Lock(this, () =>
{
_writing = true;
for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
Array.Clear(_mdArray[a], 0, Granularity);
_table.Count = 0;
_writing = false;
});
}
public long IndexOf(T item)
{
return Lock(this, () =>
{
var i = 0L;
for (; i < _table.NumberOfActiveArrays; i++)
{
while (_writing)
new SpinWait().SpinOnce();
var pos = Array.IndexOf(_mdArray[i], item, 0);
if (pos != -1)
return i * Granularity + pos;
}
return -1;
});
}
public BigArray<T> Copy(long newsize)
{
return Lock(this, () =>
{
var temp = new BigArray<T>(newsize);
for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
{
while (_writing)
new SpinWait().SpinOnce();
Array.Copy(_mdArray[a], temp._mdArray[a], Granularity);
}
temp._table.Count = Count;
return temp;
});
}
public void FromArray(T[][] array)
{
Lock(this, () =>
{
_table.NumberOfActiveArrays = array.GetUpperBound(0) + 1;
var val = (long) _table.NumberOfActiveArrays * Granularity;
_table.Length = Interlocked.Read(ref val);
_mdArray = new T[_table.NumberOfActiveArrays][];
for (var i = 0; i < _table.NumberOfActiveArrays; ++i)
_mdArray[i] = new T[Granularity];
for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
Array.Copy(array[a], _mdArray[a], Granularity);
});
}
public T[][] ToArray()
{
return Lock(this, () =>
{
var ta = new T[_table.NumberOfActiveArrays][];
for (var i = 0; i < _table.NumberOfActiveArrays; ++i)
ta[i] = new T[Granularity];
for (var a = 0L; a < _table.NumberOfActiveArrays; a++)
Array.Copy(_mdArray[a], ta[a], Granularity);
return ta;
});
}
private void Dispose(bool disposing)
{
if (!_disposed)
if (disposing)
_mdArray = null;
_disposed = true;
}
public IEnumerator<T> GetEnumerator()
{
return Lock(this, () =>
{
return GetEnum();
});
}
public IEnumerator<T> GetEnum()
{
for (var i = 0; i < Count; i++)
yield return this[i];
}
private class Table
{
public long Count;
public long Length;
public volatile int MaximumNumberOfArrays;
public volatile int NumberOfActiveArrays;
}
}