Big Array Class Arrays Over 2GB Limit<\/p>\n\n\n\n
Updated: Dec-14,2020<\/p>\n\n\n\n
using System;\nusing System.Collections;\nusing System.Collections.Generic;\nusing System.Diagnostics;\nusing System.Threading;\nusing Microsoft.VisualBasic.Devices;\n[DebuggerDisplay(\"Count = {Count}\")]\n[Serializable]\npublic class BigArray<T> : MonitorActionFunc, IEnumerable<T>, IDisposable\n{\n public const int ShiftCount = 19;\n public const int Granularity = 1 << ShiftCount;\n private bool _disposed;\n private volatile T[][] _mdArray;\n private volatile Table _table = new Table();\n private volatile bool _writing;\n public BigArray() : this(0)\n {\n }\n public BigArray(long size)\n {\n if (size < Granularity)\n size = Granularity;\n var i = 0;\n try\n {\n _table.MaximumNumberOfArrays = (int) (new ComputerInfo().AvailablePhysicalMemory \/ Granularity);\n _table.NumberOfActiveArrays = (int) ((size + (Granularity - 1)) \/ Granularity);\n var val = (long) _table.NumberOfActiveArrays * Granularity;\n _table.Length = Interlocked.Read(ref val);\n var ms = new MeasureSize<T>();\n var oas = ms.GetByteSize(Granularity);\n var am = new ComputerInfo().AvailablePhysicalMemory;\n var rm = (ulong) (oas * _table.NumberOfActiveArrays * Granularity);\n var maa = am \/ (ulong) (oas * Granularity);\n if (rm > am || (ulong) _table.NumberOfActiveArrays > maa)\n throw new Exception($\"Requested memory {rm} exceeds available memory {am}\");\n _mdArray = new T[maa][];\n for (i = 0; i < _table.NumberOfActiveArrays; ++i)\n _mdArray[i] = new T[Granularity];\n _writing = false;\n }\n catch (Exception ex)\n {\n throw new Exception($\"Exception: {ex.Message}\");\n }\n }\n public long Count\n {\n get\n {\n return Lock(_table.Count, () =>\n {\n return _table.Count;\n });\n }\n }\n public long Length\n {\n get\n {\n return Lock(_table.Length, () =>\n {\n return _table.Length;\n });\n }\n }\n public T this[long index]\n {\n get\n {\n while (_writing)\n new SpinWait().SpinOnce();\n if (index >= _table.Length)\n throw new Exception($\"Getter: Index out of bounds, Index: '{index}' must be less than the Length: '{Length}'.\");\n return _mdArray[index >> ShiftCount][index & (Granularity - 1)];\n }\n set\n {\n Lock(this, () =>\n {\n if (index + 1 > _table.Length)\n ResizeArray();\n _writing = true;\n _mdArray[index >> ShiftCount][index & (Granularity - 1)] = value;\n _table.Count++;\n _writing = false;\n });\n }\n }\n public void Dispose()\n {\n Dispose(true);\n }\n IEnumerator<T> IEnumerable<T>.GetEnumerator()\n {\n return GetEnumerator();\n }\n IEnumerator IEnumerable.GetEnumerator()\n {\n return GetEnumerator();\n }\n public void Add(T Item)\n {\n Lock(this, () =>\n {\n if (_table.Count + 1 > _table.Length)\n ResizeArray();\n _writing = true;\n var x = _table.Count >> ShiftCount;\n var y = _table.Count & (Granularity - 1);\n _mdArray[x][y] = Item;\n _table.Count++;\n _writing = false;\n });\n }\n private void ResizeArray()\n {\n try\n {\n Interlocked.Increment(ref _table.NumberOfActiveArrays);\n var val = (long) _table.NumberOfActiveArrays * Granularity;\n _table.Length = Interlocked.Read(ref val);\n var ms = new MeasureSize<T>();\n var oas = ms.GetByteSize(Granularity);\n var am = new ComputerInfo().AvailablePhysicalMemory;\n var rm = (ulong) (oas * _table.NumberOfActiveArrays * Granularity);\n var maa = am \/ (ulong) (oas * Granularity);\n if (rm > am || (ulong) _table.NumberOfActiveArrays > maa)\n throw new Exception($\"Requested memory {rm} exceeds available memory {am}\");\n _mdArray = new T[maa][];\n _mdArray[_table.NumberOfActiveArrays - 1] = new T[Granularity];\n }\n catch (Exception ex)\n {\n throw new Exception($\"Exception: {ex.Message}\");\n }\n }\n public void Clear()\n {\n Lock(this, () =>\n {\n _writing = true;\n for (var a = 0L; a < _table.NumberOfActiveArrays; a++)\n Array.Clear(_mdArray[a], 0, Granularity);\n _table.Count = 0;\n _writing = false;\n });\n }\n public long IndexOf(T item)\n {\n return Lock(this, () =>\n {\n var i = 0L;\n for (; i < _table.NumberOfActiveArrays; i++)\n {\n while (_writing)\n new SpinWait().SpinOnce();\n var pos = Array.IndexOf(_mdArray[i], item, 0);\n if (pos != -1)\n return i * Granularity + pos;\n }\n return -1;\n });\n }\n public BigArray<T> Copy(long newsize)\n {\n return Lock(this, () =>\n {\n var temp = new BigArray<T>(newsize);\n for (var a = 0L; a < _table.NumberOfActiveArrays; a++)\n {\n while (_writing)\n new SpinWait().SpinOnce();\n Array.Copy(_mdArray[a], temp._mdArray[a], Granularity);\n }\n temp._table.Count = Count;\n return temp;\n });\n }\n public void FromArray(T[][] array)\n {\n Lock(this, () =>\n {\n _table.NumberOfActiveArrays = array.GetUpperBound(0) + 1;\n var val = (long) _table.NumberOfActiveArrays * Granularity;\n _table.Length = Interlocked.Read(ref val);\n _mdArray = new T[_table.NumberOfActiveArrays][];\n for (var i = 0; i < _table.NumberOfActiveArrays; ++i)\n _mdArray[i] = new T[Granularity];\n for (var a = 0L; a < _table.NumberOfActiveArrays; a++)\n Array.Copy(array[a], _mdArray[a], Granularity);\n });\n }\n public T[][] ToArray()\n {\n return Lock(this, () =>\n {\n var ta = new T[_table.NumberOfActiveArrays][];\n for (var i = 0; i < _table.NumberOfActiveArrays; ++i)\n ta[i] = new T[Granularity];\n for (var a = 0L; a < _table.NumberOfActiveArrays; a++)\n Array.Copy(_mdArray[a], ta[a], Granularity);\n return ta;\n });\n }\n private void Dispose(bool disposing)\n {\n if (!_disposed)\n if (disposing)\n _mdArray = null;\n _disposed = true;\n }\n public IEnumerator<T> GetEnumerator()\n {\n return Lock(this, () =>\n {\n return GetEnum();\n });\n }\n public IEnumerator<T> GetEnum()\n {\n for (var i = 0; i < Count; i++)\n yield return this[i];\n }\n private class Table\n {\n public long Count;\n public long Length;\n public volatile int MaximumNumberOfArrays;\n public volatile int NumberOfActiveArrays;\n }\n}<\/pre>\n","protected":false},"excerpt":{"rendered":"Big Array Class Arrays Over 2GB Limit Updated: Dec-14,2020<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[2],"tags":[139,35,138,84,87],"_links":{"self":[{"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/posts\/280"}],"collection":[{"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/comments?post=280"}],"version-history":[{"count":3,"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/posts\/280\/revisions"}],"predecessor-version":[{"id":283,"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/posts\/280\/revisions\/283"}],"wp:attachment":[{"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/media?parent=280"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/categories?post=280"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/michaeljohnsteiner.com\/index.php\/wp-json\/wp\/v2\/tags?post=280"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}