Odtnhj

I'm trying to serialize a big data. I'm getting an error OutOfMemoryException when serialize data. I did some researches. I found some information that it is said that there is an error about MemoryStream. Because memorystream supports 2GB data size. It has Capacity as Integer. I found another class can be used instead of memory stream. It is called MemoryTributary. I tried this class. But it gave me really bad performance. Can I find a framework or another thing like memorystream for big data ?

I would like to share what I did shortly :

• Get Data From SQLServer


• Convert Data To A Structure


• Serialize Data


• Compress Data By Using LZ4 Algorithm


• Upload Data To Azure

• Get Data From Azure


• Decompress Data By Using LZ4 Algorithm


• Deserialize Data


• Use data for operation

I have some codes like below :

 public static Stream ConvertObjectToStream<T>(T obj)
 
 if (obj == null) return null;
 var boisSerializer = new BoisSerializer();
 try
 
 var mem = new MemoryStream();
 boisSerializer.Serialize(obj, mem);
 return new MemoryStream(mem.ToArray()); 
 
 catch (OutOfMemoryException)
 
 var mem = new MemoryTributary();
 boisSerializer.Serialize(obj, mem);
 return new MemoryTributary(mem.ToArray());
 
 

 public static Stream ConvertObjectToStream(object obj)
 
 if (obj == null) return null;
 var boisSerializer = new BoisSerializer();
 try
 
 var mem = new MemoryStream();
 boisSerializer.Serialize(obj, obj.GetType(), mem);
 return new MemoryStream(mem.ToArray());
 
 catch (OutOfMemoryException)
 
 var mem = new MemoryTributary();
 boisSerializer.Serialize(obj, mem);
 return new MemoryTributary(mem.ToArray());
 
 

 public static T ConvertStreamToObject<T>(Stream stream)
 
 stream.Position = 0;
 var boisSerializer = new BoisSerializer();
 var o = boisSerializer.Deserialize<T>(stream);
 return o;
 

 public static object ConvertStreamToObject(Stream stream, Type type)
 
 stream.Position = 0;
 var boisSerializer = new BoisSerializer();
 object o = boisSerializer.Deserialize(stream, type);
 return o;
 

 public static Stream CompressStream(this Stream stream, LZ4Level lz4Level = LZ4Level.L00_FAST)
 
 stream.Position = 0;
 var ms = new MemoryStream();
 var settings = new LZ4EncoderSettings CompressionLevel = lz4Level ;
 LZ4EncoderStream target = LZ4Stream.Encode(ms, settings);
 stream.CopyTo(target);
 target.Dispose();
 ms.Dispose();
 return new MemoryStream(ms.ToArray());
 

 public static Stream DecompressStream(this Stream stream)
 
 var ms = new MemoryStream();
 stream.Position = 0;
 LZ4DecoderStream source = LZ4Stream.Decode(stream);
 source.CopyTo(ms);
 source.Dispose();
 ms.Dispose();
 return new MemoryStream(ms.ToArray());

 

public async Task UploadValueToContainer<T>(string containerName, string blobName, T obj)
 
 Stream stream = Core.Tools.Helpers.ConvertObjectToStream(obj).CompressStream();
 CloudBlobContainer container = _blobClient.GetContainerReference(containerName);
 if (!container.Exists()) throw new Exception($"containerName Container has not been found.");
 CloudBlockBlob blobData = container.GetBlockBlobReference(blobName);
 await blobData.UploadFromStreamAsync(stream);
 

 public async Task<T> DownloadValueFromContainer<T>(string containerName, string blobName) where T : new()
 
 CloudBlobContainer container = _blobClient.GetContainerReference(containerName);
 if (!container.Exists()) throw new Exception($"containerName Container has not been found.");
 CloudBlockBlob blobData = container.GetBlockBlobReference(blobName);
 if (!blobData.Exists())
 
 return new T();
 

 try
 
 var ms = new MemoryStream();
 await blobData.DownloadToStreamAsync(ms);
 return Core.Tools.Helpers.ConvertStreamToObject<T>(ms.DecompressStream());
 
 catch (OutOfMemoryException)
 
 var ms = new MemoryTributary();
 await blobData.DownloadToStreamAsync(ms);
 return Core.Tools.Helpers.ConvertStreamToObject<T>(ms.DecompressStream());