c# - Better Architecture for .Net Performance : Caching, Serializing or Writing to disk? -
after having hard times performance in winform app, decided redesign separate processing step visual reporting step.
picture worth thousand words, drew purpose of question :
needless computing/visualisation layers separated, problem first configuration amount of data stored in live memory displayed on flow growing, growing, taking more , more ram space, resulted in computing part squeezed , taking longer , longer (linearly in time)
that's why came new design. proved helpful , efficient : processing first, using 100% memory compute, , handling stored results displayed ( a la "generate report" button)
so here simple question : options that, , performance-friendly, (caching, serializing, storing in files reading them back, lazy-loading....)
thanks in advance
edit :
data formed of simple , plain line records (actually times series) stored in csv format example
with edit, sounds data pretty simple flat records. since volume high, @ file-based storage - becomes matter of format. csv/tsv pretty easy write, , suitable streaming readers (like this one) won't have buffer data in memory first. of course, if displays need data in memory, goes out window.
for large volume, sheer bandwidth becomes main bottleneck, plus processing time. csv , friends typically compress (gzip etc) nicely, take cpu. personally, i'm long-time fan of "protocol buffers", , seem nice fit here - well-suited streaming access, low bandwidth, , binary - benefits of smaller io costs without cpu overhead of compression. plus more object-based, easier apply existing data without interpretation steps.
i'm not entirely sure if i'm answering same question you're asking - i'll stop there - please feel free clarify i've got horribly wrong.
here's basic test rig show typical usages of 2 methods discussed above:
using system; using system.collections.generic; using system.diagnostics; using system.io; using system.text; using protobuf; static class program { static void main() { var rand = new random(123456); var timeorigin = new datetime(2010,1,1); serializer.prepareserializer<myfunrecord>(); console.writeline("writing .proto ..."); const int loop = 500000; using (var file = file.create("raw.data")) { var watch = stopwatch.startnew(); double total = 0; (int = 0; < loop; i++) { var obj = new myfunrecord(); obj.id = i; obj.count = rand.next(500); obj.value = rand.nextdouble() * 4000; obj.when = timeorigin.adddays(rand.next(1000)); obj.name = randomstring(rand); serializer.serializewithlengthprefix(file, obj, prefixstyle.base128, serializer.listitemtag); total += obj.value; } watch.stop(); console.writeline(file.length / (1024 * 1024)+ "mb"); console.writeline(total + " (check)"); console.writeline(watch.elapsedmilliseconds + "ms"); } rand = new random(123456); console.writeline(); console.writeline("writing tsv ..."); using (var file = file.create("raw.tsv")) { using (var writer = new streamwriter(file)) { var watch = stopwatch.startnew(); double total = 0; (int = 0; < loop; i++) { var obj = new myfunrecord(); obj.id = i; obj.count = rand.next(500); obj.value = rand.nextdouble() * 4000; obj.when = timeorigin.adddays(rand.next(1000)); obj.name = randomstring(rand); write(writer, obj); total += obj.value; } watch.stop(); console.writeline(file.length / (1024 * 1024) + "mb"); console.writeline(total + " (check)"); console.writeline(watch.elapsedmilliseconds + "ms"); } } console.writeline(); console.writeline("reading .proto ..."); using(var file = file.openread("raw.data")) { var watch = stopwatch.startnew(); double total = 0; foreach (var obj in serializer.deserializeitems<myfunrecord>(file, prefixstyle.base128, serializer.listitemtag)) { total += obj.value; } watch.stop(); console.writeline(total + " (check again)"); console.writeline(watch.elapsedmilliseconds + "ms"); } console.writeline(); console.writeline("reading tsv ..."); using (var file = file.openread("raw.tsv")) using (var reader = new streamreader(file)) { var watch = stopwatch.startnew(); double total = 0; foreach (var obj in read(reader)) { total += obj.value; } watch.stop(); console.writeline(total + " (check again)"); console.writeline(watch.elapsedmilliseconds + "ms"); } } private static void write(textwriter writer, myfunrecord obj) { writer.write(obj.id); writer.write('\t'); writer.write(obj.name); writer.write('\t'); writer.write(obj.when); writer.write('\t'); writer.write(obj.value); writer.write('\t'); writer.write(obj.count); writer.writeline(); } private static ienumerable<myfunrecord> read(textreader reader) { string line; char[] delim = new[] { '\t' }; while ((line = reader.readline()) != null) { string[] parts = line.split(delim); var obj = new myfunrecord(); obj.id = int.parse(parts[0]); obj.name = parts[1]; obj.when = datetime.parse(parts[2]); obj.value = double.parse(parts[3]); obj.count = int.parse(parts[4]); yield return obj; } } static string randomstring(random rand) { int len = rand.next(1, 20); var sb = new stringbuilder(len); (int = 0; < len; i++) { sb.append('a' + rand.next(26)); } return sb.tostring(); } } [protocontract] class myfunrecord { [protomember(1)]public int id { get; set; } [protomember(2)]public string name { get; set; } [protomember(3)] public datetime when { get; set; } [protomember(4)] public double value { get; set; } [protomember(5)] public int count { get; set; } }
Comments
Post a Comment