Tech Shorts

While working with the FileSystemWatcher I found that if too many files were created the built in buffer will overflowed and files will be skipped.  After much research I found out about the Producer-Consumer Problem.  Then I found that .Net 4 has the BlockingCollection which helps solve the issue.  But how to use it with the FileSystemWatcher?

On StackOverflow I found Making PLINQ and BlockingCollection work together.  I’m not so interested in the PLINQ issue but this is a great example of using The BlockingCollection with FileSystemWatcher.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;

namespace ConsoleApplication4
{
    public class Program
    {
        private const string Folder = "C:\\Temp\\InputData";

        static void Main(string[] args) {

            var cts = new CancellationTokenSource();
            foreach (var obj in Input(cts.Token))
                Console.WriteLine(obj);
        }

        public static IEnumerable<object> Input(CancellationToken cancellationToken) {
            var fileList = new BlockingCollection<string>();

            var watcher = new FileSystemWatcher(Folder);
            watcher.Created += (source, e) => {
                if (cancellationToken.IsCancellationRequested)
                    watcher.EnableRaisingEvents = false;
                else if (Path.GetFileName(e.FullPath) == "STOP") {
                    watcher.EnableRaisingEvents = false;
                    fileList.CompleteAdding();
                    File.Delete(e.FullPath);
                } else
                    fileList.Add(e.FullPath);
            };
            watcher.EnableRaisingEvents = true;

            return from file in
                       fileList
                            .GetConsumingEnumerable(cancellationToken)
                            .AsParallel()
                            .WithMergeOptions(ParallelMergeOptions.NotBuffered)
                            .WithCancellation(cancellationToken)
                            .WithDegreeOfParallelism(5)
                   let obj = CreateMyObject(file)
                   select obj;
        }

        private static object CreateMyObject(string file) {
            return file;
        }
    }
}

Threw together a quick parallel stopwatch test. Not sure if the times prove anything.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;

namespace Scratch.ParallelProcessing
{
    class Program
    {
        static void Main(string[] args)
        {
            const int count = 10000000;
            var source1 = Enumerable.Range(0, count).ToArray();
            var source2 = Enumerable.Range(0, count).ToArray();
            var source3 = Enumerable.Range(0, count).ToArray();

            Stopwatch stopwatch = new Stopwatch();

            var parallelElapsedTimes = new List<TimeSpan>();
            var linearElapsedTimes = new List<TimeSpan>();
            var linqSelectElapsedTimes = new List<TimeSpan>();

            for (int i = 0; i < 10; i++)
            {
                stopwatch.Reset();
                stopwatch.Start();
                var parallelResults = Parallel.ForEach(source1, s => s %= 2);
                parallelElapsedTimes.Add(stopwatch.Elapsed);
                stopwatch.Reset();

                stopwatch.Start();
                LinearAction(source2, s => s %= 2);
                linearElapsedTimes.Add(stopwatch.Elapsed);
                stopwatch.Reset();

                stopwatch.Reset();
                stopwatch.Start();
                Array.ForEach(source3, s=>s = s%2);
                linqSelectElapsedTimes.Add(stopwatch.Elapsed);
                stopwatch.Reset();

            }

            Console.WriteLine("Elapsed Time\t\tMin\t\tMax\t\t\tAvg");
            Console.WriteLine("============\t\t===\t\t===\t\t\t===");
            Console.WriteLine("{0}\t\t{1}\t\t{2}\t\t\t{3}", "Parallel", parallelElapsedTimes.Min(t => t.Milliseconds), parallelElapsedTimes.Max(t => t.Milliseconds), parallelElapsedTimes.Average(t => t.Milliseconds));
            Console.WriteLine("{0}\t\t\t{1}\t\t{2}\t\t\t{3}", "Linear", linearElapsedTimes.Min(t => t.Milliseconds), linearElapsedTimes.Max(t => t.Milliseconds), linearElapsedTimes.Average(t => t.Milliseconds));
            Console.WriteLine("{0}\t\t\t{1}\t\t{2}\t\t\t{3}", "Linq", linqSelectElapsedTimes.Min(t => t.Milliseconds), linqSelectElapsedTimes.Max(t => t.Milliseconds), linqSelectElapsedTimes.Average(t => t.Milliseconds));

        }

        public static void LinearAction<T>(IEnumerable<T> source, Action<T> action)
        {
            foreach (var s in source) action(s);
        }
    }
}

Results of the timer:

Elapsed Time Min Max Avg
============ === === ===
Parallel 63 191 79.5
Linear 138 143 140.3
Linq 54 56 54.5
Press any key to continue . . .


I’m running 64 bit Vista on a Intel Core2 Duo with 4GB RAM. The Parallel seems to be inconsistent, and depends a lot on whether or not it grabs that second CPU.

I’ve seen a lot of code to generate SQL statements. Invariable the programmer has an array of strings that they loop through (for example to put into an IN clause) and they always have a check to see of the current item is the first or last in the list. The typical usage is to have a StringBuilder and an if statement which determines if an extra comma (or plus sign or whatever) is added or left out.

I say: Stop Doing That!

Use the string.Join().

var strings = new[] {  "Darren", "Dawn", "Thomas", "Zoey" };

var results = string.Format("Replace \"{0}\" with {1} Question Marks: ({2})",
	string.Join(",", strings), strings.Length,
	string.Join(",", Enumerable.Repeat("?", strings.Length))
	);

Console.WriteLine(results);

The resulting output is:

Replace "Darren,Dawn,Thomas,Zoey" with 4 Question Marks: (?,?,?,?)

Damon Payne has a post AddRange for ObservableCollection in Silverlight 3. It is pretty short and sweet way to improve performance when batch adding data to an ObservableCollection.

He created a SmartCollection:

    public class SmartCollection<T> : ObservableCollection<T>
    {
        public SmartCollection()
        {
            _suspendCollectionChangeNotification = false;
        }

        bool _suspendCollectionChangeNotification;

        protected override void OnCollectionChanged(NotifyCollectionChangedEventArgs e)
        {
            if (!_suspendCollectionChangeNotification)
            {
                base.OnCollectionChanged(e);
            }
        }

        public void SuspendCollectionChangeNotification()
        {
            _suspendCollectionChangeNotification = true;
        }

        public void ResumeCollectionChangeNotification()
        {
            _suspendCollectionChangeNotification = false;
        }

        public void AddRange(IEnumerable<T> items)
        {
            this.SuspendCollectionChangeNotification();
            try
            {
                foreach (var i in items)
                    base.InsertItem(Count, i);
            }
            finally
            {
                this.ResumeCollectionChangeNotification();

                var arg = new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset);
                this.OnCollectionChanged(arg);
            }
        }
    }

I created a scratch / test app to see if it works:

<UserControl
    x:Class="SilverlightApplication10.MainPage"
    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
    xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
    xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
    xmlns:my="clr-namespace:System.Windows.Controls;assembly=System.Windows.Controls.Data"
    mc:Ignorable="d"
    d:DesignHeight="300" d:DesignWidth="400">

    <Grid x:Name="LayoutRoot" Background="White">
        <StackPanel Orientation="Horizontal">
            <StackPanel Margin="5">
                <StackPanel Orientation="Horizontal" Margin="5">
                    <TextBlock x:Name="SlowText" Height="50"  Width="100"/>
                    <Button x:Name="SlowButton" Content="Slow" Click="SlowButtonClick"/>
                </StackPanel>
                <my:DataGrid x:Name="SlowDataView"
                             Height="300"
                             AutoGenerateColumns="True"
                             RowBackground="Cornsilk"
                             AlternatingRowBackground="LightGray"/>
            </StackPanel>

            <StackPanel Margin="5">
                <StackPanel Orientation="Horizontal" Margin="5">
                    <TextBlock x:Name="FastText" Height="50"  Width="100"/>
                    <Button x:Name="FastButton" Content="Fast" Click="FastButtonClick"/>
                </StackPanel>
                <my:DataGrid x:Name="FastDataView"
                             Height="300"
                             AutoGenerateColumns="True"
                             RowBackground="Cornsilk"
                             AlternatingRowBackground="LightGray"/>
            </StackPanel>
        </StackPanel>
    </Grid>
</UserControl>

    public partial class MainPage : UserControl
    {
        public MainPage()
        {
            InitializeComponent();
            _slowCollection = new ObservableCollection<MyClass>();
            SlowDataView.ItemsSource = _slowCollection;

            _fastCollection = new SmartCollection<MyClass>();
            FastDataView.ItemsSource = _fastCollection;
        }

        #region Slow

        private readonly ObservableCollection<MyClass> _slowCollection;
        private void SlowButtonClick(object sender, RoutedEventArgs e)
        {
            var timeSpan = Time(() => { for (int i = 0; i < 25000; i++) _slowCollection.Add(MyClass.BuildRandom()); });
            Dispatcher.BeginInvoke(() => this.SlowText.Text = timeSpan.ToString());
        }

        #endregion

        #region Fast

        private readonly SmartCollection<MyClass> _fastCollection;
        private void FastButtonClick(object sender, RoutedEventArgs e)
        {

            var timeSpan = Time(() =>
                                    {
                                        var list = new List<MyClass>();
                                        for (int i = 0; i < 25000; i++) list.Add(MyClass.BuildRandom());
                                        _fastCollection.AddRange(list);
                                    });

            Dispatcher.BeginInvoke(() => this.FastText.Text = timeSpan.ToString());
        }

        #endregion

        private static TimeSpan Time(Action a)
        {
            var start = DateTime.Now;
            a();
            var end = DateTime.Now;
            return end - start;
        }
    }

    internal class MyClass : INotifyPropertyChanged
    {
        private static Random _random;

        #region Id

        private string _id;

        public string Id
        {
            get { return _id; }
            set
            {
                if (value != _id)
                {
                    _id = value;
                    NotifyPropertyChanged("Id");
                }
            }
        }

        #endregion

        #region SomeRandomNumber

        private int _someRandomNumber;

        public int SomeRandomNumber
        {
            get { return _someRandomNumber; }
            set
            {
                if (value != _someRandomNumber)
                {
                    _someRandomNumber = value;
                    NotifyPropertyChanged("SomeRandomNumber");
                }
            }
        }

        #endregion

        #region INotifyPropertyChanged

        public event PropertyChangedEventHandler PropertyChanged;

        public void NotifyPropertyChanged(string propertyName)
        {
            var propertyChanged = PropertyChanged;
            if (propertyChanged != null)
                propertyChanged(this, new PropertyChangedEventArgs(propertyName));
        }

        #endregion

        public static MyClass BuildRandom()
        {
            if(_random==null)
                _random = new Random((int) DateTime.Now.Ticks);

            var myclass = new MyClass
                              {
                                  Id = Guid.NewGuid().ToString("N"),
                                  SomeRandomNumber = _random.Next(1, 100)
                              };

            return myclass;
        }
    }

Running the code shows that indeed the AddRange works well to speed up inserts. I was getting about 15 to 25 seconds on the Slow Data and from 3 to 0.09 seconds on the Fast Data. It was interesting to note that if the Slow button was pressed a second time it actually took longer than the first click, but the Fast button had the opposite affect.