Update, fixes

Fixed #8
Fixed #7

Fixed a bug in the parsing of the domain option.

Added referer option, which works very much like the domains option, except it operates against the referer field of the headers when doing a match. Thus it is possible to have a global referer rule, or domain specific referer rules, etc etc.

Author: TechnikEmpire

DistillNET/DistillNET/DistillNET.nuspec

@@ -2,10 +2,10 @@
 <package >
   <metadata>
     <id>DistillNET</id>
-    <version>1.1.4</version>
+    <version>1.3.1</version>
     <title>DistillNET</title>
     <authors>TechnikEmpire</authors>
-	<releaseNotes>Modified in memory connection string, as well as filter lookup API, to enable threaded reads. Connections are now created on read (pooled) to comply with one transaction per connection.</releaseNotes>
+	<releaseNotes>Fixed an error in the parsing of the domains option. Added a new referer option.</releaseNotes>
     <owners>TechnikEmpire</owners>
 	<projectUrl>https://github.com/TechnikEmpire/DistillNET</projectUrl>
 	<requireLicenseAcceptance>true</requireLicenseAcceptance>

DistillNET/DistillNET/DistillNET/AbpFormatRuleParser.cs

@@ -227,8 +227,11 @@ private Filter ParseUrlFilter(string rule, int optionsStartOffset, bool hasOptio
             string originalRuleCopy = rule;
 
             string[] allOptions = null;
-            List<string> applicableDomains = new List<string>();
-            List<string> exceptionDomains = new List<string>();
+            HashSet<string> applicableReferers = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            HashSet<string> exceptReferers = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+            HashSet<string> applicableDomains = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            HashSet<string> exceptionDomains = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
 
             // Trim off the leading "@@" chracters if it's an exception.
             if(isException)
@@ -250,16 +253,24 @@ private Filter ParseUrlFilter(string rule, int optionsStartOffset, bool hasOptio
             var enumOptions = UrlFilter.UrlFilterOptions.None;
 
             if(allOptions != null)
-            {
-                string domainsOption = null;//allOptions.Where(r => r.IndexOf('=') != -1).FirstOrDefault();
+            {   
+                string domainsOption = null;
+
+                string refererOption = null;
+
                 var allOptLen = allOptions.Length;
                 for(int i = 0; i < allOptLen; ++i)
                 {
-                    if(allOptions[i].Length > 7 && allOptions[i][6] == '=')
+                    if(allOptions[i].Length > 7 && allOptions[i][0] == 'd' && allOptions[i][7] == '=')
                     {
                         domainsOption = allOptions[i];
                         allOptions[i] = string.Empty;
-                        break;
+                    }
+
+                    if(allOptions[i].Length > 7 && allOptions[i][0] == 'r' && allOptions[i][7] == '=')
+                    {
+                        refererOption = allOptions[i];
+                        allOptions[i] = string.Empty;
                     }
                 }
 
@@ -268,13 +279,11 @@ private Filter ParseUrlFilter(string rule, int optionsStartOffset, bool hasOptio
                     // If we got a domains option, split it out of the main options collection, as
                     // this is the only type of option that's special aka has to parsed differently.
                     // Differentiate simply by string length for speed, but do ordinal compare when
-                    // lengths are equal. allOptions = allOptions.Where(r => (r.Length !=
-                    // domainsOption.Length || !r.Equals(domainsOption,
-                    // StringComparison.OrdinalIgnoreCase))).ToArray();
+                    // lengths are equal. 
 
                     // Trim off the "domains=" part, then split by the domains delimiter, which is a
                     // pipe.
-                    domainsOption = domainsOption.Substring(7);
+                    domainsOption = domainsOption.Substring(8);
                     var rawDomains = domainsOption.Split(s_domainsDelim, StringSplitOptions.None);
 
                     // Get applicable and exception domains. Exception domains in the list start with tilde,
@@ -301,6 +310,42 @@ private Filter ParseUrlFilter(string rule, int optionsStartOffset, bool hasOptio
                     }
                 }
 
+                if(refererOption != null)
+                {   
+                    // If we got a referers option, split it out of the main options collection, as
+                    // this is the only type of option that's special aka has to parsed differently.
+                    // Differentiate simply by string length for speed, but do ordinal compare when
+                    // lengths are equal. 
+
+                    // Trim off the "referer=" part, then split by the domains delimiter, which is a
+                    // pipe.
+                    refererOption = refererOption.Substring(8);
+                    var rawReferers = refererOption.Split(s_domainsDelim, StringSplitOptions.None);
+
+                    // Get applicable and exception referers. Exception referers in the list start with tilde,
+                    // applicable referers don't. Applicable here meaning that the rule should apply to such
+                    // a domain.
+
+                    var referersLen = rawReferers.Length;
+                    for(int i = 0; i < referersLen; ++i)
+                    {
+                        switch(rawReferers[i][0])
+                        {
+                            case '~':
+                            {
+                                exceptReferers.Add(rawReferers[i].Substring(1));
+                            }
+                            break;
+
+                            default:
+                            {
+                                applicableReferers.Add(rawReferers[i]);
+                            }
+                            break;
+                        }
+                    }
+                }
+
                 // Parse out the rest of the options.
                 UrlFilter.UrlFilterOptions asOpt;
                 foreach(var opt in allOptions)
@@ -452,7 +497,7 @@ private Filter ParseUrlFilter(string rule, int optionsStartOffset, bool hasOptio
                 compiledParts.Add(new UrlFilter.StringLiteralFragment(rule.Substring(lastCol), enumOptions.HasFlag(UrlFilter.UrlFilterOptions.MatchCase)));
             }
 
-            return new UrlFilter(originalRuleCopy, compiledParts, enumOptions, applicableDomains, exceptionDomains, isException, categoryId);
+            return new UrlFilter(originalRuleCopy, compiledParts, enumOptions, applicableDomains, exceptionDomains, applicableReferers, exceptReferers, isException, categoryId);
         }
     }
 }

DistillNET/DistillNET/DistillNET/FilterDbCollection.cs

@@ -23,7 +23,7 @@ namespace DistillNET
     /// rather than serialized/deserialized, because the parser is much faster than such utilities
     /// such as protobuf.
     /// </summary>
-    public class FilterDbCollection
+    public class FilterDbCollection : IDisposable
     {
         /// <summary>
         /// Our rule parser.
@@ -217,7 +217,7 @@ public async Task<Tuple<int, int>> ParseStoreRules(string[] rawRuleStrings, shor
                         if(filter.ApplicableDomains.Count > 0)
                         {
                             foreach(var dmn in filter.ApplicableDomains)
-                            {
+                            {   
                                 cmd.Parameters[0].Value = dmn;
                                 cmd.Parameters[1].Value = categoryId;
                                 cmd.Parameters[2].Value = filter.IsException;
@@ -428,5 +428,44 @@ public List<Filter> GetFiltersForRequest(Uri requestString, string referer = "")
         {
             return null;
         }
+
+        #region IDisposable Support
+        private bool disposedValue = false; // To detect redundant calls
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if(!disposedValue)
+            {
+                if(disposing)
+                {
+                    if(m_connection != null)
+                    {
+                        m_connection.Close();
+                        m_connection = null;
+                    }
+                }
+
+                // TODO: free unmanaged resources (unmanaged objects) and override a finalizer below.
+                // TODO: set large fields to null.
+
+                disposedValue = true;
+            }
+        }
+
+        // TODO: override a finalizer only if Dispose(bool disposing) above has code to free unmanaged resources.
+        // ~FilterDbCollection() {
+        //   // Do not change this code. Put cleanup code in Dispose(bool disposing) above.
+        //   Dispose(false);
+        // }
+
+        // This code added to correctly implement the disposable pattern.
+        public void Dispose()
+        {
+            // Do not change this code. Put cleanup code in Dispose(bool disposing) above.
+            Dispose(true);
+            // TODO: uncomment the following line if the finalizer is overridden above.
+            // GC.SuppressFinalize(this);
+        }
+        #endregion
     }
 }

DistillNET/DistillNET/DistillNET/UrlFilter.cs

@@ -460,21 +460,41 @@ public override int IsMatch(Uri source, int lastPosition)
         }
 
         /// <summary>
-        /// Gets an array of all domains that this URL filter rule applies to. In the event that this
+        /// Gets a hashset of all referers that this URL filter rule applies to. In the event that
+        /// this array is empty, the referer field on requests will not be checked.
+        /// </summary>
+        public HashSet<string> ApplicableReferers
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Gets a hashset of all referers that this URL filter rule applies to. In the event that
+        /// this array is empty, the referer field on requests will not be checked.
+        /// </summary>
+        public HashSet<string> ExceptReferers
+        {
+            get;
+            private set;
+        }
+
+        /// <summary>
+        /// Gets a hashset of all domains that this URL filter rule applies to. In the event that this
         /// array is empty, the rule applies globally, to all domains.
         /// </summary>
-        public List<string> ApplicableDomains
+        public HashSet<string> ApplicableDomains
         {
             get;
             private set;
         }
 
         /// <summary>
-        /// Gets an array of all domains that this URL filter should not be applied to. In the event
+        /// Gets a hashset of all domains that this URL filter should not be applied to. In the event
         /// that this array is empty, the rule applies either globally, or exclusively to the list of
         /// applicable domains, if that property is not empty.
         /// </summary>
-        public List<string> ExceptionDomains
+        public HashSet<string> ExceptionDomains
         {
             get;
             private set;
@@ -538,13 +558,16 @@ public List<UrlFilteringRuleFragment> Parts
         /// <param name="categoryId">
         /// The category ID of the category this filter belongs to.
         /// </param>
-        internal UrlFilter(string originalRule, List<UrlFilteringRuleFragment> parts, UrlFilterOptions options, List<string> applicableDomains, List<string> exceptionDomains, bool isException, short categoryId) : base(originalRule, isException, categoryId)
+        internal UrlFilter(string originalRule, List<UrlFilteringRuleFragment> parts, UrlFilterOptions options, HashSet<string> applicableDomains, HashSet<string> exceptionDomains, HashSet<string> applicableReferers, HashSet<string> exceptionReferers, bool isException, short categoryId) : base(originalRule, isException, categoryId)
         {
             Parts = parts;
             Options = options;
 
             ApplicableDomains = applicableDomains;
             ExceptionDomains = exceptionDomains;
+
+            ApplicableReferers = applicableReferers;
+            ExceptReferers = exceptionReferers;
         }
 
         /// <summary>
@@ -564,23 +587,23 @@ public bool IsMatch(Uri uri, NameValueCollection rawHeaders)
         {
             // Make sure that the headers match up with our options.
             if(this.Options != UrlFilterOptions.None)
-            {   
+            {
                 string headerVal = null;
                 long xmlHttpRequestBits = ((OptionsLong & (long)UrlFilterOptions.ExceptXmlHttpRequest) | (OptionsLong & (long)UrlFilterOptions.XmlHttpRequest));
                 if((headerVal = rawHeaders.Get("X-Requested-With")) != null)
                 {
                     if(headerVal.Equals("XMLHttpRequest", StringComparison.OrdinalIgnoreCase))
-                    {   
+                    {
                         xmlHttpRequestBits &= ~(long)UrlFilterOptions.XmlHttpRequest;
                     }
                     else
-                    {   
+                    {
                         xmlHttpRequestBits &= ~(long)UrlFilterOptions.ExceptXmlHttpRequest;
                     }
                 }
 
                 if(xmlHttpRequestBits != 0)
-                {   
+                {
                     // XML HttpRequest bits were not cleared, meaning that one of those options was not satisifed.
                     return false;
                 }
@@ -596,6 +619,21 @@ public bool IsMatch(Uri uri, NameValueCollection rawHeaders)
                     {
                         thirdPartyBits &= ~(long)UrlFilterOptions.ThirdParty;
                     }
+
+                    // While we have the referer field, let's go ahead and check if we have referer
+                    // options and if we do or don't have a match.
+                    //
+                    // This is a shortcut. We unfortunately need to also execute this code also when
+                    // there are no options.
+                    if(ApplicableReferers.Count > 0 && !ApplicableReferers.Contains(headerVal))
+                    {
+                        return false;
+                    }
+
+                    if(ExceptReferers.Count > 0 && ExceptReferers.Contains(headerVal))
+                    {
+                        return false;
+                    }
                 }
                 else
                 {
@@ -617,7 +655,7 @@ public bool IsMatch(Uri uri, NameValueCollection rawHeaders)
                 if((headerVal = rawHeaders.Get("Content-Type")) != null)
                 {
                     if(headerVal.IndexOfQuick("script") != -1)
-                    {   
+                    {
                         contentTypeBits &= ~(long)UrlFilterOptions.Script;
                     }
                     else
@@ -650,7 +688,31 @@ public bool IsMatch(Uri uri, NameValueCollection rawHeaders)
                     return false;
                 }
             }
-            
+            else
+            {
+                if(ApplicableReferers.Count > 0 || ExceptReferers.Count > 0)
+                {
+                    string headerVal = null;
+                    if((headerVal = rawHeaders.Get("Referer")) != null)
+                    {
+                        // While we have the referer field, let's go ahead and check if we have referer
+                        // options and if we do or don't have a match.
+                        //
+                        // This is a shortcut. We unfortunately need to also execute this code also when
+                        // there are no options.
+                        if(ApplicableReferers.Count > 0 && !ApplicableReferers.Contains(headerVal))
+                        {
+                            return false;
+                        }
+
+                        if(ExceptReferers.Count > 0 && ExceptReferers.Contains(headerVal))
+                        {
+                            return false;
+                        }
+                    }
+                }
+            }
+
             int matchIndex = 0;
             foreach(var part in Parts)
             {   

DistillNET/DistillNET/Properties/AssemblyInfo.cs

@@ -6,12 +6,10 @@
  */
 
 using System.Reflection;
-using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
-// General Information about an assembly is controlled through the following 
-// set of attributes. Change these attribute values to modify the information
-// associated with an assembly.
+// General Information about an assembly is controlled through the following set of attributes.
+// Change these attribute values to modify the information associated with an assembly.
 [assembly: AssemblyTitle("DistillNET")]
 [assembly: AssemblyDescription("A library for matching and filtering HTTP requests and HTML response content using the Adblock Plus Filter format.")]
 [assembly: AssemblyConfiguration("")]
@@ -21,23 +19,18 @@
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
-// Setting ComVisible to false makes the types in this assembly not visible 
-// to COM components.  If you need to access a type in this assembly from 
-// COM, set the ComVisible attribute to true on that type.
+// Setting ComVisible to false makes the types in this assembly not visible to COM components. If you
+// need to access a type in this assembly from COM, set the ComVisible attribute to true on that type.
 [assembly: ComVisible(false)]
 
 // The following GUID is for the ID of the typelib if this project is exposed to COM
 [assembly: Guid("66aefe15-e2ac-4ce1-a83e-4e3e3ca2f9de")]
 
 // Version information for an assembly consists of the following four values:
 //
-//      Major Version
-//      Minor Version 
-//      Build Number
-//      Revision
+// Major Version Minor Version Build Number Revision
 //
-// You can specify all the values or you can default the Build and Revision Numbers 
-// by using the '*' as shown below:
-// [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.1.5.0")]
-[assembly: AssemblyFileVersion("1.1.5.0")]
+// You can specify all the values or you can default the Build and Revision Numbers by using the '*'
+// as shown below: [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.3.1.0")]
+[assembly: AssemblyFileVersion("1.3.1.0")]